exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tool pipelines;

Browse Source
This commit is contained in:
extio-de
2024-12-19 19:20:45 +01:00
parent bfb4e8be3b
commit 74eaa5c72b
15 changed files with 483 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/main/java/de/extio/lmdatasetprep/LmDatasetPreparerApplication.java
View File

@@ -1,11 +1,19 @@
package de.extio.lmdatasetprep;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.function.Consumer;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanNotOfRequiredTypeException;
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -14,7 +22,7 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.annotation.ComponentScan;
import de.extio.lmdatasetprep.preparer.DatasetTool;
import de.extio.lmdatasetprep.tools.DatasetTool;
@SpringBootApplication
@EnableCaching
@@ -29,25 +37,65 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
public static ApplicationContext applicationContext;
@SuppressWarnings("unchecked")
@Override
public void run(final String... args) throws Exception {
if (args.length == 0) {
LOGGER.error("Commandline argument is missing: Provide the tool name as argument");
if (args.length < 2) {
LOGGER.error("Commandline arguments are invalid. Usage: <tool properties file> [ <tool 1>, ... ]");
this.logAvailableTools();
return;
}
final var bean = applicationContext.getBean(args[0]);
if (!(bean instanceof Consumer) || !(bean instanceof DatasetTool)) {
LOGGER.error("Invalid bean {}", args[0]);
this.logAvailableTools();
return;
LOGGER.info("Loading tools");
final List<DatasetTool> tools = new ArrayList<>(args.length - 1);
for (int i = 1; i < args.length; i++) {
try {
tools.add(applicationContext.getBean(args[i], DatasetTool.class));
}
catch (NoSuchBeanDefinitionException | BeanNotOfRequiredTypeException e) {
LOGGER.error("Invalid bean {}", args[i]);
this.logAvailableTools();
return;
}
}
LOGGER.info("Executing {}", args[0]);
LOGGER.info("Loading tool properties");
final Properties toolProps = new Properties();
toolProps.load(Files.newInputStream(Path.of(args[0])));
try {
((Consumer<String[]>) bean).accept(args);
LOGGER.info("Executing {}", tools.stream().map(tool -> tool.getClass().getSimpleName()).collect(Collectors.joining(", ")));
final List<Thread> threads = new ArrayList<>(tools.size());
for (final var tool : tools) {
final Thread t = new Thread(() -> tool.accept(toolProps));
t.setName(StringUtils.abbreviateMiddle(tool.getClass().getSimpleName(), ".", 15));
t.start();
}
do {
Thread.sleep(1000);
} while (Execution.startedFileConsumers.get() == 0 || Execution.startedFileConsumers.get() > Execution.finishedFileConsumers.get());
LOGGER.info("All file consumers are finished");
int cnt = 0;
while (cnt < 3) {
Thread.sleep(1000);
if (!Execution.tasks.isEmpty() || Execution.work.values().stream().anyMatch(q -> q.values().stream().anyMatch(bq -> !bq.isEmpty()))) {
cnt = 0;
}
else {
cnt++;
}
}
LOGGER.info("All queues are empty");
Execution.shutdown.set(true);
for (final var thread : threads) {
thread.join();
}
LOGGER.info("Shutdown");
Execution.scheduler.shutdown();
Execution.executorService.shutdown();
}
catch (final Exception e) {
LOGGER.error("Error executing the component", e);