exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New Tool: GenMergedText; Fixes;

Browse Source
This commit is contained in:
extio-de
2025-02-09 17:58:39 +01:00
parent c15e539e33
commit 40d50f96ec
6 changed files with 81 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/de/extio/lmdatasetprep/LmDatasetPreparerApplication.java
View File

@@ -72,22 +72,19 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
t.start();
}
do {
Thread.sleep(1000);
} while (Execution.startedFileConsumers.get() == 0 || Execution.startedFileConsumers.get() > Execution.finishedFileConsumers.get());
LOGGER.info("All file consumers are finished");
int cnt = 0;
while (cnt < 5) {
Thread.sleep(1000);
if (Execution.tasksRunning.get() != 0 || Execution.work.values().stream().anyMatch(q -> q.values().stream().anyMatch(bq -> !bq.isEmpty()))) {
if (Execution.startedFileConsumers.get() == 0 || Execution.startedFileConsumers.get() > Execution.finishedFileConsumers.get() ||
Execution.tasksRunning.get() != 0 || Execution.work.values().stream().anyMatch(q -> q.values().stream().anyMatch(bq -> !bq.isEmpty()))) {
cnt = 0;
}
else {
cnt++;
}
}
LOGGER.info("All queues are empty");
LOGGER.info("All file consumers are finished, All queues are empty");
Execution.shutdown.set(true);
for (final var thread : threads) {