exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New Tool: GenMergedText; Fixes;

Browse Source
This commit is contained in:
extio-de
2025-02-09 17:58:39 +01:00
parent c15e539e33
commit 40d50f96ec
6 changed files with 81 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/main/java/de/extio/lmdatasetprep/Execution.java
View File

@@ -14,8 +14,10 @@ import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@@ -45,6 +47,8 @@ public class Execution {
private static final Semaphore taskSemaphore;
private static final ThreadLocal<List<Future<?>>> localOutWorkers = ThreadLocal.withInitial(ArrayList::new);
private static final Logger LOGGER = LoggerFactory.getLogger(Execution.class);
static {
@@ -120,7 +124,7 @@ public class Execution {
try {
taskSemaphore.acquire();
tasksRunning.incrementAndGet();
executorService.execute(() -> {
localOutWorkers.get().add(executorService.submit(() -> {
try {
r.run();
}
@@ -128,7 +132,7 @@ public class Execution {
tasksRunning.decrementAndGet();
taskSemaphore.release();
}
});
}));
}
catch (final InterruptedException e) {
throw new RuntimeException("Interrupted", e);
@@ -182,17 +186,15 @@ public class Execution {
return;
}
final var outStream = new ProxyOutputStream(streams);
try {
localOutWorkers.get().clear();
try (final var outStream = new ProxyOutputStream(streams)) {
consumer.accept(outStream);
for (final var w : localOutWorkers.get()) {
w.get();
}
}
finally {
try {
outStream.close();
}
catch (final IOException e) {
LOGGER.error("IO exception", e);
}
catch (InterruptedException | ExecutionException | IOException e) {
LOGGER.error("Error while streaming out", e);
}
for (int i = 0; i < streamFuncs.size(); i++) {

11
src/main/java/de/extio/lmdatasetprep/LmDatasetPreparerApplication.java
View File

@@ -72,22 +72,19 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
t.start();
}
do {
Thread.sleep(1000);
} while (Execution.startedFileConsumers.get() == 0 || Execution.startedFileConsumers.get() > Execution.finishedFileConsumers.get());
LOGGER.info("All file consumers are finished");
int cnt = 0;
while (cnt < 5) {
Thread.sleep(1000);
if (Execution.tasksRunning.get() != 0 || Execution.work.values().stream().anyMatch(q -> q.values().stream().anyMatch(bq -> !bq.isEmpty()))) {
if (Execution.startedFileConsumers.get() == 0 || Execution.startedFileConsumers.get() > Execution.finishedFileConsumers.get() ||
Execution.tasksRunning.get() != 0 || Execution.work.values().stream().anyMatch(q -> q.values().stream().anyMatch(bq -> !bq.isEmpty()))) {
cnt = 0;
}
else {
cnt++;
}
}
LOGGER.info("All queues are empty");
LOGGER.info("All file consumers are finished, All queues are empty");
Execution.shutdown.set(true);
for (final var thread : threads) {

3
src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java
View File

@@ -189,4 +189,7 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
protected record ConversationsLine(List<ConversationLine> conversation) {
}
protected record TextLine(String text) {
}
}

57
src/main/java/de/extio/lmdatasetprep/tools/GenMergedText.java Normal file
View File

@@ -0,0 +1,57 @@
package de.extio.lmdatasetprep.tools;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.Execution;
@Component
public class GenMergedText extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(GenMergedText.class);
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
for (int i = 0; i < variations; i++) {
final int fi = i;
tasks.add(() -> {
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final Path out = Execution.suffixFilename(p.file().getFileName(),
"text",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi),
".jsonl");
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> {
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
final var mapping = this.createCharacterNameMapping(paragraphs.get(j), properties);
final var paragraph = this.renameCharacters(paragraphs.get(j), mapping);
final var textLine = new TextLine(paragraph);
this.writeJsonLine(fos, textLine);
}
});
});
}
}
return tasks;
};
}
}

12
src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java
View File

@@ -2,9 +2,9 @@ package de.extio.lmdatasetprep.tools;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
@@ -26,7 +26,7 @@ public class MergeJsonl implements DatasetTool {
@Override
public void accept(final Properties properties) {
final var filters = properties.getProperty("mergeJsonl.filter", "*");
final var filters = StringUtils.defaultIfEmpty(properties.getProperty("mergeJsonl.filter", "*"), "*");
for (final var filter : StringUtils.split(filters, ',')) {
final List<String> lines = Collections.synchronizedList(new ArrayList<>());
for (int i = 0; i < 99; i++) {
@@ -40,13 +40,7 @@ public class MergeJsonl implements DatasetTool {
}
return List.of(() -> {
try {
lines.addAll(Files.readAllLines(p.file()));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
lines.addAll(Arrays.asList(StringUtils.split(p.text(), '\n')));
});
});
}

2
src/main/java/de/extio/lmdatasetprep/tools/Text2Jsonl.java
View File

@@ -27,7 +27,7 @@ public class Text2Jsonl implements DatasetTool {
@Override
public void accept(final Properties properties) {
final Path out = Path.of("dataset.jsonl");
final Path out = Path.of("dataset-text.jsonl");
Execution.streamOut(out, "text2Jsonl.destination", properties, fos -> {
Execution.transform(properties.getProperty("text2Jsonl.source"), p -> {
return List.of(() -> {