exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MergeJsonl: Separate outputs per filter

Browse Source
This commit is contained in:
extio-de
2025-01-25 17:36:05 +01:00
parent 36ba17ef23
commit 77637436f2
1 changed files with 36 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java
View File

@@ -10,6 +10,7 @@ import java.util.List;
import java.util.Properties;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -25,6 +26,8 @@ public class MergeJsonl implements DatasetTool {
@Override
public void accept(final Properties properties) {
final var filters = properties.getProperty("mergeJsonl.filter", "*");
for (final var filter : StringUtils.split(filters, ',')) {
final List<String> lines = Collections.synchronizedList(new ArrayList<>());
for (int i = 0; i < 99; i++) {
final String source = properties.getProperty("mergeJsonl.source." + i, "");
@@ -32,8 +35,7 @@ public class MergeJsonl implements DatasetTool {
break;
}
Execution.transform(source, p -> {
final var filter = properties.getProperty("mergeJsonl.filter", "");
if (!filter.isEmpty() && !p.file().getFileName().toString().contains(filter)) {
if (!filter.equals("*") && !p.file().getFileName().toString().contains(filter)) {
return List.of();
}
@@ -51,7 +53,8 @@ public class MergeJsonl implements DatasetTool {
Collections.shuffle(lines, ThreadLocalRandom.current());
final Path out = Path.of("dataset-merged.jsonl");
final Path out = Path.of("dataset-merged" + (filter.equals("*") ? "" : filter) + ".jsonl");
Execution.streamOut(out, "mergeJsonl.destination", properties, fos -> {
try {
for (final String line : lines) {
@@ -66,3 +69,5 @@ public class MergeJsonl implements DatasetTool {
}
}
}