exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MergeJsonl: Separate outputs per filter

Browse Source
This commit is contained in:
extio-de
2025-01-25 17:36:05 +01:00
parent 36ba17ef23
commit 77637436f2
1 changed files with 36 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java
View File

@@ -10,6 +10,7 @@ import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -25,6 +26,8 @@ public class MergeJsonl implements DatasetTool {
@Override @Override
public void accept(final Properties properties) { public void accept(final Properties properties) {
final var filters = properties.getProperty("mergeJsonl.filter", "*");
for (final var filter : StringUtils.split(filters, ',')) {
final List<String> lines = Collections.synchronizedList(new ArrayList<>()); final List<String> lines = Collections.synchronizedList(new ArrayList<>());
for (int i = 0; i < 99; i++) { for (int i = 0; i < 99; i++) {
final String source = properties.getProperty("mergeJsonl.source." + i, ""); final String source = properties.getProperty("mergeJsonl.source." + i, "");
@@ -32,8 +35,7 @@ public class MergeJsonl implements DatasetTool {
break; break;
} }
Execution.transform(source, p -> { Execution.transform(source, p -> {
final var filter = properties.getProperty("mergeJsonl.filter", ""); if (!filter.equals("*") && !p.file().getFileName().toString().contains(filter)) {
if (!filter.isEmpty() && !p.file().getFileName().toString().contains(filter)) {
return List.of(); return List.of();
} }
@@ -51,7 +53,8 @@ public class MergeJsonl implements DatasetTool {
Collections.shuffle(lines, ThreadLocalRandom.current()); Collections.shuffle(lines, ThreadLocalRandom.current());
final Path out = Path.of("dataset-merged.jsonl"); final Path out = Path.of("dataset-merged" + (filter.equals("*") ? "" : filter) + ".jsonl");
Execution.streamOut(out, "mergeJsonl.destination", properties, fos -> { Execution.streamOut(out, "mergeJsonl.destination", properties, fos -> {
try { try {
for (final String line : lines) { for (final String line : lines) {
@@ -65,4 +68,6 @@ public class MergeJsonl implements DatasetTool {
}); });
} }
}
} }