exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MergeJsonl: Separate outputs per filter

Browse Source
This commit is contained in:
extio-de
2025-01-25 17:36:05 +01:00
parent 36ba17ef23
commit 77637436f2
1 changed files with 36 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java
View File

@@ -10,6 +10,7 @@ import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -25,44 +26,48 @@ public class MergeJsonl implements DatasetTool {
@Override @Override
public void accept(final Properties properties) { public void accept(final Properties properties) {
final List<String> lines = Collections.synchronizedList(new ArrayList<>()); final var filters = properties.getProperty("mergeJsonl.filter", "*");
for (int i = 0; i < 99; i++) { for (final var filter : StringUtils.split(filters, ',')) {
final String source = properties.getProperty("mergeJsonl.source." + i, ""); final List<String> lines = Collections.synchronizedList(new ArrayList<>());
if (source.isBlank()) { for (int i = 0; i < 99; i++) {
break; final String source = properties.getProperty("mergeJsonl.source." + i, "");
} if (source.isBlank()) {
Execution.transform(source, p -> { break;
final var filter = properties.getProperty("mergeJsonl.filter", "");
if (!filter.isEmpty() && !p.file().getFileName().toString().contains(filter)) {
return List.of();
} }
Execution.transform(source, p -> {
return List.of(() -> { if (!filter.equals("*") && !p.file().getFileName().toString().contains(filter)) {
try { return List.of();
lines.addAll(Files.readAllLines(p.file()));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
} }
return List.of(() -> {
try {
lines.addAll(Files.readAllLines(p.file()));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
});
}); });
}
Collections.shuffle(lines, ThreadLocalRandom.current());
final Path out = Path.of("dataset-merged" + (filter.equals("*") ? "" : filter) + ".jsonl");
Execution.streamOut(out, "mergeJsonl.destination", properties, fos -> {
try {
for (final String line : lines) {
fos.write(line.getBytes(StandardCharsets.UTF_8));
fos.write(NEWLINE);
}
}
catch (final IOException e1) {
LOGGER.error("IO exception", e1);
}
}); });
} }
Collections.shuffle(lines, ThreadLocalRandom.current());
final Path out = Path.of("dataset-merged.jsonl");
Execution.streamOut(out, "mergeJsonl.destination", properties, fos -> {
try {
for (final String line : lines) {
fos.write(line.getBytes(StandardCharsets.UTF_8));
fos.write(NEWLINE);
}
}
catch (final IOException e1) {
LOGGER.error("IO exception", e1);
}
});
} }
} }