exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MergeJsonl tools accepts multiple directories;

Browse Source
This commit is contained in:
extio-de
2024-12-18 20:48:58 +01:00
parent 1c94b97b92
commit 56b7dcb265
1 changed files with 14 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java
View File

@@ -24,26 +24,28 @@ public class MergeJsonl implements Consumer<String[]>, DatasetTool {
@Override
public void accept(final String[] args) {
if (args.length < 3) {
LOGGER.error("Arguments are missing. <Path to jsonl files> <Path to output dir>");
LOGGER.error("Arguments are missing. <Path to output dir> [ <Path to jsonl files>, ... ]");
return;
}
final List<String> lines = Collections.synchronizedList(new ArrayList<>());
Utils.transformDirectory(args[1], p -> {
return List.of(() -> {
try {
lines.addAll(Files.readAllLines(p));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
for (int i = 2; i < args.length; i++) {
Utils.transformDirectory(args[i], p -> {
return List.of(() -> {
try {
lines.addAll(Files.readAllLines(p));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
});
});
});
}
Collections.shuffle(lines, ThreadLocalRandom.current());
final Path out = Path.of(args[2], "dataset-merged.jsonl");
final Path out = Path.of(args[1], "dataset-merged.jsonl");
try (var fos = Files.newOutputStream(out)) {
for (final String line : lines) {
fos.write(line.getBytes(StandardCharsets.UTF_8));