exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MergeJsonl tools accepts multiple directories;

Browse Source
This commit is contained in:
extio-de
2024-12-18 20:48:58 +01:00
parent 1c94b97b92
commit 56b7dcb265
1 changed files with 14 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java
View File

@@ -24,12 +24,13 @@ public class MergeJsonl implements Consumer<String[]>, DatasetTool {
@Override @Override
public void accept(final String[] args) { public void accept(final String[] args) {
if (args.length < 3) { if (args.length < 3) {
LOGGER.error("Arguments are missing. <Path to jsonl files> <Path to output dir>"); LOGGER.error("Arguments are missing. <Path to output dir> [ <Path to jsonl files>, ... ]");
return; return;
} }
final List<String> lines = Collections.synchronizedList(new ArrayList<>()); final List<String> lines = Collections.synchronizedList(new ArrayList<>());
Utils.transformDirectory(args[1], p -> { for (int i = 2; i < args.length; i++) {
Utils.transformDirectory(args[i], p -> {
return List.of(() -> { return List.of(() -> {
try { try {
lines.addAll(Files.readAllLines(p)); lines.addAll(Files.readAllLines(p));
@@ -40,10 +41,11 @@ public class MergeJsonl implements Consumer<String[]>, DatasetTool {
}); });
}); });
}
Collections.shuffle(lines, ThreadLocalRandom.current()); Collections.shuffle(lines, ThreadLocalRandom.current());
final Path out = Path.of(args[2], "dataset-merged.jsonl"); final Path out = Path.of(args[1], "dataset-merged.jsonl");
try (var fos = Files.newOutputStream(out)) { try (var fos = Files.newOutputStream(out)) {
for (final String line : lines) { for (final String line : lines) {
fos.write(line.getBytes(StandardCharsets.UTF_8)); fos.write(line.getBytes(StandardCharsets.UTF_8));