diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java b/src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java index d3decc4..193e448 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java @@ -24,26 +24,28 @@ public class MergeJsonl implements Consumer, DatasetTool { @Override public void accept(final String[] args) { if (args.length < 3) { - LOGGER.error("Arguments are missing. "); + LOGGER.error("Arguments are missing. [ , ... ]"); return; } final List lines = Collections.synchronizedList(new ArrayList<>()); - Utils.transformDirectory(args[1], p -> { - return List.of(() -> { - try { - lines.addAll(Files.readAllLines(p)); - } - catch (final IOException e) { - throw new RuntimeException("Cannot read file", e); - } - + for (int i = 2; i < args.length; i++) { + Utils.transformDirectory(args[i], p -> { + return List.of(() -> { + try { + lines.addAll(Files.readAllLines(p)); + } + catch (final IOException e) { + throw new RuntimeException("Cannot read file", e); + } + + }); }); - }); + } Collections.shuffle(lines, ThreadLocalRandom.current()); - final Path out = Path.of(args[2], "dataset-merged.jsonl"); + final Path out = Path.of(args[1], "dataset-merged.jsonl"); try (var fos = Files.newOutputStream(out)) { for (final String line : lines) { fos.write(line.getBytes(StandardCharsets.UTF_8));