Client: Collect and show more statistics; MergeJsonl: Shuffle merged json; Rewrite: Fix progress indicator;

src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java

@@ -1,8 +1,11 @@
 package de.extio.lmdatasetprep.preparer;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 
@@ -10,9 +13,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
+import de.extio.lmdatasetprep.XorShift128Random;
+
 @Component
 public class MergeJsonl implements Consumer<String[]> {
 	
+	private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
+	
 	private static final Logger LOGGER = LoggerFactory.getLogger(MergeJsonl.class);
 	
 	@Override
@@ -22,22 +29,27 @@ public class MergeJsonl implements Consumer<String[]> {
 			return;
 		}
 		
+		final List<String> lines = Collections.synchronizedList(new ArrayList<>());
+		Utils.transformDirectory(args[1], p -> {
+			return List.of(() -> {
+				try {
+					lines.addAll(Files.readAllLines(p));
+				}
+				catch (final IOException e) {
+					throw new RuntimeException("Cannot read file", e);
+				}
+				
+			});
+		});
+		
+		Collections.shuffle(lines, new XorShift128Random());
+		
 		final Path out = Path.of(args[2], "dataset-merged.jsonl");
 		try (var fos = Files.newOutputStream(out)) {
-			Utils.transformDirectory(args[1], p -> {
-				return List.of(() -> {
-					try {
-						final byte[] b = Files.readAllBytes(p);
-						synchronized (fos) {
-							fos.write(b);
-						}
-					}
-					catch (final IOException e) {
-						throw new RuntimeException("Cannot read file", e);
-					}
-					
-				});
-			});
+			for (final String line : lines) {
+				fos.write(line.getBytes(StandardCharsets.UTF_8));
+				fos.write(NEWLINE);
+			}
 		}
 		catch (final IOException e1) {
 			LOGGER.error("IO exception", e1);

src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java

@@ -115,7 +115,7 @@ public class Rewrite implements Consumer<String[]> {
 		final List<String> splits = Utils.splitParagraphs(text, 1250, 350, false);
 		
 		for (final String split : splits) {
-			LOGGER.info("Split " + splits.indexOf(split) + "/" + splits.size());
+			LOGGER.info("Split " + (splits.indexOf(split) + 1) + "/" + splits.size());
 			final var completion = this.client.completion("You are a helpful assistant with great authoring skills.",
 					prompt,
 					split,
@@ -123,5 +123,4 @@ public class Rewrite implements Consumer<String[]> {
 			consumer.accept(completion.response());
 		}
 	}
-	
 }