Fix MixtralPromptStrategy; Refactoring;

src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java

@@ -10,10 +10,14 @@ public class MixtralPromptStrategy implements PromptStrategy {
 		final StringBuilder prompt = new StringBuilder();
 		prompt.append("<s>[INST] ");
 		prompt.append(instruction);
-		if (!question.isEmpty() && !text.isEmpty()) {
-			prompt.append(" ");
+		if (!instruction.isEmpty() && !question.isEmpty()) {
+			prompt.append("\n");
 		}
 		prompt.append(question);
+		if ((!instruction.isEmpty() || !question.isEmpty()) && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
 		prompt.append(" [/INST]");
 		return prompt;
 	}
@@ -32,6 +36,11 @@ public class MixtralPromptStrategy implements PromptStrategy {
 		prompt.append(" [/INST]");
 	}
 	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("</s>", "");
+	}
+	
 	@Override
 	public String getPromptName() {
 		return "mixtral";

src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java

@@ -32,7 +32,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]> {
 	@Override
 	public void accept(final String[] args) {
 		if (args.length < 6) {
-			LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> (<chunk norm> <chunk var>, ...)");
+			LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> [<chunk norm> <chunk var>, ...]");
 			return;
 		}
 		final int variations = Integer.parseInt(args[3]);
@@ -57,7 +57,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]> {
 							String.valueOf(i),
 							".jsonl");
 					if (Files.exists(out)) {
-						LOGGER.info("Skipping " + out);
+						LOGGER.info("Skipping {}", out);
 						continue;
 					}
 					

src/main/java/de/extio/lmdatasetprep/preparer/Utils.java

@@ -19,9 +19,11 @@ import org.slf4j.LoggerFactory;
 
 public class Utils {
 	
+	private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
+	
 	private static final List<String> PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?");
 	
-	private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
+	private static final int EXECUTORS = 4;
 	
 	static String normalizeText(final String text) {
 		String result = text;
@@ -92,7 +94,7 @@ public class Utils {
 			if (slidingWindows && pos > chunks_var) {
 				for (final String delimiter : PARAGRAPH_DELIMITERS) {
 					final int o = text.indexOf(delimiter, pos - chunks_var);
-					if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) {
+					if (o > -1 && o < pos - 10) {
 						pos = o + delimiter.length();
 						break;
 					}
@@ -113,9 +115,6 @@ public class Utils {
 	}
 	
 	static void transformDirectory(final String path, final Function<Path, List<Runnable>> createTasks) {
-		final int EXECUTORS = 4;
-		
-		try (Stream<Path> stream = Files.list(Paths.get(path))) {
 		final AtomicBoolean finished = new AtomicBoolean(false);
 		
 		final BlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
@@ -140,6 +139,7 @@ public class Utils {
 			threads.add(executor);
 		}
 		
+		try (Stream<Path> stream = Files.list(Paths.get(path))) {
 			stream
 					.filter(p -> Files.isRegularFile(p))
 					.forEach(p -> {
@@ -152,18 +152,28 @@ public class Utils {
 							}
 						}
 					});
+		}
+		catch (final IOException e) {
+			LOGGER.error("Error", e);
+		}
 		
 		while (!tasks.isEmpty()) {
+			try {
 				Thread.sleep(1);
 			}
+			catch (final InterruptedException e) {
+				LOGGER.error("Interrupted", e);
+			}
+		}
 		finished.set(true);
 		
 		for (int i = 0; i < EXECUTORS; i++) {
+			try {
 				threads.get(i).join();
 			}
+			catch (final InterruptedException e) {
+				LOGGER.error("Interrupted", e);
 			}
-		catch (final IOException | InterruptedException e) {
-			LOGGER.error("Error", e);
 		}
 	}