Text2Jsonl2HistoryInstructCompletion: Improved llm prompt; Utils: normalizeModelResponse(), streamOut();

pom.xml

@@ -36,7 +36,11 @@
 			<groupId>com.github.ben-manes.caffeine</groupId>
 			<artifactId>caffeine</artifactId>
 		</dependency>
-
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        
         <dependency>
             <groupId>de.extio</groupId>
             <artifactId>lm-lib</artifactId>

src/main/java/de/extio/lmdatasetprep/preparer/AbstractContextualPrompts.java

@@ -49,13 +49,13 @@ abstract class AbstractContextualPrompts implements InitializingBean, Consumer<S
 	
 	@Override
 	public void accept(final String[] args) {
-		if (args.length < 6) {
+		if (args.length < 7) {
-			LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> [<chunk norm> <chunk var>, ...]");
+			LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <model prefix> <variations> [<chunk norm> <chunk var>, ...]");
 			return;
 		}
-		final int variations = Integer.parseInt(args[3]);
+		final int variations = Integer.parseInt(args[4]);
 		final List<ChunkCfg> chunkConfigurations = new ArrayList<>();
-		for (int i = 4; i < args.length; i += 2) {
+		for (int i = 5; i < args.length; i += 2) {
 			final var cfg = new ChunkCfg(Integer.parseInt(args[i]), Integer.parseInt(args[i + 1]));
 			chunkConfigurations.add(cfg);
 		}
@@ -135,24 +135,13 @@ abstract class AbstractContextualPrompts implements InitializingBean, Consumer<S
 		return result;
 	}
 	
-	protected String paragraphToContextualPrompt(final String paragraph, final String instruction) {
+	protected String createPrompt(final String paragraph, final String instruction) {
 		final var completion = this.clientService.getClient(ModelCategory.COLD).completion(ModelCategory.COLD,
 				"You are an assistant with great text writing skills.",
 				instruction,
 				paragraph);
 		
-		String prompt = completion.response();
+		return Utils.normalizeModelResponse(completion.response(), true);
-		final int colon = prompt.indexOf(':');
-		if (colon > -1 && colon < prompt.length() + 1) {
-			prompt = prompt.substring(colon + 1);
-		}
-		final int preamble = prompt.indexOf("\n\n");
-		if (preamble > -1 && preamble < prompt.length() + 2) {
-			prompt = prompt.substring(preamble + 2);
-		}
-		prompt = prompt.trim();
-		
-		return prompt;
 	}
 	
 	protected void writeJsonLine(final ObjectMapper mapper, final OutputStream fos, final Object line) {

src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java

@@ -81,7 +81,7 @@ public class Rewrite implements Consumer<String[]>, DatasetTool {
 		}
 		
 		LOGGER.info("Rewriting " + f.getFileName() + " " + identifier + " " + modelCategory.name() + " " + i);
-		try (var fos = Files.newOutputStream(out)) {
+		Utils.streamOut(out, fos -> {
 			final AtomicBoolean first = new AtomicBoolean(true);
 			this.rewrite(f, prompt, modelCategory, chunk -> {
 				try {
@@ -94,10 +94,7 @@ public class Rewrite implements Consumer<String[]>, DatasetTool {
 					LOGGER.error("IO exception", e);
 				}
 			});
-		}
+		});
-		catch (final IOException e1) {
-			LOGGER.error("IO exception", e1);
-		}
 	}
 	
 	void rewrite(final Path file, final String prompt, final ModelCategory modelCategory, final Consumer<String> consumer) {

src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2HistoryInstructCompletion.java

@@ -1,6 +1,5 @@
 package de.extio.lmdatasetprep.preparer;
 
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -29,6 +28,7 @@ public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualProm
 				for (int i = 0; i < variations; i++) {
 					final Path out = Utils.suffixFilename(Path.of(args[2]).resolve(p.getFileName()),
 							"histinstrcompl",
+							args[3],
 							String.valueOf(chunkCfg.chunksNorm()),
 							String.valueOf(i),
 							".jsonl");
@@ -38,21 +38,18 @@ public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualProm
 					}
 					
 					tasks.add(() -> {
-						try (var fos = Files.newOutputStream(out)) {
+						Utils.streamOut(out, fos -> {
 							for (int j = 1; j < paragraphs.size(); j++) {
 								LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
 								
 								final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
 								final var mapping = this.createCharacterNameMapping(inScope);
-								final var instruct = this.paragraphToContextualPrompt(this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+								final var instruct = this.createPrompt(this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 								final var qaLine = new QaLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
 								
 								this.writeJsonLine(mapper, fos, qaLine);
 							}
-						}
+						});
-						catch (final IOException e1) {
-							LOGGER.error("IO exception", e1);
-						}
 					});
 				}
 			}

src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java

@@ -29,6 +29,7 @@ public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts
 				for (int i = 0; i < variations; i++) {
 					final Path out = Utils.suffixFilename(Path.of(args[2]).resolve(p.getFileName()),
 							"contextualds",
+							args[3],
 							String.valueOf(chunkCfg.chunksNorm()),
 							String.valueOf(i),
 							".jsonl");
@@ -38,7 +39,7 @@ public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts
 					}
 					
 					tasks.add(() -> {
-						try (var fos = Files.newOutputStream(out)) {
+						Utils.streamOut(out, fos -> {
 							for (int j = 0; j < paragraphs.size(); j++) {
 								LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
 								
@@ -46,22 +47,19 @@ public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts
 								if (j > 2) {
 									final var history = String.join("\n", paragraphs.subList(j - 3, j));
 									final var mapping = this.createCharacterNameMapping(history);
-									final var previous = this.paragraphToContextualPrompt(this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+									final var previous = this.createPrompt(this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 									qaLine = new QaLine(previous, this.renameCharacters(paragraphs.get(j), mapping));
 								}
 								else {
 									final var mapping = this.createCharacterNameMapping(paragraphs.get(j));
 									final var paragraph = this.renameCharacters(paragraphs.get(j), mapping);
-									final var userPrompt = this.paragraphToContextualPrompt(paragraph, "Write a summary of the following story as a user prompt that asks to write the story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+									final var userPrompt = this.createPrompt(paragraph, "Write a summary of the following story as a user prompt that asks to write the story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 									qaLine = new QaLine(userPrompt, paragraph);
 								}
 								
 								this.writeJsonLine(mapper, fos, qaLine);
 							}
-						}
+						});
-						catch (final IOException e1) {
-							LOGGER.error("IO exception", e1);
-						}
 					});
 				}
 			}

src/main/java/de/extio/lmdatasetprep/preparer/Translate.java

@@ -40,7 +40,7 @@ public class Translate implements Consumer<String[]>, DatasetTool {
 					LOGGER.info("Skipping " + out);
 				}
 				else {
-					try (var fos = Files.newOutputStream(out)) {
+					Utils.streamOut(out, fos -> {
 						final AtomicBoolean first = new AtomicBoolean(true);
 						this.translate(p, chunk -> {
 							try {
@@ -53,10 +53,7 @@ public class Translate implements Consumer<String[]>, DatasetTool {
 								LOGGER.error("IO exception", e);
 							}
 						});
-					}
+					});
-					catch (final IOException e1) {
-						LOGGER.error("IO exception", e1);
-					}
 				}
 			});
 		});

src/main/java/de/extio/lmdatasetprep/preparer/Utils.java

@@ -1,6 +1,7 @@
 package de.extio.lmdatasetprep.preparer;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -11,9 +12,11 @@ import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Stream;
 
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,6 +66,21 @@ public class Utils {
 		return result;
 	}
 	
+	static String normalizeModelResponse(final String response, final boolean removePreamble) {
+		var result = StringUtils.replace(response, "\r", "");
+		
+		final int colon = result.indexOf(':');
+		if (colon > -1 && colon < result.length() - 2 && (colon < 50 || result.charAt(colon + 1) == '\n')) {
+			result = result.substring(colon + 1);
+		}
+		
+		result = StringUtils.trim(result);
+		result = StringUtils.strip(result, "´`'“”„‟«»\"\r\n");
+		result = StringUtils.trim(result);
+		
+		return result;
+	}
+	
 	static List<String> splitParagraphs(final String text, final int chunks_norm, final int chunks_var, final boolean slidingWindows) {
 		final int CHUNKS_MIN = chunks_norm - chunks_var;
 		final int CHUNKS_MAX = chunks_norm + chunks_var;
@@ -209,4 +227,21 @@ public class Utils {
 		
 		return f.getParent().resolve(parts[0] + sb.toString() + last);
 	}
+	
+	static void streamOut(final Path f, final Consumer<OutputStream> consumer) {
+		final var tmp = f.resolveSibling(f.getFileName() + ".tmp");
+		try (var fos = Files.newOutputStream(tmp)) {
+			consumer.accept(fos);
+		}
+		catch (final IOException e1) {
+			LOGGER.error("IO exception", e1);
+			return;
+		}
+		try {
+			Files.move(tmp, f);
+		}
+		catch (final IOException e) {
+			LOGGER.error("IO exception", e);
+		}
+	}
 }

src/main/resources/application.properties

@@ -7,6 +7,7 @@ profile.model.m=llama3-cold
 profile.model.l=llama3-cold
 profile.model.hot=llama3-hot
 profile.model.cold=llama3-cold
+#profile.model.cold=mixtral-cold
 
 agent.threads=4