GenConversation: Provide history in the prompt;

src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java

@@ -27,6 +27,8 @@ import de.extio.lmdatasetprep.Execution;
 import de.extio.lmdatasetprep.Execution.WorkPacket;
 import de.extio.lmdatasetprep.TextUtils;
 import de.extio.lmlib.client.ClientService;
+import de.extio.lmlib.client.Completion;
+import de.extio.lmlib.client.Conversation;
 
 abstract class AbstractContextualPrompts implements InitializingBean, DatasetTool {
 	
@@ -138,15 +140,24 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
 		return result;
 	}
 	
-	protected String createPrompt(final Properties properties, final String paragraph, final String instruction) {
+	protected String requestCompletion(final Properties properties, final String paragraph, final String instruction) {
 		final var completion = this.getClient(properties, this.clientService).completion(this.getModelCategory(properties),
 				"You are an assistant with great text writing skills.",
 				instruction,
 				paragraph);
 		
-		return TextUtils.normalizeModelResponse(completion.response(), Boolean.parseBoolean(properties.getProperty("contextualPrompts.removePreamble", "false")));
+		return processResponse(properties, completion);
+	}
+
+	protected String requestCompletion(final Properties properties, final Conversation conversation) {
+		final var completion = this.getClient(properties, this.clientService).conversation(this.getModelCategory(properties), conversation);
+		return processResponse(properties, completion);
 	}
 	
+	private String processResponse(final Properties properties, final Completion completion) {
+		return TextUtils.normalizeModelResponse(completion.response(), Boolean.parseBoolean(properties.getProperty("contextualPrompts.removePreamble", "false")));
+	}
+
 	protected void writeJsonLine(final OutputStream fos, final Object line) {
 		try {
 			final var jsonb = this.mapper

src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructions.java

@@ -41,9 +41,9 @@ public class GenContextualInstructions extends AbstractContextualPrompts {
 								
 								final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
 								final var mapping = this.createCharacterNameMapping(inScope, properties);
-								final var summary = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j - 1), mapping), "Rewrite the following story as a summary of the previous paragraph and write that it is the summary. Return the summary, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+								final var summary = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j - 1), mapping), "Rewrite the following story as a summary of the previous paragraph and write that it is the summary. Return the summary, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 								final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
-								final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+								final var instruction = this.requestCompletion(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 								final var qaLine = new QaLine(String.join(" ", summary, instruction), currentParagraph);
 								
 								this.writeJsonLine(fos, qaLine);

src/main/java/de/extio/lmdatasetprep/tools/GenContextualPrompts.java

@@ -43,13 +43,13 @@ public class GenContextualPrompts extends AbstractContextualPrompts {
 								if (j > 2) {
 									final var history = String.join("\n", paragraphs.subList(j - 3, j));
 									final var mapping = this.createCharacterNameMapping(history, properties);
-									final var previous = this.createPrompt(properties, this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+									final var previous = this.requestCompletion(properties, this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 									qaLine = new QaLine(previous, this.renameCharacters(paragraphs.get(j), mapping));
 								}
 								else {
 									final var mapping = this.createCharacterNameMapping(paragraphs.get(j), properties);
 									final var paragraph = this.renameCharacters(paragraphs.get(j), mapping);
-									final var userPrompt = this.createPrompt(properties, paragraph, "Write a summary of the following story as a user prompt that asks to write the story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+									final var userPrompt = this.requestCompletion(properties, paragraph, "Write a summary of the following story as a user prompt that asks to write the story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 									qaLine = new QaLine(userPrompt, paragraph);
 								}
 								

src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java

@@ -11,6 +11,9 @@ import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
 import de.extio.lmdatasetprep.Execution;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.Conversation.TurnType;
 
 @Component
 public class GenConversation extends AbstractContextualPrompts {
@@ -37,10 +40,21 @@ public class GenConversation extends AbstractContextualPrompts {
 								mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties));
 							}
 							
+							String previousParagraph = null;
+							String currentParagraph = null;
 							for (int j = 0; j < paragraphs.size(); j++) {
 								LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
-								final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
+								
-								final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+								previousParagraph = currentParagraph;
+								currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
+								
+								final var c = Conversation.create("Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation.", previousParagraph != null ? previousParagraph : currentParagraph);
+								if (previousParagraph != null) {
+									c.addTurn(new Turn(TurnType.ASSISTANT, conversation.getLast().user()));
+									c.addTurn(new Turn(TurnType.USER, currentParagraph));
+								}
+								
+								final var instruction = this.requestCompletion(properties, c);
 								conversation.add(new ConversationLine(instruction, currentParagraph));
 							}
 						}

src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java

@@ -41,7 +41,7 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
 								
 								final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
 								final var mapping = this.createCharacterNameMapping(inScope, properties);
-								final var instruct = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+								final var instruct = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
 								final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
 								
 								this.writeJsonLine(fos, qaLine);