Reworked instruction generation tools;

src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java

@@ -20,6 +20,7 @@ import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
+import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
@@ -58,6 +59,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
 		try (var in = new ClassPathResource("female.txt").getInputStream()) {
 			this.femaleNames = new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.toList());
 		}
+		
+		this.mapper.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
 	}
 	
 	@Override
@@ -146,12 +149,12 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
 				instruction,
 				paragraph);
 		
-		return processResponse(properties, completion);
+		return this.processResponse(properties, completion);
 	}
 	
 	protected String requestCompletion(final Properties properties, final Conversation conversation) {
 		final var completion = this.getClient(properties, this.clientService).conversation(this.getModelCategory(properties), conversation);
-		return processResponse(properties, completion);
+		return this.processResponse(properties, completion);
 	}
 	
 	private String processResponse(final Properties properties, final Completion completion) {
@@ -160,10 +163,7 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
 	
 	protected void writeJsonLine(final OutputStream fos, final Object line) {
 		try {
-			final var jsonb = this.mapper
+			this.mapper.writeValue(fos, line);
-					.writeValueAsString(line)
-					.getBytes(StandardCharsets.UTF_8);
-			fos.write(jsonb);
 			fos.write(NEWLINE);
 		}
 		catch (final IOException e) {

src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructionsSplitHistory.java

@@ -12,9 +12,9 @@ import org.springframework.stereotype.Component;
 import de.extio.lmdatasetprep.Execution;
 
 @Component
-public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
+public class GenContextualInstructionsSplitHistory extends AbstractContextualPrompts {
 	
-	private static final Logger LOGGER = LoggerFactory.getLogger(GenHistoryInstructCompletion.class);
+	private static final Logger LOGGER = LoggerFactory.getLogger(GenContextualInstructionsSplitHistory.class);
 	
 	@Override
 	protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
@@ -29,7 +29,7 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
 						final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
 						
 						final Path out = Execution.suffixFilename(p.file().getFileName(),
-								"histinstrcompl",
+								"contextualinstrhist",
 								properties.getProperty("contextualPrompts.model"),
 								String.valueOf(chunkCfg.chunksNorm()),
 								String.valueOf(fi),
@@ -41,8 +41,10 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
 								
 								final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
 								final var mapping = this.createCharacterNameMapping(inScope, properties);
-								final var instruct = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+								final var summary = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j - 1), mapping), "Rewrite the following story as a summary of the previous paragraph and write that it is the summary. Return the summary, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
-								final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
+								final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
+								final var instruction = this.requestCompletion(properties, currentParagraph, "Summarize the following text as a single concise statement. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
+								final var qaLine = new QaLine("History: " + summary + "\nInstruction: " + instruction, currentParagraph);
 								
 								this.writeJsonLine(fos, qaLine);
 							}

src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java

@@ -1,76 +0,0 @@
-package de.extio.lmdatasetprep.tools;
-
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Properties;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import de.extio.lmdatasetprep.Execution;
-import de.extio.lmlib.client.Conversation;
-import de.extio.lmlib.client.Conversation.Turn;
-import de.extio.lmlib.client.Conversation.TurnType;
-
-@Component
-public class GenConversation extends AbstractContextualPrompts {
-	
-	private static final Logger LOGGER = LoggerFactory.getLogger(GenConversation.class);
-	
-	@Override
-	protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
-		return p -> {
-			final List<Runnable> tasks = new ArrayList<>();
-			
-			for (final ChunkCfg chunkCfg : chunkConfigurations) {
-				for (int i = 0; i < variations; i++) {
-					final int fi = i;
-					
-					tasks.add(() -> {
-						final var conversation = new ArrayList<ConversationLine>();
-						{
-							final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
-							
-							final var mapping = new HashMap<String, String>();
-							for (int j = 0; j < paragraphs.size(); j++) {
-								LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size());
-								mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties));
-							}
-							
-							String previousParagraph = null;
-							String currentParagraph = null;
-							for (int j = 0; j < paragraphs.size(); j++) {
-								LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
-								
-								previousParagraph = currentParagraph;
-								currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
-								
-								final var c = Conversation.create("Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation.", previousParagraph != null ? previousParagraph : currentParagraph);
-								if (previousParagraph != null) {
-									c.addTurn(new Turn(TurnType.ASSISTANT, conversation.getLast().user()));
-									c.addTurn(new Turn(TurnType.USER, currentParagraph));
-								}
-								
-								final var instruction = this.requestCompletion(properties, c);
-								conversation.add(new ConversationLine(instruction, currentParagraph));
-							}
-						}
-						
-						final Path out = Execution.suffixFilename(p.file().getFileName(),
-								"conversation",
-								properties.getProperty("contextualPrompts.model"),
-								String.valueOf(chunkCfg.chunksNorm()),
-								String.valueOf(fi),
-								".jsonl");
-						Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> this.writeJsonLine(fos, new ConversationsLine(conversation)));
-					});
-				}
-			}
-			
-			return tasks;
-		};
-	}
-}

src/main/java/de/extio/lmdatasetprep/tools/GenInstructionalConversation.java

@@ -0,0 +1,79 @@
+package de.extio.lmdatasetprep.tools;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import de.extio.lmdatasetprep.Execution;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.Conversation.TurnType;
+
+@Component
+public class GenInstructionalConversation extends AbstractContextualPrompts {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(GenInstructionalConversation.class);
+	
+	@Override
+	protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
+		return p -> {
+			final List<Runnable> tasks = new ArrayList<>();
+			
+			for (final ChunkCfg chunkCfg : chunkConfigurations) {
+				for (int i = 0; i < variations; i++) {
+					final int fi = i;
+					
+					tasks.add(() -> {
+						final Path out = Execution.suffixFilename(p.file().getFileName(),
+								"instrconversation",
+								properties.getProperty("contextualPrompts.model"),
+								String.valueOf(chunkCfg.chunksNorm()),
+								String.valueOf(fi),
+								".jsonl");
+						
+						Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> {
+							final var conversation = new ArrayList<ConversationLine>();
+							{
+								final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
+								
+								final var mapping = new HashMap<String, String>();
+								for (int j = 0; j < paragraphs.size(); j++) {
+									LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size());
+									mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties));
+								}
+								
+								String previousParagraph = null;
+								String currentParagraph = null;
+								for (int j = 0; j < paragraphs.size(); j++) {
+									LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
+									
+									previousParagraph = currentParagraph;
+									currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
+									
+									final var c = Conversation.create("Summarize the following text as a single concise statement. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation.", previousParagraph != null ? previousParagraph : currentParagraph);
+									if (previousParagraph != null) {
+										c.addTurn(new Turn(TurnType.ASSISTANT, conversation.getLast().user()));
+										c.addTurn(new Turn(TurnType.USER, currentParagraph));
+									}
+									
+									final var instruction = this.requestCompletion(properties, c);
+									conversation.add(new ConversationLine(instruction, currentParagraph));
+								}
+							}
+							
+							this.writeJsonLine(fos, new ConversationsLine(conversation));
+						});
+					});
+				}
+			}
+			
+			return tasks;
+		};
+	}
+}