diff --git a/src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java b/src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java index 228d07a..ed52b4a 100644 --- a/src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java +++ b/src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java @@ -2,6 +2,7 @@ package de.extio.lmdatasetprep.tools; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Properties; @@ -26,7 +27,23 @@ public class GenConversation extends AbstractContextualPrompts { final int fi = i; tasks.add(() -> { - final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); + final var conversation = new ArrayList(); + { + final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); + + final var mapping = new HashMap(); + for (int j = 0; j < paragraphs.size(); j++) { + LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size()); + mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties)); + } + + for (int j = 0; j < paragraphs.size(); j++) { + LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size()); + final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping); + final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); + conversation.add(new ConversationLine(instruction, currentParagraph)); + } + } final Path out = Execution.suffixFilename(p.file().getFileName(), "conversation", @@ -34,21 +51,7 @@ public class GenConversation extends AbstractContextualPrompts { String.valueOf(chunkCfg.chunksNorm()), String.valueOf(fi), ".jsonl"); - - Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> { - List conversation = new ArrayList<>(); - - for (int j = 0; j < paragraphs.size(); j++) { - LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size()); - - final var mapping = this.createCharacterNameMapping(paragraphs.get(j), properties); - final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping); - final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); - conversation.add(new ConversationLine(instruction, currentParagraph)); - } - - this.writeJsonLine(fos, new ConversationsLine(conversation)); - }); + Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> this.writeJsonLine(fos, new ConversationsLine(conversation))); }); } }