diff --git a/src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java b/src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java index a543c54..684ee46 100644 --- a/src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java +++ b/src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -58,6 +59,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo try (var in = new ClassPathResource("female.txt").getInputStream()) { this.femaleNames = new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.toList()); } + + this.mapper.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); } @Override @@ -146,24 +149,21 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo instruction, paragraph); - return processResponse(properties, completion); + return this.processResponse(properties, completion); } - + protected String requestCompletion(final Properties properties, final Conversation conversation) { final var completion = this.getClient(properties, this.clientService).conversation(this.getModelCategory(properties), conversation); - return processResponse(properties, completion); + return this.processResponse(properties, completion); } private String processResponse(final Properties properties, final Completion completion) { return TextUtils.normalizeModelResponse(completion.response(), Boolean.parseBoolean(properties.getProperty("contextualPrompts.removePreamble", "false"))); } - + protected void writeJsonLine(final OutputStream fos, final Object line) { try { - final var jsonb = this.mapper - .writeValueAsString(line) - .getBytes(StandardCharsets.UTF_8); - fos.write(jsonb); + this.mapper.writeValue(fos, line); fos.write(NEWLINE); } catch (final IOException e) { @@ -182,11 +182,11 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo protected record HistInstrComplLine(String history, String instruct, String completion) { } - + protected record ConversationLine(String user, String assistant) { } protected record ConversationsLine(List conversation) { } - + } diff --git a/src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java b/src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructionsSplitHistory.java similarity index 59% rename from src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java rename to src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructionsSplitHistory.java index 3ebf433..75448c4 100644 --- a/src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java +++ b/src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructionsSplitHistory.java @@ -12,9 +12,9 @@ import org.springframework.stereotype.Component; import de.extio.lmdatasetprep.Execution; @Component -public class GenHistoryInstructCompletion extends AbstractContextualPrompts { +public class GenContextualInstructionsSplitHistory extends AbstractContextualPrompts { - private static final Logger LOGGER = LoggerFactory.getLogger(GenHistoryInstructCompletion.class); + private static final Logger LOGGER = LoggerFactory.getLogger(GenContextualInstructionsSplitHistory.class); @Override protected CreateTasks createTasks(final Properties properties, final List chunkConfigurations, final int variations) { @@ -29,7 +29,7 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts { final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); final Path out = Execution.suffixFilename(p.file().getFileName(), - "histinstrcompl", + "contextualinstrhist", properties.getProperty("contextualPrompts.model"), String.valueOf(chunkCfg.chunksNorm()), String.valueOf(fi), @@ -41,8 +41,10 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts { final var inScope = String.join("\n", paragraphs.subList(j - 1, j)); final var mapping = this.createCharacterNameMapping(inScope, properties); - final var instruct = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); - final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping)); + final var summary = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j - 1), mapping), "Rewrite the following story as a summary of the previous paragraph and write that it is the summary. Return the summary, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); + final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping); + final var instruction = this.requestCompletion(properties, currentParagraph, "Summarize the following text as a single concise statement. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); + final var qaLine = new QaLine("History: " + summary + "\nInstruction: " + instruction, currentParagraph); this.writeJsonLine(fos, qaLine); } diff --git a/src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java b/src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java deleted file mode 100644 index 22eba82..0000000 --- a/src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java +++ /dev/null @@ -1,76 +0,0 @@ -package de.extio.lmdatasetprep.tools; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import de.extio.lmdatasetprep.Execution; -import de.extio.lmlib.client.Conversation; -import de.extio.lmlib.client.Conversation.Turn; -import de.extio.lmlib.client.Conversation.TurnType; - -@Component -public class GenConversation extends AbstractContextualPrompts { - - private static final Logger LOGGER = LoggerFactory.getLogger(GenConversation.class); - - @Override - protected CreateTasks createTasks(final Properties properties, final List chunkConfigurations, final int variations) { - return p -> { - final List tasks = new ArrayList<>(); - - for (final ChunkCfg chunkCfg : chunkConfigurations) { - for (int i = 0; i < variations; i++) { - final int fi = i; - - tasks.add(() -> { - final var conversation = new ArrayList(); - { - final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); - - final var mapping = new HashMap(); - for (int j = 0; j < paragraphs.size(); j++) { - LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size()); - mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties)); - } - - String previousParagraph = null; - String currentParagraph = null; - for (int j = 0; j < paragraphs.size(); j++) { - LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size()); - - previousParagraph = currentParagraph; - currentParagraph = this.renameCharacters(paragraphs.get(j), mapping); - - final var c = Conversation.create("Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation.", previousParagraph != null ? previousParagraph : currentParagraph); - if (previousParagraph != null) { - c.addTurn(new Turn(TurnType.ASSISTANT, conversation.getLast().user())); - c.addTurn(new Turn(TurnType.USER, currentParagraph)); - } - - final var instruction = this.requestCompletion(properties, c); - conversation.add(new ConversationLine(instruction, currentParagraph)); - } - } - - final Path out = Execution.suffixFilename(p.file().getFileName(), - "conversation", - properties.getProperty("contextualPrompts.model"), - String.valueOf(chunkCfg.chunksNorm()), - String.valueOf(fi), - ".jsonl"); - Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> this.writeJsonLine(fos, new ConversationsLine(conversation))); - }); - } - } - - return tasks; - }; - } -} diff --git a/src/main/java/de/extio/lmdatasetprep/tools/GenInstructionalConversation.java b/src/main/java/de/extio/lmdatasetprep/tools/GenInstructionalConversation.java new file mode 100644 index 0000000..e386580 --- /dev/null +++ b/src/main/java/de/extio/lmdatasetprep/tools/GenInstructionalConversation.java @@ -0,0 +1,79 @@ +package de.extio.lmdatasetprep.tools; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import de.extio.lmdatasetprep.Execution; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.Conversation.Turn; +import de.extio.lmlib.client.Conversation.TurnType; + +@Component +public class GenInstructionalConversation extends AbstractContextualPrompts { + + private static final Logger LOGGER = LoggerFactory.getLogger(GenInstructionalConversation.class); + + @Override + protected CreateTasks createTasks(final Properties properties, final List chunkConfigurations, final int variations) { + return p -> { + final List tasks = new ArrayList<>(); + + for (final ChunkCfg chunkCfg : chunkConfigurations) { + for (int i = 0; i < variations; i++) { + final int fi = i; + + tasks.add(() -> { + final Path out = Execution.suffixFilename(p.file().getFileName(), + "instrconversation", + properties.getProperty("contextualPrompts.model"), + String.valueOf(chunkCfg.chunksNorm()), + String.valueOf(fi), + ".jsonl"); + + Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> { + final var conversation = new ArrayList(); + { + final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); + + final var mapping = new HashMap(); + for (int j = 0; j < paragraphs.size(); j++) { + LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size()); + mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties)); + } + + String previousParagraph = null; + String currentParagraph = null; + for (int j = 0; j < paragraphs.size(); j++) { + LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size()); + + previousParagraph = currentParagraph; + currentParagraph = this.renameCharacters(paragraphs.get(j), mapping); + + final var c = Conversation.create("Summarize the following text as a single concise statement. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation.", previousParagraph != null ? previousParagraph : currentParagraph); + if (previousParagraph != null) { + c.addTurn(new Turn(TurnType.ASSISTANT, conversation.getLast().user())); + c.addTurn(new Turn(TurnType.USER, currentParagraph)); + } + + final var instruction = this.requestCompletion(properties, c); + conversation.add(new ConversationLine(instruction, currentParagraph)); + } + } + + this.writeJsonLine(fos, new ConversationsLine(conversation)); + }); + }); + } + } + + return tasks; + }; + } +}