exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Randomize character names only once per conversation;

Browse Source
This commit is contained in:
extio-de
2025-01-12 10:57:37 +01:00
parent dad9915870
commit 538e92ca0a
1 changed files with 19 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java
View File

@@ -2,6 +2,7 @@ package de.extio.lmdatasetprep.tools;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
@@ -26,29 +27,31 @@ public class GenConversation extends AbstractContextualPrompts {
final int fi = i; final int fi = i;
tasks.add(() -> { tasks.add(() -> {
final var conversation = new ArrayList<ConversationLine>();
{
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final var mapping = new HashMap<String, String>();
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size());
mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties));
}
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
conversation.add(new ConversationLine(instruction, currentParagraph));
}
}
final Path out = Execution.suffixFilename(p.file().getFileName(), final Path out = Execution.suffixFilename(p.file().getFileName(),
"conversation", "conversation",
properties.getProperty("contextualPrompts.model"), properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()), String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi), String.valueOf(fi),
".jsonl"); ".jsonl");
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> this.writeJsonLine(fos, new ConversationsLine(conversation)));
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> {
List<ConversationLine> conversation = new ArrayList<>();
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
final var mapping = this.createCharacterNameMapping(paragraphs.get(j), properties);
final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
conversation.add(new ConversationLine(instruction, currentParagraph));
}
this.writeJsonLine(fos, new ConversationsLine(conversation));
});
}); });
} }
} }