exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Text2Jsonl2WithContextualPrompts: Prompt improved; Name mapping requires at least 5 characters (otherwise we may replace random syllables);

Browse Source
This commit is contained in:
extio-de
2024-12-01 22:50:53 +01:00
parent b027c2a892
commit 57f7ec2e48
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
View File

@@ -87,7 +87,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]>, Ini
if (j > 2) {
final var history = String.join("\n", paragraphs.subList(j - 3, j));
final var mapping = this.createCharacterNameMapping(history);
final var previous = this.paragraphToContextualPrompt(this.renameCharacters(history, mapping), "Rewrite the following text as a user prompt that asks to continue a story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var previous = this.paragraphToContextualPrompt(this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
qaLine = new QaLine(previous, this.renameCharacters(paragraphs.get(j), mapping));
}
else {
@@ -156,9 +156,12 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]>, Ini
if (names.females != null) {
names.females.forEach(name -> mapping.put(name, this.femaleNames.get(random.nextInt(this.femaleNames.size()))));
}
mapping.keySet().removeIf(key -> key.length() < 5);
if (mapping.isEmpty()) {
return Map.of();
}
LOGGER.info(mapping.toString());
return mapping;
}