From 57f7ec2e48e39bc74f66967121df5c6002cd6fd5 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sun, 1 Dec 2024 22:50:53 +0100 Subject: [PATCH] Text2Jsonl2WithContextualPrompts: Prompt improved; Name mapping requires at least 5 characters (otherwise we may replace random syllables); --- .../preparer/Text2Jsonl2WithContextualPrompts.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java index 68964cc..e8b4557 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java @@ -87,7 +87,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer, Ini if (j > 2) { final var history = String.join("\n", paragraphs.subList(j - 3, j)); final var mapping = this.createCharacterNameMapping(history); - final var previous = this.paragraphToContextualPrompt(this.renameCharacters(history, mapping), "Rewrite the following text as a user prompt that asks to continue a story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); + final var previous = this.paragraphToContextualPrompt(this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); qaLine = new QaLine(previous, this.renameCharacters(paragraphs.get(j), mapping)); } else { @@ -156,9 +156,12 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer, Ini if (names.females != null) { names.females.forEach(name -> mapping.put(name, this.femaleNames.get(random.nextInt(this.femaleNames.size())))); } + mapping.keySet().removeIf(key -> key.length() < 5); if (mapping.isEmpty()) { return Map.of(); } + + LOGGER.info(mapping.toString()); return mapping; }