From dda9400cc196de3063e88c17f7541084354c832e Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 16 Nov 2024 10:54:06 +0100 Subject: [PATCH] Text2Jsonl2WithContextualPrompts: Overlap chunks; --- .../Text2Jsonl2WithContextualPrompts.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java index 52135b8..7fab460 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java @@ -63,9 +63,19 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { tasks.add(() -> { try (var fos = Files.newOutputStream(out)) { - for (final String paragraph : paragraphs) { - LOGGER.info("Paragraph " + paragraphs.indexOf(paragraph) + "/" + paragraphs.size()); - final var qaLine = this.paragraphToContextualPrompt(paragraph); + for (int j = 0; j < paragraphs.size(); j++) { + LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size()); + + final var paragraph = paragraphs.get(j); + final var prompt = this.paragraphToContextualPrompt(paragraph); + QaLine qaLine; + if (j < paragraphs.size() - 1) { + qaLine = new QaLine(prompt, paragraphs.get(j) + "\n\n" + paragraphs.get(j + 1)); + } + else { + qaLine = new QaLine(prompt, paragraphs.get(j)); + } + this.writeJsonLine(mapper, fos, qaLine); } } @@ -91,12 +101,12 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { throw new RuntimeException("Cannot read file", e); } - return Utils.splitParagraphs(text, chunkNorm, chunkVar, true); + return Utils.splitParagraphs(text, chunkNorm, chunkVar, false); } - private QaLine paragraphToContextualPrompt(final String paragraph) { + private String paragraphToContextualPrompt(final String paragraph) { final var completion = this.client.completion("You are an assistant with great text rewriting skills.", - "Rewrite the following text as a user prompt. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:", + "Rewrite the following text as a user prompt. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:", paragraph, ModelCategory.COLD); @@ -111,8 +121,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { } prompt = prompt.trim(); - final var qaLine = new QaLine(prompt, paragraph); - return qaLine; + return prompt; } private void writeJsonLine(final ObjectMapper mapper, final OutputStream fos, final QaLine qaLine) {