diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java b/src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java index cfdf513..fb95982 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java @@ -112,7 +112,7 @@ public class Rewrite implements Consumer { throw new RuntimeException("Cannot read file", e); } - final List splits = Utils.splitParagraphs(text, 1250, 350); + final List splits = Utils.splitParagraphs(text, 1250, 350, false); for (final String split : splits) { LOGGER.info("Split " + splits.indexOf(split) + "/" + splits.size()); diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl.java b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl.java index 8dcd4c5..cf88e36 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl.java @@ -67,18 +67,9 @@ public class Text2Jsonl implements Consumer { final List jsonl = new ArrayList<>(); final ObjectMapper mapper = new ObjectMapper(); - final List paragraphs = Utils.splitParagraphs(text, chunkNorm, chunkVar); + final List paragraphs = Utils.splitParagraphs(text, chunkNorm, chunkVar, true); for (int i = 0; i < paragraphs.size(); i++) { - String split = paragraphs.get(i); - - if (i > 0) { // Add sliding window - final String prevSplit = paragraphs.get(i - 1); - final List prevSplitted = Utils.splitParagraphs(prevSplit, prevSplit.length() - chunkVar, chunkVar - 1); - if (prevSplitted.size() > 1) { - split = prevSplitted.getLast() + "\n\n" + split; - } - } - + final String split = paragraphs.get(i); try { jsonl.add(mapper.writeValueAsString(new TextLine(split))); } diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java index 33c923f..bd752bb 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java @@ -48,6 +48,8 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { final List tasks = new ArrayList<>(); for (final ChunkCfg chunkCfg : chunkConfigurations) { + final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); + for (int i = 0; i < variations; i++) { final Path out = Utils.suffixFilename(Path.of(args[2]).resolve(p.getFileName()), "contextualds", @@ -61,7 +63,6 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { tasks.add(() -> { try (var fos = Files.newOutputStream(out)) { - final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); for (final String paragraph : paragraphs) { LOGGER.info("Paragraph " + paragraphs.indexOf(paragraph) + "/" + paragraphs.size()); final var qaLine = this.paragraphToContextualPrompt(paragraph); @@ -80,7 +81,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { } List fileToParagraphs(final Path file, final int chunkNorm, final int chunkVar) { - LOGGER.info("Splitting to jsonl " + file); + LOGGER.info("Splitting to jsonl " + file + " -> " + chunkNorm + "±" + chunkVar); String text; try { @@ -90,7 +91,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { throw new RuntimeException("Cannot read file", e); } - return Utils.splitParagraphs(text, chunkNorm, chunkVar); + return Utils.splitParagraphs(text, chunkNorm, chunkVar, true); } private QaLine paragraphToContextualPrompt(final String paragraph) { @@ -98,7 +99,8 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { "Enhance the given text by crafting a prompt that effectively provides additional context, enabling the language model to better understand the subject matter and generate more informative and relevant responses. This prompt should be clear, concise, and tailored to facilitate contextual learning, allowing the model to grasp the essential details and nuances of the topic. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:", paragraph, ModelCategory.COLD); - final var qaLine = new QaLine(completion.response().replace("Prompt:", ""), paragraph); + String q = completion.response().replace("Prompt:", "").trim(); + final var qaLine = new QaLine(q, paragraph); return qaLine; } diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Translate.java b/src/main/java/de/extio/lmdatasetprep/preparer/Translate.java index ad0aa0d..fcd30da 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Translate.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Translate.java @@ -73,7 +73,7 @@ public class Translate implements Consumer { throw new RuntimeException("Cannot read file", e); } - final List splits = Utils.splitParagraphs(text, 1250, 350); + final List splits = Utils.splitParagraphs(text, 1250, 350, false); for (final String split : splits) { LOGGER.info("Split " + splits.indexOf(split) + "/" + splits.size()); diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java index 26c7904..24c292d 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java @@ -19,6 +19,8 @@ import org.slf4j.LoggerFactory; public class Utils { + private static final List PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?"); + private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); static String normalizeText(final String text) { @@ -59,7 +61,7 @@ public class Utils { return result; } - static List splitParagraphs(final String text, final int chunks_norm, final int chunks_var) { + static List splitParagraphs(final String text, final int chunks_norm, final int chunks_var, final boolean slidingWindows) { final int CHUNKS_MIN = chunks_norm - chunks_var; final int CHUNKS_MAX = chunks_norm + chunks_var; @@ -70,34 +72,39 @@ public class Utils { final List splits = new ArrayList<>(); int pos = 0; - paragraphs: do { + do { + int next = -1; + int delLength = 1; if (text.length() - pos > CHUNKS_MAX) { - delimiters: for (final String delimiter : List.of("\n\n", "\n", ".")) { - int next = text.indexOf(delimiter, pos); - while (next > -1) { - if (next < pos + CHUNKS_MIN) { - next = text.indexOf(delimiter, next + 1); - } - else if (next > pos + CHUNKS_MAX) { - continue delimiters; - } - else { - String split = text.substring(pos, next).trim(); - if (!split.endsWith(".")) { - split = split + "."; - } - splits.add(split); - pos = next + delimiter.length(); - continue paragraphs; - } + for (final String delimiter : PARAGRAPH_DELIMITERS) { + final int o = text.indexOf(delimiter, pos + CHUNKS_MIN); + if (o > -1 && o <= pos + CHUNKS_MAX) { + next = o; + delLength = delimiter.length(); + break; } } } + if (next == -1) { + next = Math.min(pos + CHUNKS_MAX, text.length()); + } - final int next = Math.min(pos + CHUNKS_MAX, text.length()); - final String split = text.substring(pos, next).trim(); + if (slidingWindows) { + for (final String delimiter : PARAGRAPH_DELIMITERS) { + final int o = text.indexOf(delimiter, Math.max(0, pos - chunks_var)); + if (o > -1 && o < pos) { + pos = o; + break; + } + } + } + String split = text.substring(pos, next).trim(); + if (!split.endsWith(".") && !split.endsWith("!") && !split.endsWith("?")) { + split = split + "."; + } splits.add(split); - pos = next + 1; + + pos = next + delLength; } while (pos < text.length()); LOGGER.info("Created {} splits", splits.size());