From 14bd1c1915efcc328d8f94c00d0547c8dc44ea57 Mon Sep 17 00:00:00 2001 From: extio-de Date: Fri, 1 Nov 2024 15:31:24 +0100 Subject: [PATCH] splitParagraphs --- src/main/java/de/extio/lmdatasetprep/preparer/Utils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java index 24c292d..5112087 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java @@ -89,9 +89,9 @@ public class Utils { next = Math.min(pos + CHUNKS_MAX, text.length()); } - if (slidingWindows) { + if (slidingWindows && pos > chunks_var) { for (final String delimiter : PARAGRAPH_DELIMITERS) { - final int o = text.indexOf(delimiter, Math.max(0, pos - chunks_var)); + final int o = text.indexOf(delimiter, pos - chunks_var); if (o > -1 && o < pos) { pos = o; break;