From e40e648557c7891b22f33fe7e90c4c086ca33b42 Mon Sep 17 00:00:00 2001
From: extio-de <github@extio.de>
Date: Fri, 1 Nov 2024 15:48:28 +0100
Subject: [PATCH] splitParagraphs

---
 src/main/java/de/extio/lmdatasetprep/preparer/Utils.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
index 5112087..77f371d 100644
--- a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
+++ b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
@@ -92,8 +92,8 @@ public class Utils {
 			if (slidingWindows && pos > chunks_var) {
 				for (final String delimiter : PARAGRAPH_DELIMITERS) {
 					final int o = text.indexOf(delimiter, pos - chunks_var);
-					if (o > -1 && o < pos) {
-						pos = o;
+					if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) {
+						pos = o + delimiter.length();
 						break;
 					}
 				}