splitParagraphs

2024-11-01 15:48:28 +01:00
parent 14bd1c1915
commit e40e648557
1 changed files with 2 additions and 2 deletions
--- a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
+++ b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
@@ -92,8 +92,8 @@ public class Utils {
 			if (slidingWindows && pos > chunks_var) {
 				for (final String delimiter : PARAGRAPH_DELIMITERS) {
 					final int o = text.indexOf(delimiter, pos - chunks_var);
-					if (o > -1 && o < pos) {
-						pos = o;
+					if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) {
+						pos = o + delimiter.length();
 						break;
 					}
 				}