exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

splitParagraphs

Browse Source
This commit is contained in:
extio-de
2024-11-01 15:48:28 +01:00
parent 14bd1c1915
commit e40e648557
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -92,8 +92,8 @@ public class Utils {
if (slidingWindows && pos > chunks_var) { if (slidingWindows && pos > chunks_var) {
for (final String delimiter : PARAGRAPH_DELIMITERS) { for (final String delimiter : PARAGRAPH_DELIMITERS) {
final int o = text.indexOf(delimiter, pos - chunks_var); final int o = text.indexOf(delimiter, pos - chunks_var);
if (o > -1 && o < pos) { if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) {
pos = o; pos = o + delimiter.length();
break; break;
} }
} }