exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

splitParagraphs

Browse Source
This commit is contained in:
extio-de
2024-11-01 15:31:24 +01:00
parent 65e50e1dcc
commit 14bd1c1915
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -89,9 +89,9 @@ public class Utils {
next = Math.min(pos + CHUNKS_MAX, text.length()); next = Math.min(pos + CHUNKS_MAX, text.length());
} }
if (slidingWindows) { if (slidingWindows && pos > chunks_var) {
for (final String delimiter : PARAGRAPH_DELIMITERS) { for (final String delimiter : PARAGRAPH_DELIMITERS) {
final int o = text.indexOf(delimiter, Math.max(0, pos - chunks_var)); final int o = text.indexOf(delimiter, pos - chunks_var);
if (o > -1 && o < pos) { if (o > -1 && o < pos) {
pos = o; pos = o;
break; break;