exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

splitParagraphs: Split by the closest occurence to chunk_norm instead of chunk_min;

Browse Source
This commit is contained in:
extio-de
2024-11-16 10:52:28 +01:00
parent 82f969d666
commit 70822207dd
1 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -79,8 +79,20 @@ public class Utils {
int delLength = 1; int delLength = 1;
if (text.length() - pos > CHUNKS_MAX) { if (text.length() - pos > CHUNKS_MAX) {
for (final String delimiter : PARAGRAPH_DELIMITERS) { for (final String delimiter : PARAGRAPH_DELIMITERS) {
final int o = text.indexOf(delimiter, pos + CHUNKS_MIN); final int oForw = text.substring(Math.min(pos + chunks_norm, text.length()), Math.min(pos + CHUNKS_MAX, text.length())).indexOf(delimiter);
if (o > -1 && o <= pos + CHUNKS_MAX) { final int oBack = text.substring(Math.min(pos + CHUNKS_MIN, text.length()), Math.min(pos + chunks_norm, text.length())).lastIndexOf(delimiter);
int o = -1;
if (oForw > -1 && oBack == -1) {
o = pos + chunks_norm + oForw;
}
else if (oForw == -1 && oBack > -1) {
o = pos + CHUNKS_MIN + oBack;
}
else if (oForw > -1 && oBack > -1) {
o = oForw <= (chunks_var - oBack) ? pos + chunks_norm + oForw : pos + CHUNKS_MIN + oBack;
}
if (o > -1) {
next = o; next = o;
delLength = delimiter.length(); delLength = delimiter.length();
break; break;