From 70822207dd7f45513f0c403d42b654e11a9bef9a Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 16 Nov 2024 10:52:28 +0100 Subject: [PATCH] splitParagraphs: Split by the closest occurence to chunk_norm instead of chunk_min; --- .../de/extio/lmdatasetprep/preparer/Utils.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java index 3d4d1ea..b8475cc 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java @@ -79,8 +79,20 @@ public class Utils { int delLength = 1; if (text.length() - pos > CHUNKS_MAX) { for (final String delimiter : PARAGRAPH_DELIMITERS) { - final int o = text.indexOf(delimiter, pos + CHUNKS_MIN); - if (o > -1 && o <= pos + CHUNKS_MAX) { + final int oForw = text.substring(Math.min(pos + chunks_norm, text.length()), Math.min(pos + CHUNKS_MAX, text.length())).indexOf(delimiter); + final int oBack = text.substring(Math.min(pos + CHUNKS_MIN, text.length()), Math.min(pos + chunks_norm, text.length())).lastIndexOf(delimiter); + int o = -1; + if (oForw > -1 && oBack == -1) { + o = pos + chunks_norm + oForw; + } + else if (oForw == -1 && oBack > -1) { + o = pos + CHUNKS_MIN + oBack; + } + else if (oForw > -1 && oBack > -1) { + o = oForw <= (chunks_var - oBack) ? pos + chunks_norm + oForw : pos + CHUNKS_MIN + oBack; + } + + if (o > -1) { next = o; delLength = delimiter.length(); break;