From df3034e6ca308b9ae8c02a500fe7fa563897f8d2 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 5 Jul 2025 20:04:56 +0200 Subject: [PATCH] Moved TextUtils to lm-lib --- README.md | 7 + pom.xml | 10 +- .../java/de/extio/lmlib/util/TextUtils.java | 135 ++++++++++++++++++ 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 src/main/java/de/extio/lmlib/util/TextUtils.java diff --git a/README.md b/README.md index f9270af..1e692bf 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,13 @@ To keep this library lean and compatible with different dependency trees, all de ## Optional dependencies +TextUtils require commons-lang3 + + + org.apache.commons + commons-lang3 + + Some services benefit from Spring cache framework, e.g. the model profile service: diff --git a/pom.xml b/pom.xml index e47fdc3..11f44de 100755 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ lm-lib de.extio lm-lib - 1.6.3 + 1.6.4 21 @@ -13,6 +13,7 @@ 3.5.3 1.0.0-beta.5 1.1.0 + 3.12.0 @@ -47,6 +48,13 @@ + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + provided + + org.springframework.boot spring-boot-starter-webflux diff --git a/src/main/java/de/extio/lmlib/util/TextUtils.java b/src/main/java/de/extio/lmlib/util/TextUtils.java new file mode 100644 index 0000000..8c07c79 --- /dev/null +++ b/src/main/java/de/extio/lmlib/util/TextUtils.java @@ -0,0 +1,135 @@ +package de.extio.lmlib.util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TextUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(TextUtils.class); + + private static final List PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?"); + + public static String normalizeText(final String text) { + String result = text; + + // Remove excessive newlines, spaces and tabs + int cnt = 0; + do { + cnt = result.length(); + result = result.replace("\r", "\n"); + result = result.replace("\n\n\n", "\n\n"); + result = result.replace("\t", ""); + result = result.replace(" ", " "); + result = result.replace(" .", "."); + result = result.replace(". ", ". "); + result = result.replace("\n.", "."); + result = result.replace(" !", "!"); + result = result.replace("! ", "! "); + result = result.replace("\n!", "!"); + result = result.replace(" ?", "?"); + result = result.replace("? ", "? "); + result = result.replace("\n?", "?"); + result = result.replace(" \n", "\n"); + } while (result.length() != cnt); + + // Remove newlines mid of the sentence + final StringBuilder sb = new StringBuilder(result); + for (int i = 1; i < sb.length(); i++) { + final char c = sb.charAt(i); + if (c == '\n' && sb.charAt(i - 1) != '.' && sb.charAt(i - 1) != '?' && sb.charAt(i - 1) != '!' && sb.charAt(i - 1) != '\n') { + sb.setCharAt(i, ' '); + } + } + result = sb.toString(); + + LOGGER.info("Normalizer stats: Before: {} After {}", text.length(), result.length()); + + return result; + } + + public static String normalizeModelResponse(final String response, final boolean removePreamble) { + var result = StringUtils.replace(response, "\r", ""); + + if (removePreamble) { + final int colon = result.indexOf(':'); + if (colon > -1 && colon < result.length() - 2 && (colon < 50 || result.charAt(colon + 1) == '\n')) { + result = result.substring(colon + 1); + } + } + + result = StringUtils.trim(result); + result = StringUtils.strip(result, "*#´`'“”„‟«»\"\n"); + result = StringUtils.trim(result); + + return result; + } + + public static List splitParagraphs(final String text, final int chunks_norm, final int chunks_var, final boolean slidingWindows) { + final int CHUNKS_MIN = chunks_norm - chunks_var; + final int CHUNKS_MAX = chunks_norm + chunks_var; + + if (text.length() <= CHUNKS_MAX) { + return Collections.singletonList(text); + } + + final List splits = new ArrayList<>(); + + int pos = 0; + do { + int next = -1; + int delLength = 1; + if (text.length() - pos > CHUNKS_MAX) { + for (final String delimiter : PARAGRAPH_DELIMITERS) { + final int oForw = text.substring(Math.min(pos + chunks_norm, text.length()), Math.min(pos + CHUNKS_MAX, text.length())).indexOf(delimiter); + final int oBack = text.substring(Math.min(pos + CHUNKS_MIN, text.length()), Math.min(pos + chunks_norm, text.length())).lastIndexOf(delimiter); + int o = -1; + if (oForw > -1 && oBack == -1) { + o = pos + chunks_norm + oForw; + } + else if (oForw == -1 && oBack > -1) { + o = pos + CHUNKS_MIN + oBack; + } + else if (oForw > -1 && oBack > -1) { + o = oForw <= (chunks_var - oBack) ? pos + chunks_norm + oForw : pos + CHUNKS_MIN + oBack; + } + + if (o > -1) { + next = o; + delLength = delimiter.length(); + break; + } + } + } + if (next == -1) { + next = Math.min(pos + CHUNKS_MAX, text.length()); + } + + if (slidingWindows && pos > chunks_var) { + for (final String delimiter : PARAGRAPH_DELIMITERS) { + final int o = text.indexOf(delimiter, pos - chunks_var); + if (o > -1 && o < pos - 10) { + pos = o + delimiter.length(); + break; + } + } + } + String split = text.substring(pos, next).trim(); + if (!split.endsWith(".") && !split.endsWith("!") && !split.endsWith("?")) { + split = split + "."; + } + splits.add(split); + + pos = next + delLength; + } while (pos < text.length()); + + LOGGER.info("Created {} splits", splits.size()); + + return splits; + } + +}