exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved rewriting;

Browse Source
This commit is contained in:
extio-de
2024-10-27 09:54:34 +01:00
parent 78a814a64e
commit 030bc5bee1
6 changed files with 21 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -20,10 +20,6 @@ public class Utils {
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
static final int PARAGRAPH_CHUNKS_NORM = 1250;
static final int PARAGRAPH_CHUNKS_VAR = 350;
static String normalizeText(final String text) {
String result = text;