exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sliding contexts for contextual prompts;

Browse Source
This commit is contained in:
extio-de
2024-11-01 15:08:26 +01:00
parent 7232387134
commit 65e50e1dcc
5 changed files with 40 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java
View File

@@ -112,7 +112,7 @@ public class Rewrite implements Consumer<String[]> {
throw new RuntimeException("Cannot read file", e);
}
final List<String> splits = Utils.splitParagraphs(text, 1250, 350);
final List<String> splits = Utils.splitParagraphs(text, 1250, 350, false);
for (final String split : splits) {
LOGGER.info("Split " + splits.indexOf(split) + "/" + splits.size());