exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mergeJsonl.filter;

Browse Source
This commit is contained in:
extio-de
2025-01-11 22:00:04 +01:00
parent 282b5ffb7f
commit 2628cce48e
2 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java
View File

@@ -32,6 +32,11 @@ public class MergeJsonl implements DatasetTool {
break; break;
} }
Execution.transform(source, p -> { Execution.transform(source, p -> {
final var filter = properties.getProperty("mergeJsonl.filter", "");
if (!filter.isEmpty() && !p.file().getFileName().toString().contains(filter)) {
return List.of();
}
return List.of(() -> { return List.of(() -> {
try { try {
lines.addAll(Files.readAllLines(p.file())); lines.addAll(Files.readAllLines(p.file()));

2
src/main/resources/tool.properties.template
View File

@@ -11,6 +11,7 @@ rewrite.improve=1
rewrite.enhance=2 rewrite.enhance=2
rewrite.model=l3dp rewrite.model=l3dp
mergeJsonl.filter=
# .n indicates that the key supports sequences. Add as many keys as you need # .n indicates that the key supports sequences. Add as many keys as you need
mergeJsonl.source.0= mergeJsonl.source.0=
# Currently we only support at most 2 destinations. This might change in the future # Currently we only support at most 2 destinations. This might change in the future
@@ -25,6 +26,7 @@ contextualPrompts.category=SMALL
contextualPrompts.source=queue1 contextualPrompts.source=queue1
contextualPrompts.destination.0=/home/xxx/outputdir contextualPrompts.destination.0=/home/xxx/outputdir
contextualPrompts.model=l3dp contextualPrompts.model=l3dp
contextualPrompts.removePreamble=true
contextualPrompts.variations=1 contextualPrompts.variations=1
contextualPrompts.chunkNorm.0=750 contextualPrompts.chunkNorm.0=750
contextualPrompts.chunkVar.0=250 contextualPrompts.chunkVar.0=250