diff --git a/src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java b/src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java index 1c1d1e7..12741f7 100644 --- a/src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java +++ b/src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java @@ -32,6 +32,11 @@ public class MergeJsonl implements DatasetTool { break; } Execution.transform(source, p -> { + final var filter = properties.getProperty("mergeJsonl.filter", ""); + if (!filter.isEmpty() && !p.file().getFileName().toString().contains(filter)) { + return List.of(); + } + return List.of(() -> { try { lines.addAll(Files.readAllLines(p.file())); diff --git a/src/main/resources/tool.properties.template b/src/main/resources/tool.properties.template index 0b4e831..b59d62e 100644 --- a/src/main/resources/tool.properties.template +++ b/src/main/resources/tool.properties.template @@ -11,6 +11,7 @@ rewrite.improve=1 rewrite.enhance=2 rewrite.model=l3dp +mergeJsonl.filter= # .n indicates that the key supports sequences. Add as many keys as you need mergeJsonl.source.0= # Currently we only support at most 2 destinations. This might change in the future @@ -25,6 +26,7 @@ contextualPrompts.category=SMALL contextualPrompts.source=queue1 contextualPrompts.destination.0=/home/xxx/outputdir contextualPrompts.model=l3dp +contextualPrompts.removePreamble=true contextualPrompts.variations=1 contextualPrompts.chunkNorm.0=750 contextualPrompts.chunkVar.0=250