mergeJsonl.filter;

src/main/java/de/extio/lmdatasetprep/tools/MergeJsonl.java

@@ -32,6 +32,11 @@ public class MergeJsonl implements DatasetTool {
 				break;
 			}
 			Execution.transform(source, p -> {
+				final var filter = properties.getProperty("mergeJsonl.filter", "");
+				if (!filter.isEmpty() && !p.file().getFileName().toString().contains(filter)) {
+					return List.of();
+				}
+				
 				return List.of(() -> {
 					try {
 						lines.addAll(Files.readAllLines(p.file()));

src/main/resources/tool.properties.template

@@ -11,6 +11,7 @@ rewrite.improve=1
 rewrite.enhance=2
 rewrite.model=l3dp
 
+mergeJsonl.filter=
 # .n indicates that the key supports sequences. Add as many keys as you need
 mergeJsonl.source.0=
 # Currently we only support at most 2 destinations. This might change in the future
@@ -25,6 +26,7 @@ contextualPrompts.category=SMALL
 contextualPrompts.source=queue1
 contextualPrompts.destination.0=/home/xxx/outputdir
 contextualPrompts.model=l3dp
+contextualPrompts.removePreamble=true
 contextualPrompts.variations=1
 contextualPrompts.chunkNorm.0=750
 contextualPrompts.chunkVar.0=250