DatasetTool marker interface;

This commit is contained in:
extio-de
2024-12-18 20:42:12 +01:00
parent 54ebdb2e51
commit 1c94b97b92
8 changed files with 22 additions and 7 deletions

View File

@@ -1,5 +1,6 @@
package de.extio.lmdatasetprep; package de.extio.lmdatasetprep;
import java.util.Arrays;
import java.util.function.Consumer; import java.util.function.Consumer;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -13,6 +14,8 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware; import org.springframework.context.ApplicationContextAware;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan;
import de.extio.lmdatasetprep.preparer.DatasetTool;
@SpringBootApplication @SpringBootApplication
@EnableCaching @EnableCaching
@ComponentScan(basePackages = { "de.extio.lmdatasetprep", "de.extio.lmlib" }) @ComponentScan(basePackages = { "de.extio.lmdatasetprep", "de.extio.lmlib" })
@@ -31,12 +34,14 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
public void run(final String... args) throws Exception { public void run(final String... args) throws Exception {
if (args.length == 0) { if (args.length == 0) {
LOGGER.error("Commandline argument is missing: Provide the tool name as argument"); LOGGER.error("Commandline argument is missing: Provide the tool name as argument");
this.logAvailableTools();
return; return;
} }
final var bean = applicationContext.getBean(args[0]); final var bean = applicationContext.getBean(args[0]);
if (!(bean instanceof Consumer)) { if (!(bean instanceof Consumer) || !(bean instanceof DatasetTool)) {
LOGGER.error("Invalid bean {}", args[0]); LOGGER.error("Invalid bean {}", args[0]);
this.logAvailableTools();
return; return;
} }
@@ -49,6 +54,10 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
} }
} }
private void logAvailableTools() {
LOGGER.error("Available tools: {}", Arrays.toString(applicationContext.getBeanNamesForType(DatasetTool.class)));
}
@Override @Override
public void setApplicationContext(final ApplicationContext appContext) throws BeansException { public void setApplicationContext(final ApplicationContext appContext) throws BeansException {
applicationContext = appContext; applicationContext = appContext;

View File

@@ -0,0 +1,6 @@
package de.extio.lmdatasetprep.preparer;
public interface DatasetTool {
}

View File

@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Component @Component
public class MergeJsonl implements Consumer<String[]> { public class MergeJsonl implements Consumer<String[]>, DatasetTool {
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8); private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);

View File

@@ -19,7 +19,7 @@ import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelCategory;
@Component @Component
public class Rewrite implements Consumer<String[]> { public class Rewrite implements Consumer<String[]>, DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class); private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class);

View File

@@ -16,7 +16,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@Component @Component
public class Text2Jsonl implements Consumer<String[]> { public class Text2Jsonl implements Consumer<String[]>, DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl.class); private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl.class);

View File

@@ -13,7 +13,7 @@ import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@Component @Component
public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts { public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts implements DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2HistoryInstructCompletion.class); private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2HistoryInstructCompletion.class);

View File

@@ -13,7 +13,7 @@ import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@Component @Component
public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts { public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts implements DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2WithContextualPrompts.class); private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2WithContextualPrompts.class);

View File

@@ -17,7 +17,7 @@ import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelCategory;
@Component @Component
public class Translate implements Consumer<String[]> { public class Translate implements Consumer<String[]>, DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Translate.class); private static final Logger LOGGER = LoggerFactory.getLogger(Translate.class);