DatasetTool marker interface;

This commit is contained in:
extio-de
2024-12-18 20:42:12 +01:00
parent 54ebdb2e51
commit 1c94b97b92
8 changed files with 22 additions and 7 deletions

View File

@@ -1,5 +1,6 @@
package de.extio.lmdatasetprep;
import java.util.Arrays;
import java.util.function.Consumer;
import org.slf4j.Logger;
@@ -13,6 +14,8 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.annotation.ComponentScan;
import de.extio.lmdatasetprep.preparer.DatasetTool;
@SpringBootApplication
@EnableCaching
@ComponentScan(basePackages = { "de.extio.lmdatasetprep", "de.extio.lmlib" })
@@ -31,12 +34,14 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
public void run(final String... args) throws Exception {
if (args.length == 0) {
LOGGER.error("Commandline argument is missing: Provide the tool name as argument");
this.logAvailableTools();
return;
}
final var bean = applicationContext.getBean(args[0]);
if (!(bean instanceof Consumer)) {
if (!(bean instanceof Consumer) || !(bean instanceof DatasetTool)) {
LOGGER.error("Invalid bean {}", args[0]);
this.logAvailableTools();
return;
}
@@ -49,6 +54,10 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
}
}
private void logAvailableTools() {
LOGGER.error("Available tools: {}", Arrays.toString(applicationContext.getBeanNamesForType(DatasetTool.class)));
}
@Override
public void setApplicationContext(final ApplicationContext appContext) throws BeansException {
applicationContext = appContext;

View File

@@ -0,0 +1,6 @@
package de.extio.lmdatasetprep.preparer;
public interface DatasetTool {
}

View File

@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@Component
public class MergeJsonl implements Consumer<String[]> {
public class MergeJsonl implements Consumer<String[]>, DatasetTool {
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);

View File

@@ -19,7 +19,7 @@ import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
@Component
public class Rewrite implements Consumer<String[]> {
public class Rewrite implements Consumer<String[]>, DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class);

View File

@@ -16,7 +16,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@Component
public class Text2Jsonl implements Consumer<String[]> {
public class Text2Jsonl implements Consumer<String[]>, DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl.class);

View File

@@ -13,7 +13,7 @@ import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
@Component
public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts {
public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts implements DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2HistoryInstructCompletion.class);

View File

@@ -13,7 +13,7 @@ import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
@Component
public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts {
public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts implements DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2WithContextualPrompts.class);

View File

@@ -17,7 +17,7 @@ import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
@Component
public class Translate implements Consumer<String[]> {
public class Translate implements Consumer<String[]>, DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Translate.class);