DatasetTool marker interface;
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
package de.extio.lmdatasetprep;
|
package de.extio.lmdatasetprep;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
import java.util.function.Consumer;
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
@@ -13,6 +14,8 @@ import org.springframework.context.ApplicationContext;
|
|||||||
import org.springframework.context.ApplicationContextAware;
|
import org.springframework.context.ApplicationContextAware;
|
||||||
import org.springframework.context.annotation.ComponentScan;
|
import org.springframework.context.annotation.ComponentScan;
|
||||||
|
|
||||||
|
import de.extio.lmdatasetprep.preparer.DatasetTool;
|
||||||
|
|
||||||
@SpringBootApplication
|
@SpringBootApplication
|
||||||
@EnableCaching
|
@EnableCaching
|
||||||
@ComponentScan(basePackages = { "de.extio.lmdatasetprep", "de.extio.lmlib" })
|
@ComponentScan(basePackages = { "de.extio.lmdatasetprep", "de.extio.lmlib" })
|
||||||
@@ -31,12 +34,14 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
|
|||||||
public void run(final String... args) throws Exception {
|
public void run(final String... args) throws Exception {
|
||||||
if (args.length == 0) {
|
if (args.length == 0) {
|
||||||
LOGGER.error("Commandline argument is missing: Provide the tool name as argument");
|
LOGGER.error("Commandline argument is missing: Provide the tool name as argument");
|
||||||
|
this.logAvailableTools();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
final var bean = applicationContext.getBean(args[0]);
|
final var bean = applicationContext.getBean(args[0]);
|
||||||
if (!(bean instanceof Consumer)) {
|
if (!(bean instanceof Consumer) || !(bean instanceof DatasetTool)) {
|
||||||
LOGGER.error("Invalid bean {}", args[0]);
|
LOGGER.error("Invalid bean {}", args[0]);
|
||||||
|
this.logAvailableTools();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,6 +54,10 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void logAvailableTools() {
|
||||||
|
LOGGER.error("Available tools: {}", Arrays.toString(applicationContext.getBeanNamesForType(DatasetTool.class)));
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setApplicationContext(final ApplicationContext appContext) throws BeansException {
|
public void setApplicationContext(final ApplicationContext appContext) throws BeansException {
|
||||||
applicationContext = appContext;
|
applicationContext = appContext;
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
package de.extio.lmdatasetprep.preparer;
|
||||||
|
|
||||||
|
|
||||||
|
public interface DatasetTool {
|
||||||
|
|
||||||
|
}
|
||||||
@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
|
|||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class MergeJsonl implements Consumer<String[]> {
|
public class MergeJsonl implements Consumer<String[]>, DatasetTool {
|
||||||
|
|
||||||
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
|
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import de.extio.lmlib.client.ClientService;
|
|||||||
import de.extio.lmlib.profile.ModelCategory;
|
import de.extio.lmlib.profile.ModelCategory;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class Rewrite implements Consumer<String[]> {
|
public class Rewrite implements Consumer<String[]>, DatasetTool {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class);
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class Text2Jsonl implements Consumer<String[]> {
|
public class Text2Jsonl implements Consumer<String[]>, DatasetTool {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl.class);
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import org.springframework.stereotype.Component;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts {
|
public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts implements DatasetTool {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2HistoryInstructCompletion.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2HistoryInstructCompletion.class);
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import org.springframework.stereotype.Component;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts {
|
public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts implements DatasetTool {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2WithContextualPrompts.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2WithContextualPrompts.class);
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import de.extio.lmlib.client.ClientService;
|
|||||||
import de.extio.lmlib.profile.ModelCategory;
|
import de.extio.lmlib.profile.ModelCategory;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class Translate implements Consumer<String[]> {
|
public class Translate implements Consumer<String[]>, DatasetTool {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(Translate.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(Translate.class);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user