exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Configurable model categories;

Browse Source
This commit is contained in:
extio-de
2024-12-20 00:14:02 +01:00
parent 74eaa5c72b
commit d44fd53323
9 changed files with 79 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java
View File

@@ -27,7 +27,6 @@ import de.extio.lmdatasetprep.Execution;
import de.extio.lmdatasetprep.Execution.WorkPacket;
import de.extio.lmdatasetprep.TextUtils;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
abstract class AbstractContextualPrompts implements InitializingBean, DatasetTool {
@@ -42,6 +41,11 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
private List<String> femaleNames = new ArrayList<>();
@Override
public String getModelCategoryPropertyName() {
return "contextualPrompts.category";
}
@Override
public void afterPropertiesSet() throws Exception {
this.maleNames = new BufferedReader(new InputStreamReader(new ClassPathResource("male.txt").getInputStream())).lines().collect(Collectors.toList());
@@ -80,8 +84,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
return TextUtils.splitParagraphs(text, chunkNorm, chunkVar, false);
}
protected Map<String, String> createCharacterNameMapping(final String paragraph) {
final var completion = this.clientService.getClient(ModelCategory.COLD).completion(ModelCategory.COLD,
protected Map<String, String> createCharacterNameMapping(final String paragraph, final Properties properties) {
final var completion = this.getClient(properties, this.clientService).completion(this.getModelCategory(properties),
"You are a helpful assistant.",
"Extract all names and genders from the following text. Return the result in json format with the following fields: {\"males\" : [ \"Male name 1\", \"Male name 2\", … ], \"females\" : [ \"Female name 1\", \"Female name 2\", … ]}. Don't return a preamble and no explanation:",
paragraph);
@@ -129,8 +133,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
return result;
}
protected String createPrompt(final String paragraph, final String instruction) {
final var completion = this.clientService.getClient(ModelCategory.COLD).completion(ModelCategory.COLD,
protected String createPrompt(final Properties properties, final String paragraph, final String instruction) {
final var completion = this.getClient(properties, this.clientService).completion(this.getModelCategory(properties),
"You are an assistant with great text writing skills.",
instruction,
paragraph);

16
src/main/java/de/extio/lmdatasetprep/tools/DatasetTool.java
View File

@@ -3,6 +3,22 @@ package de.extio.lmdatasetprep.tools;
import java.util.Properties;
import java.util.function.Consumer;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
public interface DatasetTool extends Consumer<Properties> {
default String getModelCategoryPropertyName() {
return null;
}
default ModelCategory getModelCategory(final Properties properties) {
return ModelCategory.valueOf(properties.getProperty(this.getModelCategoryPropertyName()));
}
default Client getClient(final Properties properties, final ClientService clientService) {
return clientService.getClient(this.getModelCategory(properties));
}
}

22
src/main/java/de/extio/lmdatasetprep/tools/Rewrite.java
View File

@@ -19,7 +19,6 @@ import de.extio.lmdatasetprep.Execution;
import de.extio.lmdatasetprep.Execution.WorkPacket;
import de.extio.lmdatasetprep.TextUtils;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
@Component
public class Rewrite implements DatasetTool {
@@ -52,33 +51,37 @@ public class Rewrite implements DatasetTool {
@Autowired
private ClientService clientService;
@Override
public String getModelCategoryPropertyName() {
return "rewrite.category";
}
@Override
public void accept(final Properties properties) {
Execution.transform(properties.getProperty("rewrite.source"),
packet -> {
final List<Runnable> tasks = new ArrayList<>();
tasks.add(() -> this.rewriteFile(properties, packet, 0, "impr", ModelCategory.COLD, IMPROVEMENT_PROMPT));
tasks.add(() -> this.rewriteFile(properties, packet, 0, "impr", IMPROVEMENT_PROMPT));
for (int i = 0; i < Integer.parseInt(properties.getProperty("rewrite.cnt")); i++) {
final int fi = i;
tasks.add(() -> this.rewriteFile(properties, packet, fi, "enh", ModelCategory.COLD, String.format(ENHANCE_PROMPT, ENHANCEMENTS.get(ThreadLocalRandom.current().nextInt(ENHANCEMENTS.size())))));
tasks.add(() -> this.rewriteFile(properties, packet, fi, "enh", String.format(ENHANCE_PROMPT, ENHANCEMENTS.get(ThreadLocalRandom.current().nextInt(ENHANCEMENTS.size())))));
}
return tasks;
});
}
private void rewriteFile(final Properties properties, final WorkPacket packet, final int i, final String identifier, final ModelCategory modelCategory, final String prompt) {
private void rewriteFile(final Properties properties, final WorkPacket packet, final int i, final String identifier, final String prompt) {
final Path out = Execution.suffixFilename(packet.file(),
"rewr",
properties.getProperty("rewrite.model"),
identifier,
modelCategory.toString().toLowerCase(),
String.valueOf(i));
LOGGER.info("Rewriting " + packet.file().getFileName() + " " + identifier + " " + modelCategory.name() + " " + i);
LOGGER.info("Rewriting " + packet.file().getFileName() + " " + identifier + " " + " " + i);
Execution.streamOut(out, "rewrite.destination", properties, fos -> {
final AtomicBoolean first = new AtomicBoolean(true);
this.rewrite(packet, prompt, modelCategory, chunk -> {
this.rewrite(packet, prompt, properties, chunk -> {
try {
if (!first.getAndSet(false)) {
fos.write(PARAGRAPH);
@@ -92,12 +95,13 @@ public class Rewrite implements DatasetTool {
});
}
void rewrite(final WorkPacket packet, final String prompt, final ModelCategory modelCategory, final Consumer<String> consumer) {
void rewrite(final WorkPacket packet, final String prompt, final Properties properties, final Consumer<String> consumer) {
final String text = TextUtils.normalizeText(packet.text());
final List<String> splits = TextUtils.splitParagraphs(text, 1250, 350, false);
final var client = this.getClient(properties, this.clientService);
for (final String split : splits) {
LOGGER.info("Split " + (splits.indexOf(split) + 1) + "/" + splits.size());
final var completion = this.clientService.getClient(modelCategory).completion(modelCategory,
final var completion = client.completion(this.getModelCategory(properties),
"You are a helpful assistant with great authoring skills.",
prompt,
split);

4
src/main/java/de/extio/lmdatasetprep/tools/Text2Jsonl2HistoryInstructCompletion.java
View File

@@ -41,8 +41,8 @@ public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualProm
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
final var mapping = this.createCharacterNameMapping(inScope);
final var instruct = this.createPrompt(this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var mapping = this.createCharacterNameMapping(inScope, properties);
final var instruct = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new QaLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
this.writeJsonLine(mapper, fos, qaLine);

8
src/main/java/de/extio/lmdatasetprep/tools/Text2Jsonl2WithContextualPrompts.java
View File

@@ -43,14 +43,14 @@ public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts
QaLine qaLine;
if (j > 2) {
final var history = String.join("\n", paragraphs.subList(j - 3, j));
final var mapping = this.createCharacterNameMapping(history);
final var previous = this.createPrompt(this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var mapping = this.createCharacterNameMapping(history, properties);
final var previous = this.createPrompt(properties, this.renameCharacters(history, mapping), "Write a summary of the following story as a user prompt that asks to write the story. Mention the place where they are and ask the question what could happen next. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
qaLine = new QaLine(previous, this.renameCharacters(paragraphs.get(j), mapping));
}
else {
final var mapping = this.createCharacterNameMapping(paragraphs.get(j));
final var mapping = this.createCharacterNameMapping(paragraphs.get(j), properties);
final var paragraph = this.renameCharacters(paragraphs.get(j), mapping);
final var userPrompt = this.createPrompt(paragraph, "Write a summary of the following story as a user prompt that asks to write the story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var userPrompt = this.createPrompt(properties, paragraph, "Write a summary of the following story as a user prompt that asks to write the story. Include all essential details, enabling the user to better understand the subject matter. Return the prompt, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
qaLine = new QaLine(userPrompt, paragraph);
}

14
src/main/java/de/extio/lmdatasetprep/tools/Translate.java
View File

@@ -17,7 +17,6 @@ import de.extio.lmdatasetprep.Execution;
import de.extio.lmdatasetprep.Execution.WorkPacket;
import de.extio.lmdatasetprep.TextUtils;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
@Component
public class Translate implements DatasetTool {
@@ -29,6 +28,11 @@ public class Translate implements DatasetTool {
@Autowired
private ClientService clientService;
@Override
public String getModelCategoryPropertyName() {
return "translate.category";
}
@Override
public void accept(final Properties properties) {
Execution.transform(properties.getProperty("translate.source"), p -> {
@@ -36,7 +40,7 @@ public class Translate implements DatasetTool {
final Path out = Execution.suffixFilename(p.file().getFileName(), "en");
Execution.streamOut(out, "translate.destination", properties, fos -> {
final AtomicBoolean first = new AtomicBoolean(true);
this.translate(p, chunk -> {
this.translate(p, properties, chunk -> {
try {
if (!first.getAndSet(false)) {
fos.write(PARAGRAPH);
@@ -52,16 +56,18 @@ public class Translate implements DatasetTool {
});
}
void translate(final WorkPacket packet, final Consumer<String> consumer) {
void translate(final WorkPacket packet, final Properties properties, final Consumer<String> consumer) {
LOGGER.info("Translating " + packet.file());
final String text = TextUtils.normalizeText(packet.text());
final List<String> splits = TextUtils.splitParagraphs(text, 1250, 350, false);
final var client = this.getClient(properties, this.clientService);
for (final String split : splits) {
LOGGER.info("Split " + (splits.indexOf(split) + 1) + "/" + splits.size());
final var completion = this.clientService.getClient(ModelCategory.COLD).completion(ModelCategory.COLD,
final var completion = client.completion(this.getModelCategory(properties),
"You are an assistant with great language translation and authoring skills.",
"Translate the following text to English. Return the English translation, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:",
split);

11
src/main/resources/alpaca-cold.properties Executable file
View File

@@ -0,0 +1,11 @@
prompts=alpaca
tokenEncoding=
maxTokens=2000
maxContextLength=4000
temperature=0.2
topP=1.0
maxContinuations=1
modelProvider=OAI_TEXT_COMPLETION
modelName=
url=http://localhost:5001
apiKey=

7
src/main/resources/application.properties
View File

@@ -3,11 +3,10 @@ spring.main.web-application-type=NONE
logging.level.de.extio=info
profile.model.s=llama3-cold
profile.model.m=llama3-cold
profile.model.l=llama3-cold
profile.model.m=mixtral-cold
profile.model.l=chatml-cold
profile.model.hot=llama3-hot
profile.model.cold=llama3-cold
#profile.model.cold=mixtral-cold
profile.model.cold=alpaca-cold
agent.threads=4

11
src/main/resources/chatml-cold.properties Executable file
View File

@@ -0,0 +1,11 @@
prompts=chatml
tokenEncoding=
maxTokens=2000
maxContextLength=4000
temperature=0.2
topP=1.0
maxContinuations=1
modelProvider=OAI_TEXT_COMPLETION
modelName=
url=http://localhost:5001
apiKey=