exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch dataset preparer to lmlib;

Browse Source
This commit is contained in:
extio-de
2024-12-18 20:05:03 +01:00
parent fb34561bd5
commit 75c2109d6d
38 changed files with 248 additions and 1679 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
src/main/java/de/extio/lmdatasetprep/preparer/AbstractContextualPrompts.java Normal file
View File

@@ -0,0 +1,176 @@
package de.extio.lmdatasetprep.preparer;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
abstract class AbstractContextualPrompts implements InitializingBean, Consumer<String[]> {
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractContextualPrompts.class);
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
@Autowired
protected ClientService clientService;
private List<String> maleNames = new ArrayList<>();
private List<String> femaleNames = new ArrayList<>();
@Override
public void afterPropertiesSet() throws Exception {
this.maleNames = new BufferedReader(new InputStreamReader(new ClassPathResource("male.txt").getInputStream())).lines().collect(Collectors.toList());
this.femaleNames = new BufferedReader(new InputStreamReader(new ClassPathResource("female.txt").getInputStream())).lines().collect(Collectors.toList());
}
@Override
public void accept(final String[] args) {
if (args.length < 6) {
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> [<chunk norm> <chunk var>, ...]");
return;
}
final int variations = Integer.parseInt(args[3]);
final List<ChunkCfg> chunkConfigurations = new ArrayList<>();
for (int i = 4; i < args.length; i += 2) {
final var cfg = new ChunkCfg(Integer.parseInt(args[i]), Integer.parseInt(args[i + 1]));
chunkConfigurations.add(cfg);
}
Utils.transformDirectory(args[1], this.createTasks(args, chunkConfigurations, variations)::createTasks);
}
@FunctionalInterface
static interface CreateTasks {
List<Runnable> createTasks(Path path);
}
protected abstract CreateTasks createTasks(String[] args, List<ChunkCfg> chunkConfigurations, int variations);
protected List<String> fileToParagraphs(final Path file, final int chunkNorm, final int chunkVar) {
LOGGER.info("Splitting to jsonl " + file + " -> " + chunkNorm + "±" + chunkVar);
String text;
try {
text = Utils.normalizeText(Files.readString(file));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
return Utils.splitParagraphs(text, chunkNorm, chunkVar, false);
}
protected Map<String, String> createCharacterNameMapping(final String paragraph) {
final var completion = this.clientService.getClient(ModelCategory.COLD).completion(ModelCategory.COLD,
"You are a helpful assistant.",
"Extract all names and genders from the following text. Return the result in json format with the following fields: {\"males\" : [ \"Male name 1\", \"Male name 2\", … ], \"females\" : [ \"Female name 1\", \"Female name 2\", … ]}. Don't return a preamble and no explanation:",
paragraph);
final var response = completion.response();
final var start = response.indexOf('{');
final var end = response.lastIndexOf('}');
if (start == -1 || end == -1) {
return Map.of();
}
final var json = response.substring(start, end + 1);
Names names;
final ObjectMapper mapper = new ObjectMapper();
try {
names = mapper.readValue(json, Names.class);
}
catch (final JsonProcessingException e) {
return Map.of();
}
final Random random = ThreadLocalRandom.current();
final Map<String, String> mapping = new HashMap<>();
if (names.males != null) {
names.males.forEach(name -> mapping.put(name, this.maleNames.get(random.nextInt(this.maleNames.size()))));
}
if (names.females != null) {
names.females.forEach(name -> mapping.put(name, this.femaleNames.get(random.nextInt(this.femaleNames.size()))));
}
mapping.keySet().removeIf(key -> key.length() < 5);
if (mapping.isEmpty()) {
return Map.of();
}
LOGGER.debug(mapping.toString());
return mapping;
}
protected String renameCharacters(final String paragraph, final Map<String, String> mapping) {
var result = paragraph;
for (final var entry : mapping.entrySet()) {
result = result.replaceAll("(?i)" + entry.getKey(), entry.getValue());
}
return result;
}
protected String paragraphToContextualPrompt(final String paragraph, final String instruction) {
final var completion = this.clientService.getClient(ModelCategory.COLD).completion(ModelCategory.COLD,
"You are an assistant with great text writing skills.",
instruction,
paragraph);
String prompt = completion.response();
final int colon = prompt.indexOf(':');
if (colon > -1 && colon < prompt.length() + 1) {
prompt = prompt.substring(colon + 1);
}
final int preamble = prompt.indexOf("\n\n");
if (preamble > -1 && preamble < prompt.length() + 2) {
prompt = prompt.substring(preamble + 2);
}
prompt = prompt.trim();
return prompt;
}
protected void writeJsonLine(final ObjectMapper mapper, final OutputStream fos, final Object line) {
try {
final var jsonb = mapper
.writeValueAsString(line)
.getBytes(StandardCharsets.UTF_8);
fos.write(jsonb);
fos.write(NEWLINE);
}
catch (final IOException e) {
LOGGER.error("Cannot convert split to json", e);
}
}
protected record ChunkCfg(int chunksNorm, int chunksVar) {
}
protected record Names(List<String> males, List<String> females) {
}
}

5
src/main/java/de/extio/lmdatasetprep/preparer/MergeJsonl.java
View File

@@ -7,14 +7,13 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.XorShift128Random;
@Component
public class MergeJsonl implements Consumer<String[]> {
@@ -42,7 +41,7 @@ public class MergeJsonl implements Consumer<String[]> {
});
});
Collections.shuffle(lines, new XorShift128Random());
Collections.shuffle(lines, ThreadLocalRandom.current());
final Path out = Path.of(args[2], "dataset-merged.jsonl");
try (var fos = Files.newOutputStream(out)) {

19
src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java
View File

@@ -6,7 +6,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
@@ -15,17 +15,14 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.XorShift128Random;
import de.extio.lmdatasetprep.client.Client;
import de.extio.lmdatasetprep.client.profile.ModelCategory;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
@Component
public class Rewrite implements Consumer<String[]> {
private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class);
private static final ThreadLocal<Random> RANDOM = ThreadLocal.withInitial(XorShift128Random::new);
private static final byte[] PARAGRAPH = "\n\n".getBytes(StandardCharsets.UTF_8);
private static String IMPROVEMENT_PROMPT = "Improve the following text without changing the existing sentences, improve the text in a nuanced and understated way. Return the improved text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:";
@@ -50,7 +47,7 @@ public class Rewrite implements Consumer<String[]> {
"Imbue the setting with a sense of timelessness or age through subtle phrasing and word choice");
@Autowired
private Client client;
private ClientService clientService;
@Override
public void accept(final String[] args) {
@@ -65,7 +62,7 @@ public class Rewrite implements Consumer<String[]> {
tasks.add(() -> this.rewriteFile(args, f, 0, "impr", ModelCategory.COLD, IMPROVEMENT_PROMPT));
for (int i = 0; i < Integer.parseInt(args[3]); i++) {
final int fi = i;
tasks.add(() -> this.rewriteFile(args, f, fi, "enh", ModelCategory.COLD, String.format(ENHANCE_PROMPT, ENHANCEMENTS.get(RANDOM.get().nextInt(ENHANCEMENTS.size())))));
tasks.add(() -> this.rewriteFile(args, f, fi, "enh", ModelCategory.COLD, String.format(ENHANCE_PROMPT, ENHANCEMENTS.get(ThreadLocalRandom.current().nextInt(ENHANCEMENTS.size())))));
}
return tasks;
});
@@ -116,10 +113,10 @@ public class Rewrite implements Consumer<String[]> {
for (final String split : splits) {
LOGGER.info("Split " + (splits.indexOf(split) + 1) + "/" + splits.size());
final var completion = this.client.completion("You are a helpful assistant with great authoring skills.",
final var completion = this.clientService.getClient(modelCategory).completion(modelCategory,
"You are a helpful assistant with great authoring skills.",
prompt,
split,
modelCategory);
split);
consumer.accept(completion.response());
}
}

156
src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
View File

@@ -1,67 +1,26 @@
package de.extio.lmdatasetprep.preparer;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.ResourceUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.XorShift128Random;
import de.extio.lmdatasetprep.client.Client;
import de.extio.lmdatasetprep.client.profile.ModelCategory;
@Component
public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]>, InitializingBean {
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2WithContextualPrompts.class);
private List<String> maleNames = new ArrayList<>();
private List<String> femaleNames = new ArrayList<>();
@Autowired
private Client client;
@Override
public void afterPropertiesSet() throws Exception {
this.maleNames = Files.readAllLines(ResourceUtils.getFile("classpath:male.txt").toPath());
this.femaleNames = Files.readAllLines(ResourceUtils.getFile("classpath:female.txt").toPath());
}
@Override
public void accept(final String[] args) {
if (args.length < 6) {
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> [<chunk norm> <chunk var>, ...]");
return;
}
final int variations = Integer.parseInt(args[3]);
final List<ChunkCfg> chunkConfigurations = new ArrayList<>();
for (int i = 4; i < args.length; i += 2) {
final var cfg = new ChunkCfg(Integer.parseInt(args[i]), Integer.parseInt(args[i + 1]));
chunkConfigurations.add(cfg);
}
final ObjectMapper mapper = new ObjectMapper();
Utils.transformDirectory(args[1], p -> {
protected CreateTasks createTasks(final String[] args, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
@@ -108,112 +67,9 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]>, Ini
}
return tasks;
});
};
}
List<String> fileToParagraphs(final Path file, final int chunkNorm, final int chunkVar) {
LOGGER.info("Splitting to jsonl " + file + " -> " + chunkNorm + "±" + chunkVar);
String text;
try {
text = Utils.normalizeText(Files.readString(file));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
return Utils.splitParagraphs(text, chunkNorm, chunkVar, false);
protected record QaLine(String question, String answer) {
}
private Map<String, String> createCharacterNameMapping(final String paragraph) {
final var completion = this.client.completion("You are a helpful assistant.",
"Extract all names and genders from the following text. Return the result in json format with the following fields: {\"males\" : [ \"Male name 1\", \"Male name 2\", … ], \"females\" : [ \"Female name 1\", \"Female name 2\", … ]}. Don't return a preamble and no explanation:",
paragraph,
ModelCategory.COLD);
final var response = completion.response();
final var start = response.indexOf('{');
final var end = response.lastIndexOf('}');
if (start == -1 || end == -1) {
return Map.of();
}
final var json = response.substring(start, end + 1);
Names names;
final ObjectMapper mapper = new ObjectMapper();
try {
names = mapper.readValue(json, Names.class);
}
catch (final JsonProcessingException e) {
return Map.of();
}
final Random random = new XorShift128Random();
final Map<String, String> mapping = new HashMap<>();
if (names.males != null) {
names.males.forEach(name -> mapping.put(name, this.maleNames.get(random.nextInt(this.maleNames.size()))));
}
if (names.females != null) {
names.females.forEach(name -> mapping.put(name, this.femaleNames.get(random.nextInt(this.femaleNames.size()))));
}
mapping.keySet().removeIf(key -> key.length() < 5);
if (mapping.isEmpty()) {
return Map.of();
}
LOGGER.debug(mapping.toString());
return mapping;
}
private String renameCharacters(final String paragraph, final Map<String, String> mapping) {
var result = paragraph;
for (final var entry : mapping.entrySet()) {
result = result.replaceAll("(?i)" + entry.getKey(), entry.getValue());
}
return result;
}
private String paragraphToContextualPrompt(final String paragraph, final String instruction) {
final var completion = this.client.completion("You are an assistant with great text writing skills.",
instruction,
paragraph,
ModelCategory.COLD);
String prompt = completion.response();
final int colon = prompt.indexOf(':');
if (colon > -1 && colon < prompt.length() + 1) {
prompt = prompt.substring(colon + 1);
}
final int preamble = prompt.indexOf("\n\n");
if (preamble > -1 && preamble < prompt.length() + 2) {
prompt = prompt.substring(preamble + 2);
}
prompt = prompt.trim();
return prompt;
}
private void writeJsonLine(final ObjectMapper mapper, final OutputStream fos, final QaLine qaLine) {
try {
final var jsonb = mapper
.writeValueAsString(qaLine)
.getBytes(StandardCharsets.UTF_8);
fos.write(jsonb);
fos.write(NEWLINE);
}
catch (final IOException e) {
LOGGER.error("Cannot convert split to json", e);
}
}
record QaLine(String question, String answer) {
}
record ChunkCfg(int chunksNorm, int chunksVar) {
}
record Names(List<String> males, List<String> females) {
}
}

12
src/main/java/de/extio/lmdatasetprep/preparer/Translate.java
View File

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.client.Client;
import de.extio.lmdatasetprep.client.profile.ModelCategory;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
@Component
public class Translate implements Consumer<String[]> {
@@ -24,7 +24,7 @@ public class Translate implements Consumer<String[]> {
private static final byte[] PARAGRAPH = "\n\n".getBytes(StandardCharsets.UTF_8);
@Autowired
private Client client;
private ClientService clientService;
@Override
public void accept(final String[] args) {
@@ -77,10 +77,10 @@ public class Translate implements Consumer<String[]> {
for (final String split : splits) {
LOGGER.info("Split " + splits.indexOf(split) + "/" + splits.size());
final var completion = this.client.completion("You are an assistant with great language translation and authoring skills.",
final var completion = this.clientService.getClient(ModelCategory.COLD).completion(ModelCategory.COLD,
"You are an assistant with great language translation and authoring skills.",
"Translate the following text to English. Return the English translation, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:",
split,
ModelCategory.COLD);
split);
consumer.accept(completion.response());
}
}

6
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -17,14 +17,14 @@ import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.extio.lmdatasetprep.LmDatasetPreparerApplication;
public class Utils {
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
private static final List<String> PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?");
private static final int EXECUTORS = 4;
static String normalizeText(final String text) {
String result = text;
@@ -127,8 +127,8 @@ public class Utils {
}
static void transformDirectory(final String path, final Function<Path, List<Runnable>> createTasks) {
final int EXECUTORS = Integer.parseInt(LmDatasetPreparerApplication.applicationContext.getEnvironment().getProperty("agent.threads"));
final AtomicBoolean finished = new AtomicBoolean(false);
final BlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
final List<Thread> threads = new ArrayList<>(EXECUTORS);