exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New tool: Create; New tool: Contextual instructions; Rewrite: Configure cnt for both impr and enh; Task Execution: Reimplemented task queuing approach; Fixes;

Browse Source
This commit is contained in:
extio-de
2024-12-26 21:49:51 +01:00
parent 07e5a4779a
commit 685c740c5c
14 changed files with 628 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/main/java/de/extio/lmdatasetprep/Execution.java
View File

@@ -10,13 +10,12 @@ import java.nio.file.Paths;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -38,25 +37,20 @@ public class Execution {
public static final ConcurrentMap<String, ConcurrentMap<String, BlockingQueue<WorkPacket>>> work = new ConcurrentHashMap<>();
public static final BlockingQueue<CompletableFuture<Void>> tasks;
public static final ScheduledExecutorService scheduler;
public static final ExecutorService executorService;
public static final AtomicInteger tasksRunning = new AtomicInteger();
private static final Semaphore taskSemaphore;
private static final Logger LOGGER = LoggerFactory.getLogger(Execution.class);
static {
final int EXECUTORS = Integer.parseInt(LmDatasetPreparerApplication.applicationContext.getEnvironment().getProperty("agent.threads"));
tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
executorService = Executors.newFixedThreadPool(EXECUTORS);
scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(() -> {
tasks.removeIf(future -> future.isDone());
}, 0, 1, TimeUnit.SECONDS);
taskSemaphore = new Semaphore(EXECUTORS * 2);
}
public static void transform(final String source, final Function<WorkPacket, List<Runnable>> createTasks) {
@@ -83,12 +77,7 @@ public class Execution {
})
.forEach(p -> {
for (final Runnable r : createTasks.apply(p)) {
try {
tasks.put(CompletableFuture.runAsync(r, executorService));
}
catch (final InterruptedException e) {
throw new RuntimeException("Interrupted", e);
}
executeTask(r);
}
});
}
@@ -120,16 +109,30 @@ public class Execution {
WorkPacket next;
while ((next = getNext.get()) != null) {
for (final Runnable r : createTasks.apply(next)) {
try {
tasks.put(CompletableFuture.runAsync(r, executorService));
}
catch (final InterruptedException e) {
throw new RuntimeException("Interrupted", e);
}
executeTask(r);
}
}
}
private static void executeTask(final Runnable r) {
try {
taskSemaphore.acquire();
tasksRunning.incrementAndGet();
executorService.execute(() -> {
try {
r.run();
}
finally {
tasksRunning.decrementAndGet();
taskSemaphore.release();
}
});
}
catch (final InterruptedException e) {
throw new RuntimeException("Interrupted", e);
}
}
public static Path suffixFilename(final Path f, final String... suffixes) {
final StringBuilder sb = new StringBuilder();
for (final String suffix : suffixes) {

7
src/main/java/de/extio/lmdatasetprep/LmDatasetPreparerApplication.java
View File

@@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.ComponentScan;
import de.extio.lmdatasetprep.tools.DatasetTool;
@@ -77,9 +78,9 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
LOGGER.info("All file consumers are finished");
int cnt = 0;
while (cnt < 3) {
while (cnt < 5) {
Thread.sleep(1000);
if (!Execution.tasks.isEmpty() || Execution.work.values().stream().anyMatch(q -> q.values().stream().anyMatch(bq -> !bq.isEmpty()))) {
if (Execution.tasksRunning.get() != 0 || Execution.work.values().stream().anyMatch(q -> q.values().stream().anyMatch(bq -> !bq.isEmpty()))) {
cnt = 0;
}
else {
@@ -94,8 +95,8 @@ public class LmDatasetPreparerApplication implements CommandLineRunner, Applicat
}
LOGGER.info("Shutdown");
Execution.scheduler.shutdown();
Execution.executorService.shutdown();
((ConfigurableApplicationContext) applicationContext).close();
}
catch (final Exception e) {
LOGGER.error("Error executing the component", e);

14
src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java
View File

@@ -48,8 +48,12 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
@Override
public void afterPropertiesSet() throws Exception {
this.maleNames = new BufferedReader(new InputStreamReader(new ClassPathResource("male.txt").getInputStream())).lines().collect(Collectors.toList());
this.femaleNames = new BufferedReader(new InputStreamReader(new ClassPathResource("female.txt").getInputStream())).lines().collect(Collectors.toList());
try (var in = new ClassPathResource("male.txt").getInputStream()) {
this.maleNames = new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.toList());
}
try (var in = new ClassPathResource("female.txt").getInputStream()) {
this.femaleNames = new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.toList());
}
}
@Override
@@ -160,4 +164,10 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
protected record Names(List<String> males, List<String> females) {
}
protected record QaLine(String question, String answer) {
}
protected record HistInstrComplLine(String history, String instruct, String completion) {
}
}

417
src/main/java/de/extio/lmdatasetprep/tools/Create.java Normal file
View File

@@ -0,0 +1,417 @@
package de.extio.lmdatasetprep.tools;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution;
import de.extio.lmdatasetprep.TextUtils;
import de.extio.lmlib.agent.AccumulateTextAgentResponseHandler;
import de.extio.lmlib.agent.Agent;
import de.extio.lmlib.agent.AgentContext;
import de.extio.lmlib.agent.AgentExecutorService;
import de.extio.lmlib.agent.AgentNext;
import de.extio.lmlib.agent.AgentType;
import de.extio.lmlib.agent.JsonAgentResponseHandler;
import de.extio.lmlib.agent.TextAgentResponseHandler;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
@Component
public class Create implements DatasetTool {
private static final Logger LOGGER = LoggerFactory.getLogger(Create.class);
@Autowired
private AgentExecutorService agentExecutorService;
@Override
public String getModelCategoryPropertyName() {
return "create.category";
}
@Override
public void accept(final Properties properties) {
final var modelCategory = this.getModelCategory(properties);
final var agents = new HashMap<String, Agent>();
agents.put("PlotAnalyzer",
new Agent("PlotAnalyzer",
AgentType.COMPLETION,
modelCategory,
"Analyze key points in the plot of the following part of a story. Return the text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"keyPoints\": [\"keyPoint 1\", \"keyPoint 2\", ... ] }",
"[[paragraphs]]",
new JsonAgentResponseHandler(),
null,
null,
contexts -> Agent.mergeContexts(List.of("paragraphs", "keyPoints"), contexts),
context -> new AgentNext("ThemeSplitter", null)));
agents.put("ThemeSplitter",
new Agent("ThemeSplitter",
AgentType.PROCESSING_ONLY,
modelCategory,
null,
null,
null,
null,
null,
contexts -> {
final var quantity = Integer.parseInt(properties.getProperty("create.themes"));
contexts.getFirst().setStringValue("themesCnt", String.valueOf(quantity));
final var copies = new ArrayList<AgentContext>();
for (int i = 0; i < quantity; i++) {
final var copy = new AgentContext(contexts.getFirst());
copy.setStringValue("themesIdx", Integer.toString(i));
copies.add(copy);
}
if ("true".equalsIgnoreCase(properties.getProperty("create.debug.keepOriginal", "false"))) {
final var copy = new AgentContext(contexts.getFirst());
copy.getContext().put("keepOriginal", List.of(Boolean.TRUE));
copy.getContext().put("storyPoints", copy.getContext().get("keyPoints"));
copies.add(copy);
}
return copies;
},
context -> new AgentNext("ThemeAnalyzer", null)));
agents.put("ThemeAnalyzer",
new Agent("ThemeAnalyzer",
AgentType.COMPLETION,
modelCategory,
"Describe the overall theme of the following part of a story. Return the text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Provide no preamble and no explanation.",
"[[paragraphs]]",
new TextAgentResponseHandler("themesTxt"),
null,
null,
contexts -> Agent.mergeContexts(List.of("paragraphs", "themesTxt"), contexts),
context -> new AgentNext("ThemeSummarizer", null)));
agents.put("ThemeSummarizer",
new Agent("ThemeSummarizer",
AgentType.COMPLETION,
modelCategory,
"Analyze the properties of the following themes. Return the text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"themes\": [\"Property 1\", \"Property 2\", … ] }]",
"{{themesTxt}}",
new JsonAgentResponseHandler(),
null,
null,
null,
context -> new AgentNext("CharacterAnalyzer", null)));
agents.put("CharacterAnalyzer",
new Agent("CharacterAnalyzer",
AgentType.COMPLETION,
modelCategory,
"Analyze the character properties of the persons in the following part of a story. Return the text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"characters\": [\"Name: Gender and short description\", ... ] }",
"[[paragraphs]]",
new JsonAgentResponseHandler(),
null,
null,
contexts -> {
final var merged = Agent.mergeContexts(List.of("paragraphs", "characters"), contexts);
final var characters = merged.getFirst().getContext().get("characters");
if (characters != null) {
final Map<String, String> mergedCharacters = new HashMap<>();
for (final var character : characters) {
final int pos = character.toString().indexOf(':');
if (pos > -1) {
final String name = character.toString().substring(0, pos).trim();
final String description = character.toString().substring(pos + 1).trim();
mergedCharacters.compute(name, (k, v) -> (v == null) ? description : v.concat("; ").concat(description));
}
}
merged.getFirst().setStringValues("characterSummaries", mergedCharacters.entrySet().stream().filter(e -> !e.getValue().contains(";")).map(e -> e.getKey() + ": " + e.getValue()).toList());
merged.getFirst().setStringValues("mergedCharacters", mergedCharacters.entrySet().stream().filter(e -> e.getValue().contains(";")).map(e -> e.getKey() + ": " + e.getValue()).toList());
}
return merged;
},
context -> {
if (context.getContext().get("mergedCharacters").isEmpty()) {
return new AgentNext("BranchPreparer", null);
}
return new AgentNext("CharacterSummarizer", null);
}));
agents.put("CharacterSummarizer",
new Agent("CharacterSummarizer",
AgentType.COMPLETION,
modelCategory,
"Remove duplicate parts of the following character description. Only return the summary. Return the text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Provide no preamble and no explanation.",
"[[mergedCharacters]]",
new TextAgentResponseHandler("characterSummariesModelOutput"),
null,
context -> {
final var name = StringUtils.substringBefore(context.getStringValue("mergedCharacters"), ":");
final var summary = TextUtils.normalizeModelResponse(context.getStringValue("characterSummariesModelOutput"), true);
context.setStringValue("characterSummariesModelOutput", name + ": " + summary);
},
contexts -> {
final var merged = Agent.mergeContexts(List.of("mergedCharacters", "characterSummariesModelOutput"), contexts);
final var modelOutput = merged.getFirst().getStringValues("characterSummariesModelOutput");
modelOutput.addAll(merged.getFirst().getStringValues("characterSummaries"));
merged.getFirst().setStringValues("characterSummaries", modelOutput);
return merged;
},
context -> new AgentNext("BranchPreparer", null)));
agents.put("BranchPreparer",
new Agent("BranchPreparer",
AgentType.PROCESSING_ONLY,
modelCategory,
null,
null,
null,
null,
null,
contexts -> {
if (contexts.getFirst().getContext().containsKey("keepOriginal")) {
contexts.getFirst().setStringValue("branches", "1");
contexts.getFirst().setStringValue("branch", "0");
return contexts;
}
final var branches = new ArrayList<AgentContext>();
final var quantity = Integer.parseInt(properties.getProperty("create.stories"));
contexts.getFirst().setStringValue("branches", String.valueOf(quantity));
final var splits = new ArrayList<List<String>>();
final var keyPoints = contexts.getFirst().getStringValues("keyPoints");
final var length = keyPoints.stream().map(e -> e.length()).reduce(Integer.valueOf(0), Integer::sum);
final var limit = Integer.parseInt(properties.getProperty("create.plotLimit"));
if (length.intValue() > limit) {
final var cnt = (int) Math.ceil(length.doubleValue() / limit);
final var partitionSize = keyPoints.size() / cnt + 1;
for (int i = 0; i < cnt; i++) {
final var split = keyPoints.subList(Math.min(keyPoints.size(), i * partitionSize), Math.min(keyPoints.size(), (i + 1) * partitionSize));
if (!split.isEmpty()) {
splits.add(split);
}
}
}
else {
splits.add(keyPoints);
}
for (final var split : splits) {
for (int i = 0; i < quantity; i++) {
final var copy = new AgentContext(contexts.getFirst());
copy.setStringValue("branch", Integer.toString(i));
copy.setStringValues("keyPoints", split);
branches.add(copy);
}
}
return branches;
},
context -> {
if (context.getContext().containsKey("keepOriginal")) {
return new AgentNext("ParagraphWriterPreparer", null);
}
return new AgentNext("PlotMutator", null);
}));
agents.put("PlotMutator",
new Agent("PlotMutator",
AgentType.COMPLETION,
modelCategory,
"Write a new story line based on an existing plot. Return the story line, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Return the response in JSON format with the following field: { \"storyPoints\": [ \"Story Point 1\", … ] }",
"""
Characters: {{{characterSummaries}}}
Properties: {{{themes}}}
Existing plot: {{{keyPoints}}}
({{branch}}/{{branches}}) Write a new story line based on the template""",
new JsonAgentResponseHandler(),
null,
null,
null,
context -> new AgentNext("PlotEnhancer", null)));
agents.put("PlotEnhancer",
new Agent("PlotEnhancer",
AgentType.COMPLETION,
modelCategory,
"Improve the story line. Return the story line, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Return the response in JSON format with the following field: { \"storyPoints\": [ \"Story Point 1\", … ] }",
"""
Characters: {{{characterSummaries}}}
Properties: {{{themes}}}
Existing story line: {{{storyPoints}}}
({{branch}}/{{branches}}) Instruction: Improve the story for better consistency. Fill gaps in the plot, for example when the location suddenly changes.""",
new JsonAgentResponseHandler(),
null,
null,
null,
context -> {
if ("true".equalsIgnoreCase(properties.getProperty("create.debug.returnPlot", "false"))) {
return new AgentNext(null, null);
}
return new AgentNext("ParagraphWriterPreparer", null);
}));
agents.put("ParagraphWriterPreparer",
new Agent("ParagraphWriterPreparer",
AgentType.PROCESSING_ONLY,
modelCategory,
"%s. Return the text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation.",
null,
null,
context -> {
context.getContext().computeIfAbsent("paragraphIncr", k -> List.of(Integer.valueOf(0)));
},
null,
contexts -> {
final var context = contexts.getFirst();
final var index = (int) context.getContext().get("paragraphIncr").getFirst();
final var storyPoints = context.getStringValues("storyPoints");
context.setStringValue("storyPointInstruction", storyPoints.get(index));
String systemPrompt;
String writingInstruction;
if ("true".equalsIgnoreCase(properties.getProperty("create.conciseParagraphs", "false"))) {
systemPrompt = String.format(context.getAgents().get("ParagraphWriterPreparer").systemPrompt(), "Write a concise paragraph");
writingInstruction = "Write the following in a short and concise paragraph";
}
else {
systemPrompt = String.format(context.getAgents().get("ParagraphWriterPreparer").systemPrompt(), "Write a paragraph");
writingInstruction = "Write the following in a nicely written paragraph";
}
context.setStringValue("writingInstruction", writingInstruction);
var characterSummariesStr = "";
try {
characterSummariesStr = new ObjectMapper().writeValueAsString(context.getContext().get("characterSummaries"));
}
catch (final JsonProcessingException e) {
LOGGER.error("Cannot write character summaries", e);
}
Conversation conversation;
if (index > 1) {
final var prevStoryPoints = String.join(", ", storyPoints.subList(Math.max(0, index - 6), Math.max(0, index - 2)));
final var prevStoryPointInstruction0 = storyPoints.get(index - 2);
final var prevStoryPointInstruction1 = storyPoints.get(index - 1);
final var prompt0 = String.format("""
Characters: %s
Summary of the previous paragraphs: [ %s ]
%s: %s""",
characterSummariesStr,
prevStoryPoints,
writingInstruction,
prevStoryPointInstruction0);
conversation = Conversation.create(systemPrompt, prompt0);
conversation.addTurn(new Turn(TurnType.ASSISTANT, context.getContext().get("generatedStory").get(index - 2).toString()));
conversation.addTurn(new Turn(TurnType.USER, writingInstruction + ": " + prevStoryPointInstruction1));
conversation.addTurn(new Turn(TurnType.ASSISTANT, context.getContext().get("generatedStory").get(index - 1).toString()));
}
else if (index == 1) {
final var prompt0 = String.format("""
Characters: %s
%s: %s""",
characterSummariesStr,
writingInstruction,
storyPoints.get(0).toString());
conversation = Conversation.create(systemPrompt, prompt0);
conversation.addTurn(new Turn(TurnType.ASSISTANT, context.getStringValue("generatedStory")));
}
else {
final var prompt0 = String.format("Characters: %s", characterSummariesStr);
conversation = Conversation.create(systemPrompt, prompt0);
conversation.addTurn(new Turn(TurnType.ASSISTANT, ""));
}
context.setConversation(conversation);
return contexts;
},
context -> new AgentNext("ParagraphWriter", null)));
agents.put("ParagraphWriter",
new Agent("ParagraphWriter",
AgentType.CONVERSATION_WITH_SYSTEM_PROMPT,
modelCategory,
"",
"{{writingInstruction}}: {{storyPointInstruction}}",
new AccumulateTextAgentResponseHandler("generatedStory", completion -> TextUtils.normalizeModelResponse(completion, true)),
null,
null,
null,
context -> {
var index = (int) context.getContext().get("paragraphIncr").getFirst();
if (++index >= context.getContext().get("storyPoints").size()) {
return new AgentNext(null, null);
}
context.getContext().put("paragraphIncr", List.of(index));
return new AgentNext("ParagraphWriterPreparer", null);
}));
Execution.transform(properties.getProperty("create.source"),
packet -> {
return List.of(() -> {
LOGGER.info("Template: {}", packet.file().getFileName());
final var context = new AgentContext(agents);
context.setStringValues("paragraphs", TextUtils.splitParagraphs(packet.text(), 2000, 300, false));
final var resultContexts = this.agentExecutorService.walkGraph(agents.get("PlotAnalyzer"), context);
for (int i = 0; i < resultContexts.size(); i++) {
final var resultContext = resultContexts.get(i);
LOGGER.debug(resultContext.getGraph().toString());
if (!resultContext.isError()) {
final Path out = Execution.suffixFilename(packet.file().getFileName(),
"create",
properties.getProperty("create.model"),
String.valueOf(i));
Execution.streamOut(out, "create.destination", properties, fos -> {
try {
final byte[] story;
if ("true".equalsIgnoreCase(properties.getProperty("create.debug.returnPlot", "false"))) {
story = resultContext.getContext().get("storyPoints").stream().map(Objects::toString).collect(Collectors.joining("\n")).getBytes(StandardCharsets.UTF_8);
}
else {
story = resultContext.getContext().get("generatedStory")
.stream()
.map(Objects::toString)
.collect(Collectors.joining("\n\n"))
.getBytes(StandardCharsets.UTF_8);
}
fos.write(story);
}
catch (final IOException e) {
LOGGER.error("IO exception", e);
}
});
}
}
});
});
}
}

2
src/main/java/de/extio/lmdatasetprep/tools/DatasetTool.java
View File

@@ -10,7 +10,7 @@ import de.extio.lmlib.profile.ModelCategory;
public interface DatasetTool extends Consumer<Properties> {
default String getModelCategoryPropertyName() {
return null;
throw new IllegalStateException("Tool does not set a modelCategoryPropertyName");
}
default ModelCategory getModelCategory(final Properties properties) {

62
src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructions.java Normal file
View File

@@ -0,0 +1,62 @@
package de.extio.lmdatasetprep.tools;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution;
@Component
public class GenContextualInstructions extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(GenContextualInstructions.class);
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
for (int i = 0; i < variations; i++) {
final int fi = i;
tasks.add(() -> {
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final Path out = Execution.suffixFilename(p.file().getFileName(),
"contextualinstr",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi),
".jsonl");
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> {
for (int j = 1; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
final var mapping = this.createCharacterNameMapping(inScope, properties);
final var summary = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j - 1), mapping), "Rewrite the following story as a summary of the previous paragraph and write that it is the summary. Return the summary, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new QaLine(String.join(" ", summary, instruction), currentParagraph);
this.writeJsonLine(mapper, fos, qaLine);
}
});
});
}
}
return tasks;
};
}
}

24
src/main/java/de/extio/lmdatasetprep/tools/Text2Jsonl2WithContextualPrompts.java → src/main/java/de/extio/lmdatasetprep/tools/GenContextualPrompts.java
View File

@@ -14,9 +14,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution;
@Component
public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts {
public class GenContextualPrompts extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2WithContextualPrompts.class);
private static final Logger LOGGER = LoggerFactory.getLogger(GenContextualPrompts.class);
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
@@ -25,17 +25,19 @@ public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
for (int i = 0; i < variations; i++) {
final Path out = Execution.suffixFilename(p.file().getFileName(),
"contextualds",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(i),
".jsonl");
final int fi = i;
tasks.add(() -> {
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final Path out = Execution.suffixFilename(p.file().getFileName(),
"contextualds",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi),
".jsonl");
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> {
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
@@ -65,6 +67,4 @@ public class Text2Jsonl2WithContextualPrompts extends AbstractContextualPrompts
};
}
protected record QaLine(String question, String answer) {
}
}

27
src/main/java/de/extio/lmdatasetprep/tools/Text2Jsonl2HistoryInstructCompletion.java → src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java
View File

@@ -14,9 +14,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution;
@Component
public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts {
public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2HistoryInstructCompletion.class);
private static final Logger LOGGER = LoggerFactory.getLogger(GenHistoryInstructCompletion.class);
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
@@ -25,17 +25,19 @@ public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualProm
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
for (int i = 0; i < variations; i++) {
final Path out = Execution.suffixFilename(p.file().getFileName(),
"histinstrcompl",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(i),
".jsonl");
final int fi = i;
tasks.add(() -> {
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final Path out = Execution.suffixFilename(p.file().getFileName(),
"histinstrcompl",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi),
".jsonl");
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> {
for (int j = 1; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
@@ -43,7 +45,7 @@ public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualProm
final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
final var mapping = this.createCharacterNameMapping(inScope, properties);
final var instruct = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new QaLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
this.writeJsonLine(mapper, fos, qaLine);
}
@@ -55,7 +57,4 @@ public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualProm
return tasks;
};
}
protected record QaLine(String history, String instruct, String completion) {
}
}

9
src/main/java/de/extio/lmdatasetprep/tools/Rewrite.java
View File

@@ -61,8 +61,11 @@ public class Rewrite implements DatasetTool {
Execution.transform(properties.getProperty("rewrite.source"),
packet -> {
final List<Runnable> tasks = new ArrayList<>();
tasks.add(() -> this.rewriteFile(properties, packet, 0, "impr", IMPROVEMENT_PROMPT));
for (int i = 0; i < Integer.parseInt(properties.getProperty("rewrite.cnt")); i++) {
for (int i = 0; i < Integer.parseInt(properties.getProperty("rewrite.improve")); i++) {
final int fi = i;
tasks.add(() -> this.rewriteFile(properties, packet, fi, "impr", IMPROVEMENT_PROMPT));
}
for (int i = 0; i < Integer.parseInt(properties.getProperty("rewrite.enhance")); i++) {
final int fi = i;
tasks.add(() -> this.rewriteFile(properties, packet, fi, "enh", String.format(ENHANCE_PROMPT, ENHANCEMENTS.get(ThreadLocalRandom.current().nextInt(ENHANCEMENTS.size())))));
}
@@ -71,7 +74,7 @@ public class Rewrite implements DatasetTool {
}
private void rewriteFile(final Properties properties, final WorkPacket packet, final int i, final String identifier, final String prompt) {
final Path out = Execution.suffixFilename(packet.file(),
final Path out = Execution.suffixFilename(packet.file().getFileName(),
"rewr",
properties.getProperty("rewrite.model"),
identifier,

2
src/main/resources/application.properties
View File

@@ -3,7 +3,7 @@ spring.main.web-application-type=NONE
logging.level.de.extio=info
profile.model.s=llama3-cold
profile.model.m=mixtral-cold
profile.model.m=mistral-cold
profile.model.l=chatml-cold
profile.model.hot=llama3-hot
profile.model.cold=alpaca-cold

4
src/main/resources/mixtral-cold.properties → src/main/resources/mistral-cold.properties
View File

@@ -1,8 +1,8 @@
prompts=mixtral
prompts=mistralNemo
tokenEncoding=
maxTokens=2000
maxContextLength=4000
temperature=0.2
temperature=0.3
topP=1.0
maxContinuations=1
modelProvider=OAI_TEXT_COMPLETION

19
src/main/resources/tool.properties.template
View File

@@ -1,14 +1,17 @@
translate.category=SMALL
# Source and destination can be either a path...
translate.source=/home/xxx/folder
# ... or a queue. Queues are prefixed with "queue", after the prefix comes the queue name
translate.destination=queue0
rewrite.category=SMALL
rewrite.source=queue0
rewrite.destination=queue1
rewrite.cnt=2
rewrite.improve=1
rewrite.enhance=2
rewrite.model=l3dp
# .n indicates a sequence. Add as many keys as you need
# .n indicates that the key supports sequences. Add as many keys as you need
mergeJsonl.source.0=
mergeJsonl.destination=
@@ -17,6 +20,7 @@ text2Jsonl.destination=
text2Jsonl.chunkNorm=750
text2Jsonl.chunkVar=250
contextualPrompts.category=SMALL
contextualPrompts.source=queue1
contextualPrompts.destination=/home/xxx/outputdir
contextualPrompts.model=l3dp
@@ -25,3 +29,14 @@ contextualPrompts.chunkNorm.0=750
contextualPrompts.chunkVar.0=250
contextualPrompts.chunkNorm.1=1500
contextualPrompts.chunkVar.1=300
create.source=
create.destination=
create.category=SMALL
create.model=l3dp
create.plotLimit=6000
create.conciseParagraphs=false
create.themes=2
create.stories=2
create.debug.keepOriginal=false
create.debug.returnPlot=false