exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reworked instruction generation tools;

Browse Source
This commit is contained in:
extio-de
2025-01-27 20:17:32 +01:00
parent 8be692c237
commit 6e7db0c6d8
4 changed files with 96 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java
View File

@@ -20,6 +20,7 @@ import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -58,6 +59,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
try (var in = new ClassPathResource("female.txt").getInputStream()) {
this.femaleNames = new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.toList());
}
this.mapper.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
}
@Override
@@ -146,24 +149,21 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
instruction,
paragraph);
return processResponse(properties, completion);
return this.processResponse(properties, completion);
}
protected String requestCompletion(final Properties properties, final Conversation conversation) {
final var completion = this.getClient(properties, this.clientService).conversation(this.getModelCategory(properties), conversation);
return processResponse(properties, completion);
return this.processResponse(properties, completion);
}
private String processResponse(final Properties properties, final Completion completion) {
return TextUtils.normalizeModelResponse(completion.response(), Boolean.parseBoolean(properties.getProperty("contextualPrompts.removePreamble", "false")));
}
protected void writeJsonLine(final OutputStream fos, final Object line) {
try {
final var jsonb = this.mapper
.writeValueAsString(line)
.getBytes(StandardCharsets.UTF_8);
fos.write(jsonb);
this.mapper.writeValue(fos, line);
fos.write(NEWLINE);
}
catch (final IOException e) {
@@ -182,11 +182,11 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
protected record HistInstrComplLine(String history, String instruct, String completion) {
}
protected record ConversationLine(String user, String assistant) {
}
protected record ConversationsLine(List<ConversationLine> conversation) {
}
}

12
src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java → src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructionsSplitHistory.java
View File

@@ -12,9 +12,9 @@ import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.Execution;
@Component
public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
public class GenContextualInstructionsSplitHistory extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(GenHistoryInstructCompletion.class);
private static final Logger LOGGER = LoggerFactory.getLogger(GenContextualInstructionsSplitHistory.class);
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
@@ -29,7 +29,7 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final Path out = Execution.suffixFilename(p.file().getFileName(),
"histinstrcompl",
"contextualinstrhist",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi),
@@ -41,8 +41,10 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
final var inScope = String.join("\n", paragraphs.subList(j - 1, j));
final var mapping = this.createCharacterNameMapping(inScope, properties);
final var instruct = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
final var summary = this.requestCompletion(properties, this.renameCharacters(paragraphs.get(j - 1), mapping), "Rewrite the following story as a summary of the previous paragraph and write that it is the summary. Return the summary, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
final var instruction = this.requestCompletion(properties, currentParagraph, "Summarize the following text as a single concise statement. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new QaLine("History: " + summary + "\nInstruction: " + instruction, currentParagraph);
this.writeJsonLine(fos, qaLine);
}

76
src/main/java/de/extio/lmdatasetprep/tools/GenConversation.java
View File

@@ -1,76 +0,0 @@
package de.extio.lmdatasetprep.tools;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.Execution;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
@Component
public class GenConversation extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(GenConversation.class);
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
for (int i = 0; i < variations; i++) {
final int fi = i;
tasks.add(() -> {
final var conversation = new ArrayList<ConversationLine>();
{
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final var mapping = new HashMap<String, String>();
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size());
mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties));
}
String previousParagraph = null;
String currentParagraph = null;
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
previousParagraph = currentParagraph;
currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
final var c = Conversation.create("Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation.", previousParagraph != null ? previousParagraph : currentParagraph);
if (previousParagraph != null) {
c.addTurn(new Turn(TurnType.ASSISTANT, conversation.getLast().user()));
c.addTurn(new Turn(TurnType.USER, currentParagraph));
}
final var instruction = this.requestCompletion(properties, c);
conversation.add(new ConversationLine(instruction, currentParagraph));
}
}
final Path out = Execution.suffixFilename(p.file().getFileName(),
"conversation",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi),
".jsonl");
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> this.writeJsonLine(fos, new ConversationsLine(conversation)));
});
}
}
return tasks;
};
}
}

79
src/main/java/de/extio/lmdatasetprep/tools/GenInstructionalConversation.java Normal file
View File

@@ -0,0 +1,79 @@
package de.extio.lmdatasetprep.tools;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.Execution;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
@Component
public class GenInstructionalConversation extends AbstractContextualPrompts {
private static final Logger LOGGER = LoggerFactory.getLogger(GenInstructionalConversation.class);
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
for (int i = 0; i < variations; i++) {
final int fi = i;
tasks.add(() -> {
final Path out = Execution.suffixFilename(p.file().getFileName(),
"instrconversation",
properties.getProperty("contextualPrompts.model"),
String.valueOf(chunkCfg.chunksNorm()),
String.valueOf(fi),
".jsonl");
Execution.streamOut(out, "contextualPrompts.destination", properties, fos -> {
final var conversation = new ArrayList<ConversationLine>();
{
final List<String> paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar());
final var mapping = new HashMap<String, String>();
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Name Mapping " + (j + 1) + "/" + paragraphs.size());
mapping.putAll(this.createCharacterNameMapping(paragraphs.get(j), properties));
}
String previousParagraph = null;
String currentParagraph = null;
for (int j = 0; j < paragraphs.size(); j++) {
LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size());
previousParagraph = currentParagraph;
currentParagraph = this.renameCharacters(paragraphs.get(j), mapping);
final var c = Conversation.create("Summarize the following text as a single concise statement. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation.", previousParagraph != null ? previousParagraph : currentParagraph);
if (previousParagraph != null) {
c.addTurn(new Turn(TurnType.ASSISTANT, conversation.getLast().user()));
c.addTurn(new Turn(TurnType.USER, currentParagraph));
}
final var instruction = this.requestCompletion(properties, c);
conversation.add(new ConversationLine(instruction, currentParagraph));
}
}
this.writeJsonLine(fos, new ConversationsLine(conversation));
});
});
}
}
return tasks;
};
}
}