diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2HistoryInstructCompletion.java b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2HistoryInstructCompletion.java new file mode 100644 index 0000000..23deaea --- /dev/null +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2HistoryInstructCompletion.java @@ -0,0 +1,66 @@ +package de.extio.lmdatasetprep.preparer; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; + +@Component +public class Text2Jsonl2HistoryInstructCompletion extends AbstractContextualPrompts { + + private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl2HistoryInstructCompletion.class); + + @Override + protected CreateTasks createTasks(final String[] args, final List chunkConfigurations, final int variations) { + return p -> { + final ObjectMapper mapper = new ObjectMapper(); + final List tasks = new ArrayList<>(); + + for (final ChunkCfg chunkCfg : chunkConfigurations) { + final List paragraphs = this.fileToParagraphs(p, chunkCfg.chunksNorm(), chunkCfg.chunksVar()); + + for (int i = 0; i < variations; i++) { + final Path out = Utils.suffixFilename(Path.of(args[2]).resolve(p.getFileName()), + "histinstrcompl", + String.valueOf(chunkCfg.chunksNorm()), + String.valueOf(i), + ".jsonl"); + if (Files.exists(out)) { + LOGGER.info("Skipping {}", out); + continue; + } + + tasks.add(() -> { + try (var fos = Files.newOutputStream(out)) { + for (int j = 1; j < paragraphs.size(); j++) { + LOGGER.info("Paragraph " + (j + 1) + "/" + paragraphs.size()); + + final var inScope = String.join("\n", paragraphs.subList(j - 1, j)); + final var mapping = this.createCharacterNameMapping(inScope); + final var instruct = this.paragraphToContextualPrompt(this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); + final var qaLine = new QaLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping)); + + this.writeJsonLine(mapper, fos, qaLine); + } + } + catch (final IOException e1) { + LOGGER.error("IO exception", e1); + } + }); + } + } + + return tasks; + }; + } + + protected record QaLine(String history, String instruct, String completion) { + } +}