exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AbstractContextualPrompts minor refactoring;

Browse Source
This commit is contained in:
extio-de
2024-12-30 11:58:38 +01:00
parent ad001dae84
commit c76b108c19
4 changed files with 8 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java
View File

@@ -37,6 +37,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
@Autowired @Autowired
protected ClientService clientService; protected ClientService clientService;
protected final ObjectMapper mapper = new ObjectMapper();
private List<String> maleNames = new ArrayList<>(); private List<String> maleNames = new ArrayList<>();
private List<String> femaleNames = new ArrayList<>(); private List<String> femaleNames = new ArrayList<>();
@@ -103,9 +105,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
final var json = response.substring(start, end + 1); final var json = response.substring(start, end + 1);
Names names; Names names;
final ObjectMapper mapper = new ObjectMapper();
try { try {
names = mapper.readValue(json, Names.class); names = this.mapper.readValue(json, Names.class);
} }
catch (final JsonProcessingException e) { catch (final JsonProcessingException e) {
return Map.of(); return Map.of();
@@ -146,9 +147,9 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
return TextUtils.normalizeModelResponse(completion.response(), true); return TextUtils.normalizeModelResponse(completion.response(), true);
} }
protected void writeJsonLine(final ObjectMapper mapper, final OutputStream fos, final Object line) { protected void writeJsonLine(final OutputStream fos, final Object line) {
try { try {
final var jsonb = mapper final var jsonb = this.mapper
.writeValueAsString(line) .writeValueAsString(line)
.getBytes(StandardCharsets.UTF_8); .getBytes(StandardCharsets.UTF_8);
fos.write(jsonb); fos.write(jsonb);

5
src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructions.java
View File

@@ -9,8 +9,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution; import de.extio.lmdatasetprep.Execution;
@Component @Component
@@ -21,7 +19,6 @@ public class GenContextualInstructions extends AbstractContextualPrompts {
@Override @Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) { protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> { return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>(); final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) { for (final ChunkCfg chunkCfg : chunkConfigurations) {
@@ -49,7 +46,7 @@ public class GenContextualInstructions extends AbstractContextualPrompts {
final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new QaLine(String.join(" ", summary, instruction), currentParagraph); final var qaLine = new QaLine(String.join(" ", summary, instruction), currentParagraph);
this.writeJsonLine(mapper, fos, qaLine); this.writeJsonLine(fos, qaLine);
} }
}); });
}); });

5
src/main/java/de/extio/lmdatasetprep/tools/GenContextualPrompts.java
View File

@@ -9,8 +9,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution; import de.extio.lmdatasetprep.Execution;
@Component @Component
@@ -21,7 +19,6 @@ public class GenContextualPrompts extends AbstractContextualPrompts {
@Override @Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) { protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> { return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>(); final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) { for (final ChunkCfg chunkCfg : chunkConfigurations) {
@@ -56,7 +53,7 @@ public class GenContextualPrompts extends AbstractContextualPrompts {
qaLine = new QaLine(userPrompt, paragraph); qaLine = new QaLine(userPrompt, paragraph);
} }
this.writeJsonLine(mapper, fos, qaLine); this.writeJsonLine(fos, qaLine);
} }
}); });
}); });

5
src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java
View File

@@ -9,8 +9,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution; import de.extio.lmdatasetprep.Execution;
@Component @Component
@@ -21,7 +19,6 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
@Override @Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) { protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> { return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>(); final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) { for (final ChunkCfg chunkCfg : chunkConfigurations) {
@@ -47,7 +44,7 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
final var instruct = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:"); final var instruct = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping)); final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
this.writeJsonLine(mapper, fos, qaLine); this.writeJsonLine(fos, qaLine);
} }
}); });
}); });