exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AbstractContextualPrompts minor refactoring;

Browse Source
This commit is contained in:
extio-de
2024-12-30 11:58:38 +01:00
parent ad001dae84
commit c76b108c19
4 changed files with 8 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/de/extio/lmdatasetprep/tools/AbstractContextualPrompts.java
View File

@@ -37,6 +37,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
@Autowired
protected ClientService clientService;
protected final ObjectMapper mapper = new ObjectMapper();
private List<String> maleNames = new ArrayList<>();
private List<String> femaleNames = new ArrayList<>();
@@ -103,9 +105,8 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
final var json = response.substring(start, end + 1);
Names names;
final ObjectMapper mapper = new ObjectMapper();
try {
names = mapper.readValue(json, Names.class);
names = this.mapper.readValue(json, Names.class);
}
catch (final JsonProcessingException e) {
return Map.of();
@@ -146,9 +147,9 @@ abstract class AbstractContextualPrompts implements InitializingBean, DatasetToo
return TextUtils.normalizeModelResponse(completion.response(), true);
}
protected void writeJsonLine(final ObjectMapper mapper, final OutputStream fos, final Object line) {
protected void writeJsonLine(final OutputStream fos, final Object line) {
try {
final var jsonb = mapper
final var jsonb = this.mapper
.writeValueAsString(line)
.getBytes(StandardCharsets.UTF_8);
fos.write(jsonb);

5
src/main/java/de/extio/lmdatasetprep/tools/GenContextualInstructions.java
View File

@@ -9,8 +9,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution;
@Component
@@ -21,7 +19,6 @@ public class GenContextualInstructions extends AbstractContextualPrompts {
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
@@ -49,7 +46,7 @@ public class GenContextualInstructions extends AbstractContextualPrompts {
final var instruction = this.createPrompt(properties, currentParagraph, "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new QaLine(String.join(" ", summary, instruction), currentParagraph);
this.writeJsonLine(mapper, fos, qaLine);
this.writeJsonLine(fos, qaLine);
}
});
});

5
src/main/java/de/extio/lmdatasetprep/tools/GenContextualPrompts.java
View File

@@ -9,8 +9,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution;
@Component
@@ -21,7 +19,6 @@ public class GenContextualPrompts extends AbstractContextualPrompts {
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
@@ -56,7 +53,7 @@ public class GenContextualPrompts extends AbstractContextualPrompts {
qaLine = new QaLine(userPrompt, paragraph);
}
this.writeJsonLine(mapper, fos, qaLine);
this.writeJsonLine(fos, qaLine);
}
});
});

5
src/main/java/de/extio/lmdatasetprep/tools/GenHistoryInstructCompletion.java
View File

@@ -9,8 +9,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmdatasetprep.Execution;
@Component
@@ -21,7 +19,6 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
@Override
protected CreateTasks createTasks(final Properties properties, final List<ChunkCfg> chunkConfigurations, final int variations) {
return p -> {
final ObjectMapper mapper = new ObjectMapper();
final List<Runnable> tasks = new ArrayList<>();
for (final ChunkCfg chunkCfg : chunkConfigurations) {
@@ -47,7 +44,7 @@ public class GenHistoryInstructCompletion extends AbstractContextualPrompts {
final var instruct = this.createPrompt(properties, this.renameCharacters(paragraphs.get(j), mapping), "Rewrite the following story as a concise instruction in a single paragraph. Return the instruction, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't return an explanation:");
final var qaLine = new HistInstrComplLine(this.renameCharacters(paragraphs.get(j - 1), mapping), instruct, this.renameCharacters(paragraphs.get(j), mapping));
this.writeJsonLine(mapper, fos, qaLine);
this.writeJsonLine(fos, qaLine);
}
});
});