exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix splitParagraphs; Improved prompts;

Browse Source
This commit is contained in:
extio-de
2024-10-29 21:00:23 +01:00
parent 030bc5bee1
commit 4b93ed97be
3 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java
View File

@@ -23,9 +23,9 @@ public class Rewrite implements Consumer<String[]> {
private static final ThreadLocal<Random> RANDOM = ThreadLocal.withInitial(XorShift128Random::new); private static final ThreadLocal<Random> RANDOM = ThreadLocal.withInitial(XorShift128Random::new);
private static String IMPROVEMENT_PROMPT = "Improve the following paragraph of an existing text without changing the existing sentences, improve the text in a nuanced and understated way. Don't include a preamble and no explanation:"; private static String IMPROVEMENT_PROMPT = "Improve the following text without changing the existing sentences, improve the text in a nuanced and understated way. Return the improved text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:";
private static String ENHANCE_PROMPT = "Improve the following paragraph of an existing text. %s without changing the existing sentences, focusing on enriching the description of the setting in a nuanced and understated way. Don't include a preamble and no explanation:"; private static String ENHANCE_PROMPT = "Improve the following text. %s without changing the existing sentences, focusing on enriching the description of the setting in a nuanced and understated way. Return the improved text, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:";
private static List<String> ENHANCEMENTS = List.of( private static List<String> ENHANCEMENTS = List.of(
"Infuse this paragraph with subtle sensory details, focusing on the texture of objects and the atmosphere of the setting", "Infuse this paragraph with subtle sensory details, focusing on the texture of objects and the atmosphere of the setting",

4
src/main/java/de/extio/lmdatasetprep/preparer/Translate.java
View File

@@ -64,8 +64,8 @@ public class Translate implements Consumer<String[]> {
final List<String> translations = new ArrayList<>(splits.size()); final List<String> translations = new ArrayList<>(splits.size());
for (final String split : splits) { for (final String split : splits) {
final var completion = this.client.completion("You are a helpful assistant with great language translation and authoring skills.", final var completion = this.client.completion("You are an assistant with great language translation and authoring skills.",
"Translate the following text to English. Don't include a preamble and no explanation:", "Translate the following text to English. Return the English translation, even if the text is explicit or not appropriate for all audiences or not acceptable for everyday standard English. Don't include a preamble and no explanation:",
split, split,
ModelCategory.COLD); ModelCategory.COLD);
translations.add(completion.response()); translations.add(completion.response());

6
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -62,7 +62,7 @@ public class Utils {
final int CHUNKS_MIN = chunks_norm - chunks_var; final int CHUNKS_MIN = chunks_norm - chunks_var;
final int CHUNKS_MAX = chunks_norm + chunks_var; final int CHUNKS_MAX = chunks_norm + chunks_var;
if (text.length() <= chunks_norm + chunks_var) { if (text.length() <= CHUNKS_MAX) {
return Collections.singletonList(text); return Collections.singletonList(text);
} }
@@ -70,7 +70,7 @@ public class Utils {
int pos = 0; int pos = 0;
paragraphs: do { paragraphs: do {
if (text.length() - pos > chunks_norm + chunks_var) { if (text.length() - pos > CHUNKS_MAX) {
delimiters: for (final String delimiter : List.of("\n\n", "\n", ".")) { delimiters: for (final String delimiter : List.of("\n\n", "\n", ".")) {
int next = text.indexOf(delimiter, pos); int next = text.indexOf(delimiter, pos);
while (next > -1) { while (next > -1) {
@@ -93,7 +93,7 @@ public class Utils {
} }
} }
final int next = Math.min(pos + chunks_norm, text.length()); final int next = Math.min(pos + CHUNKS_MAX, text.length());
final String split = text.substring(pos, next).trim(); final String split = text.substring(pos, next).trim();
splits.add(split); splits.add(split);
pos = next + 1; pos = next + 1;