exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix MixtralPromptStrategy; Refactoring;

Browse Source
This commit is contained in:
extio-de
2024-11-02 12:24:28 +01:00
parent 1813ab8928
commit 509f36238d
3 changed files with 57 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java
View File

@@ -10,10 +10,14 @@ public class MixtralPromptStrategy implements PromptStrategy {
final StringBuilder prompt = new StringBuilder(); final StringBuilder prompt = new StringBuilder();
prompt.append("<s>[INST] "); prompt.append("<s>[INST] ");
prompt.append(instruction); prompt.append(instruction);
if (!question.isEmpty() && !text.isEmpty()) { if (!instruction.isEmpty() && !question.isEmpty()) {
prompt.append(" "); prompt.append("\n");
} }
prompt.append(question); prompt.append(question);
if ((!instruction.isEmpty() || !question.isEmpty()) && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append(" [/INST]"); prompt.append(" [/INST]");
return prompt; return prompt;
} }
@@ -32,6 +36,11 @@ public class MixtralPromptStrategy implements PromptStrategy {
prompt.append(" [/INST]"); prompt.append(" [/INST]");
} }
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("</s>", "");
}
@Override @Override
public String getPromptName() { public String getPromptName() {
return "mixtral"; return "mixtral";

4
src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
View File

@@ -32,7 +32,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]> {
@Override @Override
public void accept(final String[] args) { public void accept(final String[] args) {
if (args.length < 6) { if (args.length < 6) {
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> (<chunk norm> <chunk var>, ...)"); LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> [<chunk norm> <chunk var>, ...]");
return; return;
} }
final int variations = Integer.parseInt(args[3]); final int variations = Integer.parseInt(args[3]);
@@ -57,7 +57,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]> {
String.valueOf(i), String.valueOf(i),
".jsonl"); ".jsonl");
if (Files.exists(out)) { if (Files.exists(out)) {
LOGGER.info("Skipping " + out); LOGGER.info("Skipping {}", out);
continue; continue;
} }

24
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -19,9 +19,11 @@ import org.slf4j.LoggerFactory;
public class Utils { public class Utils {
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
private static final List<String> PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?"); private static final List<String> PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?");
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); private static final int EXECUTORS = 4;
static String normalizeText(final String text) { static String normalizeText(final String text) {
String result = text; String result = text;
@@ -92,7 +94,7 @@ public class Utils {
if (slidingWindows && pos > chunks_var) { if (slidingWindows && pos > chunks_var) {
for (final String delimiter : PARAGRAPH_DELIMITERS) { for (final String delimiter : PARAGRAPH_DELIMITERS) {
final int o = text.indexOf(delimiter, pos - chunks_var); final int o = text.indexOf(delimiter, pos - chunks_var);
if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) { if (o > -1 && o < pos - 10) {
pos = o + delimiter.length(); pos = o + delimiter.length();
break; break;
} }
@@ -113,9 +115,6 @@ public class Utils {
} }
static void transformDirectory(final String path, final Function<Path, List<Runnable>> createTasks) { static void transformDirectory(final String path, final Function<Path, List<Runnable>> createTasks) {
final int EXECUTORS = 4;
try (Stream<Path> stream = Files.list(Paths.get(path))) {
final AtomicBoolean finished = new AtomicBoolean(false); final AtomicBoolean finished = new AtomicBoolean(false);
final BlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>(EXECUTORS * 2); final BlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
@@ -140,6 +139,7 @@ public class Utils {
threads.add(executor); threads.add(executor);
} }
try (Stream<Path> stream = Files.list(Paths.get(path))) {
stream stream
.filter(p -> Files.isRegularFile(p)) .filter(p -> Files.isRegularFile(p))
.forEach(p -> { .forEach(p -> {
@@ -152,18 +152,28 @@ public class Utils {
} }
} }
}); });
}
catch (final IOException e) {
LOGGER.error("Error", e);
}
while (!tasks.isEmpty()) { while (!tasks.isEmpty()) {
try {
Thread.sleep(1); Thread.sleep(1);
} }
catch (final InterruptedException e) {
LOGGER.error("Interrupted", e);
}
}
finished.set(true); finished.set(true);
for (int i = 0; i < EXECUTORS; i++) { for (int i = 0; i < EXECUTORS; i++) {
try {
threads.get(i).join(); threads.get(i).join();
} }
catch (final InterruptedException e) {
LOGGER.error("Interrupted", e);
} }
catch (final IOException | InterruptedException e) {
LOGGER.error("Error", e);
} }
} }