From 509f36238dbf41ca8e76c3b834cf42d4621a2809 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 2 Nov 2024 12:24:28 +0100 Subject: [PATCH] Fix MixtralPromptStrategy; Refactoring; --- .../client/prompt/MixtralPromptStrategy.java | 13 +++- .../Text2Jsonl2WithContextualPrompts.java | 4 +- .../extio/lmdatasetprep/preparer/Utils.java | 78 +++++++++++-------- 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java b/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java index 512bbeb..1b3636f 100755 --- a/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java +++ b/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java @@ -10,10 +10,14 @@ public class MixtralPromptStrategy implements PromptStrategy { final StringBuilder prompt = new StringBuilder(); prompt.append("[INST] "); prompt.append(instruction); - if (!question.isEmpty() && !text.isEmpty()) { - prompt.append(" "); + if (!instruction.isEmpty() && !question.isEmpty()) { + prompt.append("\n"); } prompt.append(question); + if ((!instruction.isEmpty() || !question.isEmpty()) && !text.isEmpty()) { + prompt.append("\n"); + } + prompt.append(text); prompt.append(" [/INST]"); return prompt; } @@ -32,6 +36,11 @@ public class MixtralPromptStrategy implements PromptStrategy { prompt.append(" [/INST]"); } + @Override + public String removeEOT(final String prompt) { + return prompt.strip().replace("", ""); + } + @Override public String getPromptName() { return "mixtral"; diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java index bd752bb..05abd88 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java @@ -32,7 +32,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { @Override public void accept(final String[] args) { if (args.length < 6) { - LOGGER.error("Arguments are missing. ( , ...)"); + LOGGER.error("Arguments are missing. [ , ...]"); return; } final int variations = Integer.parseInt(args[3]); @@ -57,7 +57,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer { String.valueOf(i), ".jsonl"); if (Files.exists(out)) { - LOGGER.info("Skipping " + out); + LOGGER.info("Skipping {}", out); continue; } diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java index 77f371d..3d4d1ea 100644 --- a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java +++ b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java @@ -19,9 +19,11 @@ import org.slf4j.LoggerFactory; public class Utils { + private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); + private static final List PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?"); - private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); + private static final int EXECUTORS = 4; static String normalizeText(final String text) { String result = text; @@ -92,7 +94,7 @@ public class Utils { if (slidingWindows && pos > chunks_var) { for (final String delimiter : PARAGRAPH_DELIMITERS) { final int o = text.indexOf(delimiter, pos - chunks_var); - if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) { + if (o > -1 && o < pos - 10) { pos = o + delimiter.length(); break; } @@ -113,33 +115,31 @@ public class Utils { } static void transformDirectory(final String path, final Function> createTasks) { - final int EXECUTORS = 4; + final AtomicBoolean finished = new AtomicBoolean(false); - try (Stream stream = Files.list(Paths.get(path))) { - final AtomicBoolean finished = new AtomicBoolean(false); - - final BlockingQueue tasks = new LinkedBlockingQueue<>(EXECUTORS * 2); - - final List threads = new ArrayList<>(EXECUTORS); - for (int i = 0; i < EXECUTORS; i++) { - final Thread executor = new Thread(() -> { - while (!Thread.interrupted() && !finished.get()) { - try { - final Runnable task = tasks.poll(1, TimeUnit.SECONDS); - if (task != null) { - task.run(); - } - } - catch (final InterruptedException e) { - break; + final BlockingQueue tasks = new LinkedBlockingQueue<>(EXECUTORS * 2); + + final List threads = new ArrayList<>(EXECUTORS); + for (int i = 0; i < EXECUTORS; i++) { + final Thread executor = new Thread(() -> { + while (!Thread.interrupted() && !finished.get()) { + try { + final Runnable task = tasks.poll(1, TimeUnit.SECONDS); + if (task != null) { + task.run(); } } - }); - executor.setName("Executor " + i); - executor.start(); - threads.add(executor); - } - + catch (final InterruptedException e) { + break; + } + } + }); + executor.setName("Executor " + i); + executor.start(); + threads.add(executor); + } + + try (Stream stream = Files.list(Paths.get(path))) { stream .filter(p -> Files.isRegularFile(p)) .forEach(p -> { @@ -152,18 +152,28 @@ public class Utils { } } }); - - while (!tasks.isEmpty()) { + } + catch (final IOException e) { + LOGGER.error("Error", e); + } + + while (!tasks.isEmpty()) { + try { Thread.sleep(1); } - finished.set(true); - - for (int i = 0; i < EXECUTORS; i++) { - threads.get(i).join(); + catch (final InterruptedException e) { + LOGGER.error("Interrupted", e); } } - catch (final IOException | InterruptedException e) { - LOGGER.error("Error", e); + finished.set(true); + + for (int i = 0; i < EXECUTORS; i++) { + try { + threads.get(i).join(); + } + catch (final InterruptedException e) { + LOGGER.error("Interrupted", e); + } } }