diff --git a/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java b/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java
index 512bbeb..1b3636f 100755
--- a/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java
+++ b/src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java
@@ -10,10 +10,14 @@ public class MixtralPromptStrategy implements PromptStrategy {
final StringBuilder prompt = new StringBuilder();
prompt.append("[INST] ");
prompt.append(instruction);
- if (!question.isEmpty() && !text.isEmpty()) {
- prompt.append(" ");
+ if (!instruction.isEmpty() && !question.isEmpty()) {
+ prompt.append("\n");
}
prompt.append(question);
+ if ((!instruction.isEmpty() || !question.isEmpty()) && !text.isEmpty()) {
+ prompt.append("\n");
+ }
+ prompt.append(text);
prompt.append(" [/INST]");
return prompt;
}
@@ -32,6 +36,11 @@ public class MixtralPromptStrategy implements PromptStrategy {
prompt.append(" [/INST]");
}
+ @Override
+ public String removeEOT(final String prompt) {
+ return prompt.strip().replace("", "");
+ }
+
@Override
public String getPromptName() {
return "mixtral";
diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
index bd752bb..05abd88 100644
--- a/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
+++ b/src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
@@ -32,7 +32,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer {
@Override
public void accept(final String[] args) {
if (args.length < 6) {
- LOGGER.error("Arguments are missing. ( , ...)");
+ LOGGER.error("Arguments are missing. [ , ...]");
return;
}
final int variations = Integer.parseInt(args[3]);
@@ -57,7 +57,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer {
String.valueOf(i),
".jsonl");
if (Files.exists(out)) {
- LOGGER.info("Skipping " + out);
+ LOGGER.info("Skipping {}", out);
continue;
}
diff --git a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
index 77f371d..3d4d1ea 100644
--- a/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
+++ b/src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
@@ -19,9 +19,11 @@ import org.slf4j.LoggerFactory;
public class Utils {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
+
private static final List PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?");
- private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
+ private static final int EXECUTORS = 4;
static String normalizeText(final String text) {
String result = text;
@@ -92,7 +94,7 @@ public class Utils {
if (slidingWindows && pos > chunks_var) {
for (final String delimiter : PARAGRAPH_DELIMITERS) {
final int o = text.indexOf(delimiter, pos - chunks_var);
- if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) {
+ if (o > -1 && o < pos - 10) {
pos = o + delimiter.length();
break;
}
@@ -113,33 +115,31 @@ public class Utils {
}
static void transformDirectory(final String path, final Function> createTasks) {
- final int EXECUTORS = 4;
+ final AtomicBoolean finished = new AtomicBoolean(false);
- try (Stream stream = Files.list(Paths.get(path))) {
- final AtomicBoolean finished = new AtomicBoolean(false);
-
- final BlockingQueue tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
-
- final List threads = new ArrayList<>(EXECUTORS);
- for (int i = 0; i < EXECUTORS; i++) {
- final Thread executor = new Thread(() -> {
- while (!Thread.interrupted() && !finished.get()) {
- try {
- final Runnable task = tasks.poll(1, TimeUnit.SECONDS);
- if (task != null) {
- task.run();
- }
- }
- catch (final InterruptedException e) {
- break;
+ final BlockingQueue tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
+
+ final List threads = new ArrayList<>(EXECUTORS);
+ for (int i = 0; i < EXECUTORS; i++) {
+ final Thread executor = new Thread(() -> {
+ while (!Thread.interrupted() && !finished.get()) {
+ try {
+ final Runnable task = tasks.poll(1, TimeUnit.SECONDS);
+ if (task != null) {
+ task.run();
}
}
- });
- executor.setName("Executor " + i);
- executor.start();
- threads.add(executor);
- }
-
+ catch (final InterruptedException e) {
+ break;
+ }
+ }
+ });
+ executor.setName("Executor " + i);
+ executor.start();
+ threads.add(executor);
+ }
+
+ try (Stream stream = Files.list(Paths.get(path))) {
stream
.filter(p -> Files.isRegularFile(p))
.forEach(p -> {
@@ -152,18 +152,28 @@ public class Utils {
}
}
});
-
- while (!tasks.isEmpty()) {
+ }
+ catch (final IOException e) {
+ LOGGER.error("Error", e);
+ }
+
+ while (!tasks.isEmpty()) {
+ try {
Thread.sleep(1);
}
- finished.set(true);
-
- for (int i = 0; i < EXECUTORS; i++) {
- threads.get(i).join();
+ catch (final InterruptedException e) {
+ LOGGER.error("Interrupted", e);
}
}
- catch (final IOException | InterruptedException e) {
- LOGGER.error("Error", e);
+ finished.set(true);
+
+ for (int i = 0; i < EXECUTORS; i++) {
+ try {
+ threads.get(i).join();
+ }
+ catch (final InterruptedException e) {
+ LOGGER.error("Interrupted", e);
+ }
}
}