exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix MixtralPromptStrategy; Refactoring;

Browse Source
This commit is contained in:
extio-de
2024-11-02 12:24:28 +01:00
parent 1813ab8928
commit 509f36238d
3 changed files with 57 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
View File

@@ -32,7 +32,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]> {
@Override
public void accept(final String[] args) {
if (args.length < 6) {
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> (<chunk norm> <chunk var>, ...)");
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <variations> [<chunk norm> <chunk var>, ...]");
return;
}
final int variations = Integer.parseInt(args[3]);
@@ -57,7 +57,7 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]> {
String.valueOf(i),
".jsonl");
if (Files.exists(out)) {
LOGGER.info("Skipping " + out);
LOGGER.info("Skipping {}", out);
continue;
}

78
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java
View File

@@ -19,9 +19,11 @@ import org.slf4j.LoggerFactory;
public class Utils {
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
private static final List<String> PARAGRAPH_DELIMITERS = List.of("\n\n", "\n", ".", "!", "?");
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
private static final int EXECUTORS = 4;
static String normalizeText(final String text) {
String result = text;
@@ -92,7 +94,7 @@ public class Utils {
if (slidingWindows && pos > chunks_var) {
for (final String delimiter : PARAGRAPH_DELIMITERS) {
final int o = text.indexOf(delimiter, pos - chunks_var);
if (o > -1 && o < pos - (PARAGRAPH_DELIMITERS.size() * 2)) {
if (o > -1 && o < pos - 10) {
pos = o + delimiter.length();
break;
}
@@ -113,33 +115,31 @@ public class Utils {
}
static void transformDirectory(final String path, final Function<Path, List<Runnable>> createTasks) {
final int EXECUTORS = 4;
final AtomicBoolean finished = new AtomicBoolean(false);
try (Stream<Path> stream = Files.list(Paths.get(path))) {
final AtomicBoolean finished = new AtomicBoolean(false);
final BlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
final List<Thread> threads = new ArrayList<>(EXECUTORS);
for (int i = 0; i < EXECUTORS; i++) {
final Thread executor = new Thread(() -> {
while (!Thread.interrupted() && !finished.get()) {
try {
final Runnable task = tasks.poll(1, TimeUnit.SECONDS);
if (task != null) {
task.run();
}
}
catch (final InterruptedException e) {
break;
final BlockingQueue<Runnable> tasks = new LinkedBlockingQueue<>(EXECUTORS * 2);
final List<Thread> threads = new ArrayList<>(EXECUTORS);
for (int i = 0; i < EXECUTORS; i++) {
final Thread executor = new Thread(() -> {
while (!Thread.interrupted() && !finished.get()) {
try {
final Runnable task = tasks.poll(1, TimeUnit.SECONDS);
if (task != null) {
task.run();
}
}
});
executor.setName("Executor " + i);
executor.start();
threads.add(executor);
}
catch (final InterruptedException e) {
break;
}
}
});
executor.setName("Executor " + i);
executor.start();
threads.add(executor);
}
try (Stream<Path> stream = Files.list(Paths.get(path))) {
stream
.filter(p -> Files.isRegularFile(p))
.forEach(p -> {
@@ -152,18 +152,28 @@ public class Utils {
}
}
});
while (!tasks.isEmpty()) {
}
catch (final IOException e) {
LOGGER.error("Error", e);
}
while (!tasks.isEmpty()) {
try {
Thread.sleep(1);
}
finished.set(true);
for (int i = 0; i < EXECUTORS; i++) {
threads.get(i).join();
catch (final InterruptedException e) {
LOGGER.error("Interrupted", e);
}
}
catch (final IOException | InterruptedException e) {
LOGGER.error("Error", e);
finished.set(true);
for (int i = 0; i < EXECUTORS; i++) {
try {
threads.get(i).join();
}
catch (final InterruptedException e) {
LOGGER.error("Interrupted", e);
}
}
}