exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix paragraphToContextualPrompt response post-processing;

Browse Source
This commit is contained in:
extio-de
2024-11-03 22:53:35 +01:00
parent cb8ac57580
commit 82f969d666
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl2WithContextualPrompts.java
View File

@@ -102,9 +102,13 @@ public class Text2Jsonl2WithContextualPrompts implements Consumer<String[]> {
String prompt = completion.response(); String prompt = completion.response();
final int colon = prompt.indexOf(':'); final int colon = prompt.indexOf(':');
if (colon > -1 && colon > prompt.length() + 1) { if (colon > -1 && colon < prompt.length() + 1) {
prompt = prompt.substring(colon + 1); prompt = prompt.substring(colon + 1);
} }
final int preamble = prompt.indexOf("\n\n");
if (preamble > -1 && preamble < prompt.length() + 2) {
prompt = prompt.substring(preamble + 2);
}
prompt = prompt.trim(); prompt = prompt.trim();
final var qaLine = new QaLine(prompt, paragraph); final var qaLine = new QaLine(prompt, paragraph);