exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Misc improvements;

Browse Source
This commit is contained in:
extio-de
2024-12-29 12:05:15 +01:00
parent 4d320fa141
commit ad001dae84
5 changed files with 20 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/de/extio/lmdatasetprep/Execution.java
View File

@@ -161,7 +161,7 @@ public class Execution {
} }
public static void streamOut(final Path f, final String propertyPrefix, final Properties properties, final Consumer<OutputStream> consumer) { public static void streamOut(final Path f, final String propertyPrefix, final Properties properties, final Consumer<OutputStream> consumer) {
final var streamFuncs = new ArrayList<OutStream>(); final var streamFuncs = new ArrayList<StreamFunc>();
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
final var destination = properties.getProperty(propertyPrefix + "." + i); final var destination = properties.getProperty(propertyPrefix + "." + i);
if (destination != null) { if (destination != null) {
@@ -197,8 +197,8 @@ public class Execution {
} }
private static OutStream setupStreamOutToFile(final Path f, final String destination) { private static StreamFunc setupStreamOutToFile(final Path f, final String destination) {
return new OutStream(() -> { return new StreamFunc(() -> {
final var out = Path.of(destination).resolve(f); final var out = Path.of(destination).resolve(f);
if (Files.exists(out)) { if (Files.exists(out)) {
LOGGER.info("File already exists: {} ", out); LOGGER.info("File already exists: {} ", out);
@@ -226,8 +226,8 @@ public class Execution {
}); });
} }
private static OutStream setupStreamOutToQueue(final Path f, final String destination) { private static StreamFunc setupStreamOutToQueue(final Path f, final String destination) {
return new OutStream(() -> { return new StreamFunc(() -> {
return new ByteArrayOutputStream(); return new ByteArrayOutputStream();
}, },
(stream) -> { (stream) -> {
@@ -239,7 +239,7 @@ public class Execution {
public record WorkPacket(Path file, String text) { public record WorkPacket(Path file, String text) {
} }
private record OutStream(Supplier<OutputStream> open, Consumer<OutputStream> afterClose) { private record StreamFunc(Supplier<OutputStream> open, Consumer<OutputStream> afterClose) {
} }
} }

2
src/main/resources/application.properties
View File

@@ -4,7 +4,7 @@ logging.level.de.extio=info
profile.model.s=llama3-cold profile.model.s=llama3-cold
profile.model.m=mistral-cold profile.model.m=mistral-cold
profile.model.l=chatml-cold profile.model.l=gemma
profile.model.hot=llama3-hot profile.model.hot=llama3-hot
profile.model.cold=alpaca-cold profile.model.cold=alpaca-cold

11
src/main/resources/gemma.properties Executable file
View File

@@ -0,0 +1,11 @@
prompts=gemma2
tokenEncoding=
maxTokens=2000
maxContextLength=4000
temperature=0.5
topP=1.0
maxContinuations=1
modelProvider=OAI_TEXT_COMPLETION
modelName=
url=http://localhost:5001
apiKey=

2
src/main/resources/llama3-cold.properties
View File

@@ -2,7 +2,7 @@ prompts=llama3
tokenEncoding= tokenEncoding=
maxTokens=2000 maxTokens=2000
maxContextLength=4000 maxContextLength=4000
temperature=0.2 temperature=0.3
topP=1.0 topP=1.0
maxContinuations=1 maxContinuations=1
modelProvider=OAI_TEXT_COMPLETION modelProvider=OAI_TEXT_COMPLETION

2
src/main/resources/llama3-hot.properties
View File

@@ -2,7 +2,7 @@ prompts=llama3
tokenEncoding= tokenEncoding=
maxTokens=2000 maxTokens=2000
maxContextLength=4000 maxContextLength=4000
temperature=0.4 temperature=0.7
topP=1.0 topP=1.0
maxContinuations=1 maxContinuations=1
modelProvider=OAI_TEXT_COMPLETION modelProvider=OAI_TEXT_COMPLETION