exo/lm-dataset-preparer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
extio-de
2024-10-20 21:04:32 +02:00
commit 59dc27a8ef
37 changed files with 15697 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md Normal file
View File

@@ -0,0 +1 @@
# lm-dataset-preparer

65
pom.xml Executable file
View File

@@ -0,0 +1,65 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>lm-dataset-preparer</artifactId>
<groupId>de.extio</groupId>
<name>lm-dataset-preparer</name>
<version>1.0.0-SNAPSHOT</version>
<properties>
<java.version>21</java.version>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<springdoc.version>2.5.0</springdoc.version>
<spring-boot.version>3.3.4</spring-boot.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>${project.artifactId}</finalName>
<defaultGoal>spring-boot:run</defaultGoal>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<executions>
<execution>
<goals>
<goal>build-info</goal>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

55
src/main/java/de/extio/lmdatasetprep/LmDatasetPreparerApplication.java Executable file
View File

@@ -0,0 +1,55 @@
package de.extio.lmdatasetprep;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
@SpringBootApplication
@EnableCaching
public class LmDatasetPreparerApplication implements CommandLineRunner, ApplicationContextAware {
private static final Logger LOGGER = LoggerFactory.getLogger(LmDatasetPreparerApplication.class);
public static void main(final String[] args) {
SpringApplication.run(LmDatasetPreparerApplication.class, args);
}
private ApplicationContext applicationContext;
@SuppressWarnings("unchecked")
@Override
public void run(final String... args) throws Exception {
if (args.length == 0) {
LOGGER.error("Commandline argument is missing: Provide the tool name as argument");
return;
}
final var bean = this.applicationContext.getBean(args[0]);
if (!(bean instanceof Consumer)) {
LOGGER.info("Invalid bean {}", args[0]);
return;
}
LOGGER.info("Executing {}", args[0]);
try {
((Consumer<String[]>) bean).accept(args);
}
catch (Exception e) {
LOGGER.error("Error executing the component", e);
}
}
@Override
public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
}

194
src/main/java/de/extio/lmdatasetprep/client/Client.java Executable file
View File

@@ -0,0 +1,194 @@
package de.extio.lmdatasetprep.client;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmdatasetprep.client.dto.Choice;
import de.extio.lmdatasetprep.client.dto.CompletionRequest;
import de.extio.lmdatasetprep.client.dto.CompletionResponse;
import de.extio.lmdatasetprep.client.profile.ModelCategory;
import de.extio.lmdatasetprep.client.profile.ModelProfile;
import de.extio.lmdatasetprep.client.profile.ModelProfileService;
import de.extio.lmdatasetprep.client.prompt.PromptStrategy;
import de.extio.lmdatasetprep.client.prompt.PromptStrategyFactory;
@Component
public class Client {
private static final Logger LOGGER = LoggerFactory.getLogger(Client.class);
@Autowired
private WebClient.Builder webClientBuilder;
@Autowired
private PromptStrategyFactory promptStrategyFactory;
@Autowired
private ModelNameSupplier modelNameSupplier;
@Autowired
private Tokenizer tokenizer;
@Autowired
private ModelProfileService modelProfileService;
@Value("${client.collectStatistics}")
private boolean collectStatistics;
private Integer promptTemplateTokenLength;
public Completion completion(final String instruction, final String question, final String fullText, final ModelCategory modelCategory) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
final List<String> texts = new ArrayList<>();
List<Long> tokenizedInstrAndQuestion = this.tokenizer.tokenize(instruction + question, modelProfile);
List<Long> tokenizedFullText = this.tokenizer.tokenize(fullText, modelProfile);
int inputTokens = tokenizedInstrAndQuestion.size() + tokenizedFullText.size() + this.getPromptTemplateTokenLength(modelProfile);
if (inputTokens > modelProfile.maxContextLength() - modelProfile.maxTokens()) {
int steps = modelProfile.maxContextLength() - modelProfile.maxTokens() - tokenizedInstrAndQuestion.size() - this.getPromptTemplateTokenLength(modelProfile);
for (int i = 0; i < tokenizedFullText.size(); i += steps) {
List<Long> split = tokenizedFullText.subList(i, i + steps);
texts.add(this.tokenizer.detokenize(split, modelProfile));
}
}
else {
texts.add(fullText);
}
LOGGER.info("Completion request. Input tokens: {}", inputTokens);
LOGGER.debug("Splitted text into parts: " + texts.size());
final List<StringBuilder> answers = new ArrayList<>();
final CompletionStatistics statistics = new CompletionStatistics();
for (final String text : texts) {
final StringBuilder answer = new StringBuilder();
final StringBuilder prompt = promptStrategy.start(instruction, question, text);
this.requestCompletionsContinuations(prompt, answer, statistics, modelProfile, promptStrategy);
answers.add(answer);
}
if (answers.size() > 1) {
LOGGER.debug("Summarizing answers");
final StringBuilder summary = new StringBuilder();
final StringBuilder prompt = promptStrategy.start(instruction, question, "");
answers.forEach(answer -> promptStrategy.next(prompt, answer.toString(), "Continue"));
promptStrategy.next(prompt, "", "Generate now a full summary");
this.requestCompletionsContinuations(prompt, summary, statistics, modelProfile, promptStrategy);
answers.clear();
answers.add(summary);
}
return new Completion(
answers.stream().collect(Collectors.joining()),
statistics.requests,
statistics.duration,
statistics.inTokens,
statistics.outTokens,
false);
}
private void requestCompletionsContinuations(final StringBuilder prompt, final StringBuilder answer, final CompletionStatistics statistics,
final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
final LocalDateTime start = LocalDateTime.now();
final String promptStr = prompt.toString();
final var response = this.requestCompletions(promptStr, modelProfile);
String content = null;
var complete = false;
if (response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
complete = response.isStoppedEos();
}
else {
final var choice = response.getChoices().getFirst();
content = promptStrategy.removeEOT(choice.getText());
complete = !Choice.FINISH_REASON_LENGTH.equals(choice.getFinishReason());
}
if (this.collectStatistics) {
statistics.add(Duration.between(start, LocalDateTime.now()), this.tokenizer.count(promptStr, modelProfile), this.tokenizer.count(content, modelProfile));
}
answer.append(content);
if (complete) {
LOGGER.debug("Prompt response complete");
break;
}
answer.append('\n');
promptStrategy.continue_(prompt, content);
}
}
private CompletionResponse requestCompletions(final String prompt, final ModelProfile modelProfile) {
LOGGER.debug("Requesting completion");
final var webClient = this.webClientBuilder.baseUrl(modelProfile.baseUrl()).build();
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.baseUrl()));
request.setPrompt(prompt);
request.setMaxTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
request.setStream(false);
return webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.bodyValue(request)
.retrieve()
.bodyToMono(CompletionResponse.class)
.block();
}
private int getPromptTemplateTokenLength(final ModelProfile modelProfile) {
if (this.promptTemplateTokenLength == null) {
final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
var prompt = promptStrategy.start(" ", " ", " ");
this.promptTemplateTokenLength = this.tokenizer.count(prompt.toString(), modelProfile);
}
return this.promptTemplateTokenLength;
}
static class CompletionStatistics {
int requests;
Duration duration = Duration.ofMillis(0l);
long inTokens;
long outTokens;
void add(final Duration duration, final int inTokens, final int outTokens) {
LOGGER.debug("Request duration: " + duration + "; in tokens: " + inTokens + "; out tokens: " + outTokens);
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
}
}
}

38
src/main/java/de/extio/lmdatasetprep/client/ClientConfiguration.java Executable file
View File

@@ -0,0 +1,38 @@
package de.extio.lmdatasetprep.client;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import io.netty.channel.ChannelOption;
import io.netty.handler.timeout.ReadTimeoutHandler;
import io.netty.handler.timeout.WriteTimeoutHandler;
import reactor.netty.http.client.HttpClient;
@Configuration
public class ClientConfiguration {
private static final int TIMEOUT = 900000;
private static final int CONNECT_TIMEOUT = 10000;
@Bean
WebClient.Builder webClientBuilder() {
final HttpClient httpClient = HttpClient.create()
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
.responseTimeout(Duration.ofMillis(TIMEOUT))
.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
return WebClient.builder()
.clientConnector(new ReactorClientHttpConnector(httpClient))
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
.defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
}
}

7
src/main/java/de/extio/lmdatasetprep/client/Completion.java Executable file
View File

@@ -0,0 +1,7 @@
package de.extio.lmdatasetprep.client;
import java.time.Duration;
public record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, boolean cached) {
}

61
src/main/java/de/extio/lmdatasetprep/client/ModelNameLoader.java Executable file
View File

@@ -0,0 +1,61 @@
package de.extio.lmdatasetprep.client;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmdatasetprep.client.dto.Model;
import de.extio.lmdatasetprep.client.dto.ModelsResponse;
public class ModelNameLoader implements Runnable {
private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class);
private final WebClient webClient;
private final Consumer<String> consumer;
public ModelNameLoader(final WebClient webClient, final Consumer<String> consumer) {
this.consumer = consumer;
this.webClient = webClient;
}
@Override
public void run() {
String modelName = null;
do {
try {
LOGGER.info("Querying model name");
modelName = this.webClient
.method(HttpMethod.GET)
.uri(uriBuilder -> uriBuilder.path("/v1/models").build())
.retrieve()
.bodyToMono(ModelsResponse.class)
.block()
.getData()
.stream()
.map(Model::getId)
.findFirst()
.orElseThrow();
}
catch (final Exception e) {
LOGGER.error(e.getMessage(), e);
try {
Thread.sleep(1000l);
}
catch (final InterruptedException e1) {
throw new RuntimeException("Interrupted", e1);
}
}
}
while (modelName == null);
LOGGER.info("Model name: {}", modelName);
this.consumer.accept(modelName);
}
}

58
src/main/java/de/extio/lmdatasetprep/client/ModelNameSupplier.java Executable file
View File

@@ -0,0 +1,58 @@
package de.extio.lmdatasetprep.client;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmdatasetprep.client.profile.ModelProfileService;
@Component
public class ModelNameSupplier implements InitializingBean {
@Autowired
private WebClient.Builder webClientBuilder;
private final Map<String, CompletableFuture<String>> modelNames = new ConcurrentHashMap<>();
private final Map<String, CountDownLatch> countDownLatches = new ConcurrentHashMap<>();
@Autowired
private ModelProfileService modelProfileService;
@Override
public void afterPropertiesSet() throws Exception {
final var urls = this.modelProfileService.getModelProfileUrls();
for (final var url : urls) {
this.countDownLatches.put(url, new CountDownLatch(1));
this.fetchModelName(url);
}
}
private void fetchModelName(final String url) {
final var webClient = this.webClientBuilder.baseUrl(url).build();
CompletableFuture.runAsync(() -> {
final var modelNameLoader = new ModelNameLoader(webClient, modelName -> {
this.modelNames.put(url, CompletableFuture.completedFuture(modelName));
this.countDownLatches.get(url).countDown();
});
modelNameLoader.run();
});
}
public String getModelName(final String url) {
try {
this.countDownLatches.get(url).await();
}
catch (final InterruptedException e) {
throw new RuntimeException("Interrupted", e);
}
return this.modelNames.get(url).join();
}
}

66
src/main/java/de/extio/lmdatasetprep/client/Tokenizer.java Executable file
View File

@@ -0,0 +1,66 @@
package de.extio.lmdatasetprep.client;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmdatasetprep.client.profile.ModelProfile;
@Component
public class Tokenizer {
private static final Logger LOGGER = LoggerFactory.getLogger(Tokenizer.class);
@Autowired
private WebClient.Builder webClientBuilder;
public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
LOGGER.debug("Requesting tokenizing");
final var webClient = this.webClientBuilder.baseUrl(modelProfile.baseUrl()).build();
final var request = new TokenizeRequest(txt);
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/tokenize").build())
.bodyValue(request)
.retrieve()
.bodyToMono(TokenizeResponse.class)
.block();
return response.tokens();
}
public int count(final String txt, final ModelProfile modelProfile) {
return this.tokenize(txt, modelProfile).size();
}
public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
LOGGER.debug("Requesting detokenizing");
final var webClient = this.webClientBuilder.baseUrl(modelProfile.baseUrl()).build();
final var request = new DetokenizeRequest(tokens);
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/detokenize").build())
.bodyValue(request)
.retrieve()
.bodyToMono(DetokenizeResponse.class)
.block();
return response.content();
}
private record TokenizeRequest(String content) {
}
private record TokenizeResponse(List<Long> tokens) {
}
private record DetokenizeRequest(List<Long> tokens) {
}
private record DetokenizeResponse(String content) {
}
}

59
src/main/java/de/extio/lmdatasetprep/client/dto/Choice.java Executable file
View File

@@ -0,0 +1,59 @@
package de.extio.lmdatasetprep.client.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
public class Choice {
public static final String FINISH_REASON_STOP = "stop";
public static final String FINISH_REASON_LENGTH = "length";
public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
private String text;
private Double score;
private String finishReason;
@JsonProperty("text")
public String getText() {
return this.text;
}
public void setText(final String text) {
this.text = text;
}
@JsonProperty("score")
public Double getScore() {
return this.score;
}
public void setScore(final Double score) {
this.score = score;
}
@JsonProperty("finish_reason")
public String getFinishReason() {
return this.finishReason;
}
public void setFinishReason(final String finishReason) {
this.finishReason = finishReason;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("Choice [text=");
builder.append(this.text);
builder.append(", score=");
builder.append(this.score);
builder.append(", finishReason=");
builder.append(this.finishReason);
builder.append("]");
return builder.toString();
}
}

72
src/main/java/de/extio/lmdatasetprep/client/dto/CompletionRequest.java Executable file
View File

@@ -0,0 +1,72 @@
package de.extio.lmdatasetprep.client.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
public class CompletionRequest {
private String model;
private String prompt;
private Integer maxTokens;
private Double temperature;
private Double topP;
private Boolean stream;
@JsonProperty("model")
public String getModel() {
return this.model;
}
public void setModel(final String model) {
this.model = model;
}
@JsonProperty("prompt")
public String getPrompt() {
return this.prompt;
}
public void setPrompt(final String prompt) {
this.prompt = prompt;
}
@JsonProperty("max_tokens")
public Integer getMaxTokens() {
return this.maxTokens;
}
public void setMaxTokens(final Integer maxTokens) {
this.maxTokens = maxTokens;
}
@JsonProperty("temperature")
public Double getTemperature() {
return this.temperature;
}
public void setTemperature(final Double temperature) {
this.temperature = temperature;
}
@JsonProperty("top_p")
public Double getTopP() {
return this.topP;
}
public void setTopP(final Double topP) {
this.topP = topP;
}
@JsonProperty("stream")
public Boolean isStream() {
return this.stream;
}
public void setStream(final Boolean stream) {
this.stream = stream;
}
}

93
src/main/java/de/extio/lmdatasetprep/client/dto/CompletionResponse.java Executable file
View File

@@ -0,0 +1,93 @@
package de.extio.lmdatasetprep.client.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
public class CompletionResponse {
private String id;
private String object;
private Integer created;
private List<Choice> choices;
private String content;
private boolean isStoppedEos;
@JsonProperty("id")
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("created")
public Integer getCreated() {
return this.created;
}
public void setCreated(final Integer created) {
this.created = created;
}
@JsonProperty("choices")
public List<Choice> getChoices() {
return this.choices;
}
public void setChoices(final List<Choice> choices) {
this.choices = choices;
}
@JsonProperty("content")
public String getContent() {
return this.content;
}
public void setContent(final String content) {
this.content = content;
}
@JsonProperty("stopped_eos")
public boolean isStoppedEos() {
return this.isStoppedEos;
}
public void setStoppedEos(final boolean isStoppedEos) {
this.isStoppedEos = isStoppedEos;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("CompletionResponse [id=");
builder.append(this.id);
builder.append(", object=");
builder.append(this.object);
builder.append(", created=");
builder.append(this.created);
builder.append(", choices=");
builder.append(this.choices);
builder.append(", content=");
builder.append(this.content);
builder.append(", isStoppedEos=");
builder.append(this.isStoppedEos);
builder.append("]");
return builder.toString();
}
}

74
src/main/java/de/extio/lmdatasetprep/client/dto/Model.java Executable file
View File

@@ -0,0 +1,74 @@
package de.extio.lmdatasetprep.client.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
public class Model {
private String id;
private String object;
private int created;
private String ownedBy;
private List<String> permission;
private String root;
@JsonProperty("id")
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("created")
public int getCreated() {
return this.created;
}
public void setCreated(final int created) {
this.created = created;
}
@JsonProperty("owned_by")
public String getOwned_by() {
return this.ownedBy;
}
public void setOwned_by(final String owned_by) {
this.ownedBy = owned_by;
}
@JsonProperty("permission")
public List<String> getPermission() {
return this.permission;
}
public void setPermission(final List<String> permission) {
this.permission = permission;
}
@JsonProperty("root")
public String getRoot() {
return this.root;
}
public void setRoot(final String root) {
this.root = root;
}
}

42
src/main/java/de/extio/lmdatasetprep/client/dto/ModelsResponse.java Executable file
View File

@@ -0,0 +1,42 @@
package de.extio.lmdatasetprep.client.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
public class ModelsResponse {
private String object;
private List<Model> data;
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("data")
public List<Model> getData() {
return this.data;
}
public void setData(final List<Model> data) {
this.data = data;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("ModelsResponse [object=");
builder.append(this.object);
builder.append(", data=");
builder.append(this.data);
builder.append("]");
return builder.toString();
}
}

16
src/main/java/de/extio/lmdatasetprep/client/profile/ModelCategory.java Executable file
View File

@@ -0,0 +1,16 @@
package de.extio.lmdatasetprep.client.profile;
public enum ModelCategory {
HOT("profile.hot"),
COLD("profile.cold");
private final String modelProfile;
ModelCategory(final String modelProfile) {
this.modelProfile = modelProfile;
}
public String getModelProfile() {
return this.modelProfile;
}
}

11
src/main/java/de/extio/lmdatasetprep/client/profile/ModelProfile.java Executable file
View File

@@ -0,0 +1,11 @@
package de.extio.lmdatasetprep.client.profile;
public record ModelProfile(
String prompt,
int maxTokens,
int maxContextLength,
double temperature,
double topP,
int maxContinuations,
String baseUrl) {
}

95
src/main/java/de/extio/lmdatasetprep/client/profile/ModelProfileService.java Executable file
View File

@@ -0,0 +1,95 @@
package de.extio.lmdatasetprep.client.profile;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.bind.Bindable;
import org.springframework.boot.context.properties.bind.Binder;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.core.env.Environment;
import org.springframework.core.io.support.ResourcePropertySource;
import org.springframework.stereotype.Service;
@Service
public class ModelProfileService {
private final Environment environment;
private static final String MODEL_PROFILE_PREFIX = "profile";
private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class);
public ModelProfileService(final Environment environment) {
this.environment = environment;
}
@Cacheable("modelProfiles")
public ModelProfile getModelProfile(final String modelProfile) {
final var modelName = this.environment.getProperty(modelProfile);
if (modelName == null) {
throw new IllegalStateException("No model found for profile: " + modelProfile);
}
try {
final var resource = new ResourcePropertySource("classpath:" + modelName + ".properties");
final var name = Optional.ofNullable(resource.getProperty("prompts"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model name found for model: " + modelName));
final var maxTokens = Optional.ofNullable(resource.getProperty("maxTokens"))
.map(Object::toString)
.map(Integer::parseInt)
.orElseThrow(() -> new IllegalStateException("No model maxTokens found for model: " + modelName));
final var maxContextLength = Optional.ofNullable(resource.getProperty("maxContextLength"))
.map(Object::toString)
.map(Integer::parseInt)
.orElseThrow(() -> new IllegalStateException("No model maxContextLength found for model: " + modelName));
final var temperature = Optional.ofNullable(resource.getProperty("temperature"))
.map(Object::toString)
.map(Double::parseDouble)
.orElseThrow(() -> new IllegalStateException("No model temperature found for model: " + modelName));
final var topP = Optional.ofNullable(resource.getProperty("topP"))
.map(Object::toString)
.map(Double::parseDouble)
.orElseThrow(() -> new IllegalStateException("No model topP found for model: " + modelName));
final var maxContinuations = Optional.ofNullable(resource.getProperty("maxContinuations"))
.map(Object::toString)
.map(Integer::parseInt)
.orElseThrow(() -> new IllegalStateException("No model maxContinuations found for model: " + modelName));
final var baseUrl = Optional.ofNullable(resource.getProperty("baseUrl"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model baseUrl found for model: " + modelName));
return new ModelProfile(name, maxTokens, maxContextLength, temperature, topP, maxContinuations, baseUrl);
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile", e);
return null;
}
}
public List<String> getModelProfileUrls() {
final var modelProfiles = Binder.get(this.environment)
.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
.orElse(Collections.emptyMap());
return modelProfiles.values().stream()
.filter(model -> model != null && !model.isEmpty())
.map(model -> {
try {
final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
return resource.getProperty("baseUrl") != null ? resource.getProperty("baseUrl").toString() : null;
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile for: " + model, e);
return null;
}
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
}

45
src/main/java/de/extio/lmdatasetprep/client/prompt/ChatMLPromptStrategy.java Executable file
View File

@@ -0,0 +1,45 @@
package de.extio.lmdatasetprep.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class ChatMLPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<|im_start|>system\n");
prompt.append(instruction);
prompt.append("<|im_end|>\n<|im_start|>user\n");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append("<|im_end|>\n<|im_start|>assistant\n");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);
prompt.append("<|im_end|>\n<|im_start|>user\n");
prompt.append(user);
prompt.append("<|im_end|>\n<|im_start|>assistant\n");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<|im_end|>", "");
}
@Override
public String getPromptName() {
return "chatml";
}
}

48
src/main/java/de/extio/lmdatasetprep/client/prompt/Gemma2PromptStrategy.java Executable file
View File

@@ -0,0 +1,48 @@
package de.extio.lmdatasetprep.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class Gemma2PromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<start_of_turn>user\n");
if (!instruction.isEmpty()) {
prompt.append(instruction);
prompt.append("\n");
}
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append("<end_of_turn>\n<start_of_turn>model\n");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);
prompt.append("<end_of_turn>\n<start_of_turn>user\n");
prompt.append(user);
prompt.append("<end_of_turn>\n<start_of_turn>model\n");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<end_of_turn>", "");
}
@Override
public String getPromptName() {
return "gemma2";
}
}

47
src/main/java/de/extio/lmdatasetprep/client/prompt/Llama3PromptStrategy.java Executable file
View File

@@ -0,0 +1,47 @@
package de.extio.lmdatasetprep.client.prompt;
import org.springframework.stereotype.Component;
/**
* @see https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
*/
@Component
public class Llama3PromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n");
prompt.append(instruction);
prompt.append("<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n\n");
}
prompt.append(text);
prompt.append("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);
prompt.append("<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n" + user + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<|eot_id|>", "");
}
@Override
public String getPromptName() {
return "llama3";
}
}

40
src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java Executable file
View File

@@ -0,0 +1,40 @@
package de.extio.lmdatasetprep.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class MixtralPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<s>[INST] ");
prompt.append(instruction);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append(" ");
}
prompt.append(question);
prompt.append(" [/INST]");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append("\n\"");
prompt.append(assistant);
prompt.append("\"</s> [INST] ");
prompt.append(user);
prompt.append(" [/INST]");
}
@Override
public String getPromptName() {
return "mixtral";
}
}

46
src/main/java/de/extio/lmdatasetprep/client/prompt/Phi3PromptStrategy.java Executable file
View File

@@ -0,0 +1,46 @@
package de.extio.lmdatasetprep.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class Phi3PromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<|user|>\n");
prompt.append(instruction);
prompt.append(" ");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append(" <|end|>\n<|assistant|>");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append("\n");
prompt.append(assistant);
prompt.append("<|user|>\n");
prompt.append(user);
prompt.append("<|end|>\n<|assistant|>");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<|end|>", "");
}
@Override
public String getPromptName() {
return "phi3";
}
}

17
src/main/java/de/extio/lmdatasetprep/client/prompt/PromptStrategy.java Executable file
View File

@@ -0,0 +1,17 @@
package de.extio.lmdatasetprep.client.prompt;
public interface PromptStrategy {
String getPromptName();
StringBuilder start(String instruction, String question, String text);
void continue_(StringBuilder prompt, String assistant);
void next(StringBuilder prompt, String assistant, String user);
default String removeEOT(final String prompt) {
return prompt.strip();
}
}

22
src/main/java/de/extio/lmdatasetprep/client/prompt/PromptStrategyFactory.java Executable file
View File

@@ -0,0 +1,22 @@
package de.extio.lmdatasetprep.client.prompt;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.stereotype.Component;
@Component
public class PromptStrategyFactory {
private final Map<String, PromptStrategy> strategies = new HashMap<>();
public PromptStrategyFactory(final List<PromptStrategy> strategies) {
strategies.forEach(strategy -> this.strategies.put(strategy.getPromptName(), strategy));
}
public PromptStrategy getStrategy(final String modelPrompt) {
return this.strategies.get(modelPrompt);
}
}

46
src/main/java/de/extio/lmdatasetprep/client/prompt/VicunaPromptStrategy.java Executable file
View File

@@ -0,0 +1,46 @@
package de.extio.lmdatasetprep.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class VicunaPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append(instruction);
prompt.append(" USER: ");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append(" ASSISTANT:");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(" ");
prompt.append(assistant);
prompt.append("</s>USER: ");
prompt.append(user);
prompt.append(" ASSISTANT:");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("</s>", "");
}
@Override
public String getPromptName() {
return "vicuna";
}
}

112
src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java Normal file
View File

@@ -0,0 +1,112 @@
package de.extio.lmdatasetprep.preparer;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.client.Client;
import de.extio.lmdatasetprep.client.profile.ModelCategory;
@Component
public class Rewrite implements Consumer<String[]> {
private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class);
private static final ThreadLocal<Random> RANDOM = ThreadLocal.withInitial(XorShift128Random::new);
private static String IMPROVEMENT_PROMPT = "Improve the following paragraph of an existing text without changing the existing sentences, improve the text in a nuanced and understated way. Don't include a preamble and no explanation:";
private static String ENHANCE_PROMPT = "Improve the following paragraph of an existing text. %s without changing the existing sentences, focusing on enriching the description of the setting in a nuanced and understated way. Don't include a preamble and no explanation:";
private static List<String> ENHANCEMENTS = List.of(
"Infuse this paragraph with evocative sensory details, focusing on the texture of objects and the atmosphere of the setting",
"Add subtle hints of the character's inner thoughts and emotions within the existing dialogue and actions",
"Enhance the paragraph with figurative language like metaphors or similes to create a richer tapestry of imagery",
"Develop the subtext of the paragraph by suggesting unspoken tensions or hidden motivations between characters",
"Introduce a faint scent or sound that adds another layer of sensory detail to the scene without disrupting the flow",
"Imbue the paragraph with a sense of foreboding or anticipation by subtly hinting at future events without revealing specifics",
"Weave in a historical or cultural reference relevant to the setting or characters to add depth and context",
"Elevate the language of the paragraph by replacing basic verbs and adjectives with more evocative synonyms",
"Add a touch of humor or irony to the paragraph through clever wordplay or unexpected observations",
"Imagine this paragraph as a still photograph. Describe the composition and lighting to further bring the scene to life");
@Autowired
private Client client;
@Override
public void accept(final String[] args) {
if (args.length < 5) {
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <Texts per input> <Model suffix>");
return;
}
Utils.transformDirectory(args[1],
f -> {
final List<Runnable> tasks = new ArrayList<>();
tasks.add(() -> this.rewriteFile(args, f, 0, "impr", ModelCategory.COLD, IMPROVEMENT_PROMPT));
for (final ModelCategory modelCategory : List.of(ModelCategory.COLD, ModelCategory.HOT)) {
for (int i = 0; i < Integer.parseInt(args[3]); i++) {
final int fi = i;
tasks.add(() -> this.rewriteFile(args, f, fi, "enh", modelCategory, String.format(ENHANCE_PROMPT, ENHANCEMENTS.get(RANDOM.get().nextInt(ENHANCEMENTS.size())))));
}
}
return tasks;
});
}
private void rewriteFile(final String[] args, final Path f, final int i, final String identifier, final ModelCategory modelCategory, final String prompt) {
final Path out = Utils.suffixFilename(Path.of(args[2]).resolve(f.getFileName()),
"rewr",
args[4],
identifier,
modelCategory.toString().toLowerCase(),
String.valueOf(i));
if (Files.exists(out)) {
LOGGER.info("Skipping " + out);
return;
}
LOGGER.info("Rewriting " + f.getFileName() + " " + identifier + " " + modelCategory.name() + " " + i);
final List<String> paragraphs = this.rewrite(f, prompt, modelCategory);
try {
Files.writeString(out, String.join("\n\n", paragraphs));
}
catch (final IOException e) {
LOGGER.error("Cannot store file", e);
}
}
List<String> rewrite(final Path file, final String prompt, final ModelCategory modelCategory) {
String text;
try {
text = Utils.normalizeText(Files.readString(file));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
final List<String> splits = Utils.splitParagraphs(text, Utils.PARAGRAPH_CHUNKS_NORM, Utils.PARAGRAPH_CHUNKS_VAR);
final List<String> paragraphs = new ArrayList<>(splits.size());
for (final String split : splits) {
final var completion = this.client.completion("You are a helpful assistant with great authoring skills.",
prompt,
split,
modelCategory);
paragraphs.add(completion.response());
}
return paragraphs;
}
}

78
src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl.java Normal file
View File

@@ -0,0 +1,78 @@
package de.extio.lmdatasetprep.preparer;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@Component
public class Text2Jsonl implements Consumer<String[]> {
private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl.class);
@Override
public void accept(final String[] args) {
if (args.length < 5) {
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <chunk norm> <chunk var>");
return;
}
final List<String> lines = Collections.synchronizedList(new ArrayList<>());
Utils.transformDirectory(args[1], p -> {
return List.of(() -> {
lines.addAll(this.splitToJsonl(p, Integer.parseInt(args[3]), Integer.parseInt(args[4])));
});
});
final Path out = Path.of(args[2], "dataset.jsonl");
try {
Files.writeString(out, String.join("\n", lines));
}
catch (final IOException e) {
LOGGER.error("Cannot store dataset", e);
}
}
List<String> splitToJsonl(final Path file, final int chunkNorm, final int chunkVar) {
LOGGER.info("Splitting to jsonl " + file);
String text;
try {
text = Utils.normalizeText(Files.readString(file));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
final ObjectMapper mapper = new ObjectMapper();
return Utils
.splitParagraphs(text, chunkNorm, chunkVar)
.stream()
.map(split -> {
try {
return mapper.writeValueAsString(new TextLine(split));
}
catch (final JsonProcessingException e) {
LOGGER.error("Cannot convert split to json", e);
return "";
}
})
.toList();
}
record TextLine(String text) {
}
}

77
src/main/java/de/extio/lmdatasetprep/preparer/Translate.java Normal file
View File

@@ -0,0 +1,77 @@
package de.extio.lmdatasetprep.preparer;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import de.extio.lmdatasetprep.client.Client;
import de.extio.lmdatasetprep.client.profile.ModelCategory;
@Component
public class Translate implements Consumer<String[]> {
private static final Logger LOGGER = LoggerFactory.getLogger(Translate.class);
@Autowired
private Client client;
@Override
public void accept(final String[] args) {
if (args.length < 3) {
LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir>");
return;
}
Utils.transformDirectory(args[1], p -> {
return List.of(() -> {
try {
Path out = Utils.suffixFilename(Path.of(args[2]).resolve(p.getFileName()), "_en");
if (Files.exists(out)) {
LOGGER.info("Skipping " + out);
}
else {
final var translations = this.translate(p);
Files.writeString(out, String.join("\n\n", translations));
}
}
catch (final IOException e) {
LOGGER.error("Cannot store file", e);
}
});
});
}
List<String> translate(final Path file) {
LOGGER.info("Translating " + file);
String text;
try {
text = Utils.normalizeText(Files.readString(file));
}
catch (final IOException e) {
throw new RuntimeException("Cannot read file", e);
}
final List<String> splits = Utils.splitParagraphs(text, Utils.PARAGRAPH_CHUNKS_NORM, Utils.PARAGRAPH_CHUNKS_VAR);
final List<String> translations = new ArrayList<>(splits.size());
for (final String split : splits) {
final var completion = this.client.completion("You are a helpful assistant with great language translation and authoring skills.",
"Translate the following text to English. Don't include a preamble and no explanation:",
split,
ModelCategory.COLD);
translations.add(completion.response());
}
return translations;
}
}

139
src/main/java/de/extio/lmdatasetprep/preparer/Utils.java Normal file
View File

@@ -0,0 +1,139 @@
package de.extio.lmdatasetprep.preparer;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Function;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Utils {
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
static final int PARAGRAPH_CHUNKS_NORM = 1250;
static final int PARAGRAPH_CHUNKS_VAR = 350;
static String normalizeText(final String text) {
String result = text;
// Remove excessive newlines, spaces and tabs
int cnt = 0;
do {
cnt = result.length();
result = result.replace("\r", "\n");
result = result.replace("\n\n\n", "\n\n");
result = result.replace("\t", "");
result = result.replace(" ", " ");
result = result.replace(" .", ".");
result = result.replace(". ", ". ");
result = result.replace("\n.", ".");
result = result.replace(" !", "!");
result = result.replace("! ", "! ");
result = result.replace("\n!", "!");
result = result.replace(" ?", "?");
result = result.replace("? ", "? ");
result = result.replace("\n?", "?");
result = result.replace(" \n", "\n");
} while (result.length() != cnt);
// Remove newlines mid of the sentence
final StringBuilder sb = new StringBuilder(result);
for (int i = 1; i < sb.length(); i++) {
final char c = sb.charAt(i);
if (c == '\n' && sb.charAt(i - 1) != '.' && sb.charAt(i - 1) != '?' && sb.charAt(i - 1) != '!' && sb.charAt(i - 1) != '\n') {
sb.setCharAt(i, ' ');
}
}
result = sb.toString();
LOGGER.info("Normalizer stats: Before: {} After {}", text.length(), result.length());
return result;
}
static List<String> splitParagraphs(final String text, final int chunks_norm, final int chunks_var) {
final int CHUNKS_MIN = chunks_norm - chunks_var;
final int CHUNKS_MAX = chunks_norm + chunks_var;
if (text.length() <= chunks_norm + chunks_var) {
return Collections.singletonList(text);
}
final List<String> splits = new ArrayList<>();
int pos = 0;
paragraphs: do {
if (text.length() - pos > chunks_norm + chunks_var) {
delimiters: for (final String delimiter : List.of("\n\n", "\n", ".")) {
int next = text.indexOf(delimiter, pos);
while (next > -1) {
if (next < pos + CHUNKS_MIN) {
next = text.indexOf(delimiter, next + 1);
}
else if (next > pos + CHUNKS_MAX) {
continue delimiters;
}
else {
String split = text.substring(pos, next).trim();
if (!split.endsWith(".")) {
split = split + ".";
}
splits.add(split);
pos = next + delimiter.length();
continue paragraphs;
}
}
}
}
final int next = Math.min(pos + chunks_norm, text.length());
final String split = text.substring(pos, next).trim();
splits.add(split);
pos = next + 1;
} while (pos < text.length());
LOGGER.info("Created {} splits", splits.size());
return splits;
}
static void transformDirectory(final String path, final Function<Path, List<Runnable>> createTasks) {
try (final ExecutorService executor = Executors.newFixedThreadPool(4);
Stream<Path> stream = Files.list(Paths.get(path))) {
final List<CompletableFuture<?>> tasks = new ArrayList<>();
stream
.filter(p -> Files.isRegularFile(p))
.forEach(p -> {
for (final Runnable r : createTasks.apply(p)) {
tasks.add(CompletableFuture.runAsync(r, executor));
}
});
CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).join();
}
catch (final IOException e) {
LOGGER.error("Error", e);
}
}
static Path suffixFilename(final Path f, final String... suffixes) {
final String[] parts = f.getFileName().toString().split("[.]");
final StringBuilder sb = new StringBuilder();
for (final String suffix : suffixes) {
sb.append('_');
sb.append(suffix);
}
sb.append('.');
return f.getParent().resolve(parts[0] + sb.toString() + parts[1]);
}
}

62
src/main/java/de/extio/lmdatasetprep/preparer/XorShift128Random.java Normal file
View File

@@ -0,0 +1,62 @@
package de.extio.lmdatasetprep.preparer;
import java.util.Random;
import java.util.UUID;
@SuppressWarnings("serial")
public final class XorShift128Random extends Random {
private long state0;
private long state1;
public XorShift128Random() {
super();
}
public XorShift128Random(final long seed) {
this.setSeed(seed);
}
public XorShift128Random(final String state) {
final UUID uuid = UUID.fromString(state);
this.state0 = uuid.getMostSignificantBits();
this.state1 = uuid.getLeastSignificantBits();
}
@Override
public synchronized void setSeed(final long seed) {
this.state0 = avalanche(seed == 0 ? -1 : seed);
this.state1 = avalanche(this.state0);
this.state0 = avalanche(this.state1);
}
@Override
protected int next(final int bits) {
return (int) (this.nextLong() & (1L << bits) - 1);
}
@Override
public long nextLong() {
long s1 = this.state0;
final long s0 = this.state1;
this.state0 = s0;
s1 ^= s1 << 23;
return (this.state1 = s1 ^ s0 ^ (s1 >>> 17) ^ (s0 >>> 26)) + s0;
}
@Override
public String toString() {
return new UUID(this.state0, this.state1).toString();
}
private static long avalanche(long k) {
k ^= k >>> 33;
k *= 0xff51afd7ed558ccdL;
k ^= k >>> 33;
k *= 0xc4ceb9fe1a85ec53L;
k ^= k >>> 33;
return k;
}
}

8
src/main/resources/application.properties Executable file
View File

@@ -0,0 +1,8 @@
spring.main.web-application-type=NONE
logging.level.de.extio=debug
profile.hot=llama3-hot
profile.cold=llama3-cold
client.collectStatistics=false

7
src/main/resources/llama3-cold.properties Executable file
View File

@@ -0,0 +1,7 @@
prompts=chatml
maxTokens=2000
maxContextLength=4000
temperature=0.2
topP=1.0
maxContinuations=1
baseUrl=http://localhost:5001

7
src/main/resources/llama3-hot.properties Executable file
View File

@@ -0,0 +1,7 @@
prompts=chatml
maxTokens=2000
maxContextLength=4000
temperature=0.5
topP=1.0
maxContinuations=1
baseUrl=http://localhost:5001

13739
src/main/resources/openai.yaml Executable file
View File

File diff suppressed because it is too large Load Diff

145
src/test/java/de/extio/lmdatasetprep/client/ClientTest.java Executable file
View File

@@ -0,0 +1,145 @@
package de.extio.lmdatasetprep.client;
import java.time.LocalDateTime;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import de.extio.lmdatasetprep.client.profile.ModelCategory;
@Disabled
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
public class ClientTest {
private static final Logger LOGGER = LoggerFactory.getLogger(ClientTest.class);
@Autowired
private Client client;
@Test
void multiPrompts() throws Exception {
final var completion = this.client.completion(
"You are a clever story teller",
"Continue this story:",
"Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!",
ModelCategory.HOT);
LOGGER.info(completion.response());
}
@Test
void hugePrompt() throws Exception {
final StringBuilder sb = new StringBuilder();
for (int i = 0; i < 10000; i++) {
sb.append(i);
sb.append(", ");
}
sb.delete(sb.length() - 2, sb.length());
final var completion = this.client.completion(
"You are counting machine",
"How many numbers do you count?",
sb.toString(),
ModelCategory.HOT);
LOGGER.info(completion.response());
}
@Test
void parallelRequests() throws Exception {
final LocalDateTime start = LocalDateTime.now();
final Tasks tasks = this.createTasks();
final ExecutorService executor = Executors.newFixedThreadPool(4);
try {
CompletableFuture.allOf(
CompletableFuture.runAsync(tasks.task1, executor),
CompletableFuture.runAsync(tasks.task2, executor),
CompletableFuture.runAsync(tasks.task3, executor),
CompletableFuture.runAsync(tasks.task4, executor))
.join();
}
finally {
executor.shutdown();
}
LOGGER.info("Duration: " + java.time.Duration.between(start, LocalDateTime.now()));
}
@Test
void serialRequests() throws Exception {
final LocalDateTime start = LocalDateTime.now();
final Tasks tasks = this.createTasks();
tasks.task1.run();
tasks.task2.run();
tasks.task3.run();
tasks.task4.run();
LOGGER.info("Duration: " + java.time.Duration.between(start, LocalDateTime.now()));
}
private Tasks createTasks() {
return new Tasks(() -> {
try {
final var completion = this.client.completion(
"You are a helpful assistant",
"",
"How do I calculate the annual profit margin?",
ModelCategory.COLD);
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
}, () -> {
try {
final var completion = this.client.completion(
"You are a helpful assistant",
"",
"How do I calculate the operating profit ratio?",
ModelCategory.COLD);
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
}, () -> {
try {
final var completion = this.client.completion(
"You are a helpful assistant",
"",
"How do I calculate the return of investment?",
ModelCategory.COLD);
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
}, () -> {
try {
final var completion = this.client.completion(
"You are a helpful assistant",
"",
"How do I calculate the return on net worth?",
ModelCategory.COLD);
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
});
}
static record Tasks(Runnable task1, Runnable task2, Runnable task3, Runnable task4) {
}
}

1
src/test/resources/application.properties Executable file
View File

@@ -0,0 +1 @@
logging.level.de.extio=debug

4
src/test/resources/junit-platform.properties Executable file
View File

@@ -0,0 +1,4 @@
junit.jupiter.execution.parallel.enabled=true
junit.jupiter.execution.parallel.config.strategy=fixed
junit.jupiter.execution.parallel.config.fixed.max-pool-size=2
junit.jupiter.execution.parallel.config.fixed.parallelism=2