initial commit

src/main/java/de/extio/lmdatasetprep/LmDatasetPreparerApplication.java

@@ -0,0 +1,55 @@
+package de.extio.lmdatasetprep;
+
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+
+@SpringBootApplication
+@EnableCaching
+public class LmDatasetPreparerApplication implements CommandLineRunner, ApplicationContextAware {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(LmDatasetPreparerApplication.class);
+	
+	public static void main(final String[] args) {
+		SpringApplication.run(LmDatasetPreparerApplication.class, args);
+	}
+	
+	private ApplicationContext applicationContext;
+	
+	@SuppressWarnings("unchecked")
+	@Override
+	public void run(final String... args) throws Exception {
+		if (args.length == 0) {
+			LOGGER.error("Commandline argument is missing: Provide the tool name as argument");
+			return;
+		}
+		
+		final var bean = this.applicationContext.getBean(args[0]);
+		if (!(bean instanceof Consumer)) {
+			LOGGER.info("Invalid bean {}", args[0]);
+			return;
+		}
+		
+		LOGGER.info("Executing {}", args[0]);
+		try {
+			((Consumer<String[]>) bean).accept(args);
+		}
+		catch (Exception e) {
+			LOGGER.error("Error executing the component", e);
+		}
+	}
+	
+	@Override
+	public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException {
+		this.applicationContext = applicationContext;
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/Client.java

@@ -0,0 +1,194 @@
+package de.extio.lmdatasetprep.client;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmdatasetprep.client.dto.Choice;
+import de.extio.lmdatasetprep.client.dto.CompletionRequest;
+import de.extio.lmdatasetprep.client.dto.CompletionResponse;
+import de.extio.lmdatasetprep.client.profile.ModelCategory;
+import de.extio.lmdatasetprep.client.profile.ModelProfile;
+import de.extio.lmdatasetprep.client.profile.ModelProfileService;
+import de.extio.lmdatasetprep.client.prompt.PromptStrategy;
+import de.extio.lmdatasetprep.client.prompt.PromptStrategyFactory;
+
+@Component
+public class Client {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(Client.class);
+	
+	@Autowired
+	private WebClient.Builder webClientBuilder;
+	
+	@Autowired
+	private PromptStrategyFactory promptStrategyFactory;
+	
+	@Autowired
+	private ModelNameSupplier modelNameSupplier;
+	
+	@Autowired
+	private Tokenizer tokenizer;
+	
+	@Autowired
+	private ModelProfileService modelProfileService;
+	
+	@Value("${client.collectStatistics}")
+	private boolean collectStatistics;
+	
+	private Integer promptTemplateTokenLength;
+	
+	public Completion completion(final String instruction, final String question, final String fullText, final ModelCategory modelCategory) {
+		final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
+		final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
+
+		final List<String> texts = new ArrayList<>();
+		List<Long> tokenizedInstrAndQuestion = this.tokenizer.tokenize(instruction + question, modelProfile);
+		List<Long> tokenizedFullText = this.tokenizer.tokenize(fullText, modelProfile);
+		int inputTokens = tokenizedInstrAndQuestion.size() + tokenizedFullText.size() + this.getPromptTemplateTokenLength(modelProfile);
+		if (inputTokens > modelProfile.maxContextLength() - modelProfile.maxTokens()) {
+			int steps = modelProfile.maxContextLength() - modelProfile.maxTokens() - tokenizedInstrAndQuestion.size() - this.getPromptTemplateTokenLength(modelProfile);
+			for (int i = 0; i < tokenizedFullText.size(); i += steps) {
+				List<Long> split = tokenizedFullText.subList(i, i + steps);
+				texts.add(this.tokenizer.detokenize(split, modelProfile));
+			}
+		}
+		else {
+			texts.add(fullText);
+		}
+		LOGGER.info("Completion request. Input tokens: {}", inputTokens);
+		LOGGER.debug("Splitted text into parts: " + texts.size());
+		
+		final List<StringBuilder> answers = new ArrayList<>();
+		
+		final CompletionStatistics statistics = new CompletionStatistics();
+		for (final String text : texts) {
+			final StringBuilder answer = new StringBuilder();
+			
+			final StringBuilder prompt = promptStrategy.start(instruction, question, text);
+			this.requestCompletionsContinuations(prompt, answer, statistics, modelProfile, promptStrategy);
+			
+			answers.add(answer);
+		}
+		
+		if (answers.size() > 1) {
+			LOGGER.debug("Summarizing answers");
+			
+			final StringBuilder summary = new StringBuilder();
+			
+			final StringBuilder prompt = promptStrategy.start(instruction, question, "");
+			answers.forEach(answer -> promptStrategy.next(prompt, answer.toString(), "Continue"));
+			promptStrategy.next(prompt, "", "Generate now a full summary");
+			
+			this.requestCompletionsContinuations(prompt, summary, statistics, modelProfile, promptStrategy);
+			
+			answers.clear();
+			answers.add(summary);
+		}
+		
+		return new Completion(
+				answers.stream().collect(Collectors.joining()),
+				statistics.requests,
+				statistics.duration,
+				statistics.inTokens,
+				statistics.outTokens,
+				false);
+	}
+	
+	private void requestCompletionsContinuations(final StringBuilder prompt, final StringBuilder answer, final CompletionStatistics statistics,
+			final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
+		for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
+			final LocalDateTime start = LocalDateTime.now();
+			
+			final String promptStr = prompt.toString();
+			
+			final var response = this.requestCompletions(promptStr, modelProfile);
+			
+			String content = null;
+			var complete = false;
+			if (response.getContent() != null) {
+				content = promptStrategy.removeEOT(response.getContent());
+				complete = response.isStoppedEos();
+			}
+			else {
+				final var choice = response.getChoices().getFirst();
+				content = promptStrategy.removeEOT(choice.getText());
+				complete = !Choice.FINISH_REASON_LENGTH.equals(choice.getFinishReason());
+			}
+			
+			if (this.collectStatistics) {
+				statistics.add(Duration.between(start, LocalDateTime.now()), this.tokenizer.count(promptStr, modelProfile), this.tokenizer.count(content, modelProfile));
+			}
+			
+			answer.append(content);
+			
+			if (complete) {
+				LOGGER.debug("Prompt response complete");
+				break;
+			}
+			
+			answer.append('\n');
+			promptStrategy.continue_(prompt, content);
+		}
+	}
+	
+	private CompletionResponse requestCompletions(final String prompt, final ModelProfile modelProfile) {
+		LOGGER.debug("Requesting completion");
+		final var webClient = this.webClientBuilder.baseUrl(modelProfile.baseUrl()).build();
+		final var request = new CompletionRequest();
+		request.setModel(this.modelNameSupplier.getModelName(modelProfile.baseUrl()));
+		request.setPrompt(prompt);
+		request.setMaxTokens(modelProfile.maxTokens());
+		request.setTemperature(modelProfile.temperature());
+		request.setTopP(modelProfile.topP());
+		request.setStream(false);
+		
+		return webClient
+				.method(HttpMethod.POST)
+				.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
+				.bodyValue(request)
+				.retrieve()
+				.bodyToMono(CompletionResponse.class)
+				.block();
+	}
+	
+	private int getPromptTemplateTokenLength(final ModelProfile modelProfile) {
+		if (this.promptTemplateTokenLength == null) {
+			final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
+			var prompt = promptStrategy.start(" ", " ", " ");
+			this.promptTemplateTokenLength = this.tokenizer.count(prompt.toString(), modelProfile);
+		}
+		return this.promptTemplateTokenLength;
+	}
+	
+	static class CompletionStatistics {
+		
+		int requests;
+		
+		Duration duration = Duration.ofMillis(0l);
+		
+		long inTokens;
+		
+		long outTokens;
+		
+		void add(final Duration duration, final int inTokens, final int outTokens) {
+			LOGGER.debug("Request duration: " + duration + "; in tokens: " + inTokens + "; out tokens: " + outTokens);
+			
+			this.requests++;
+			this.duration = this.duration.plus(duration);
+			this.inTokens += inTokens;
+			this.outTokens += outTokens;
+		}
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/ClientConfiguration.java

@@ -0,0 +1,38 @@
+package de.extio.lmdatasetprep.client;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import io.netty.channel.ChannelOption;
+import io.netty.handler.timeout.ReadTimeoutHandler;
+import io.netty.handler.timeout.WriteTimeoutHandler;
+import reactor.netty.http.client.HttpClient;
+
+@Configuration
+public class ClientConfiguration {
+	
+	private static final int TIMEOUT = 900000;
+	
+	private static final int CONNECT_TIMEOUT = 10000;
+	
+	@Bean
+	WebClient.Builder webClientBuilder() {
+		final HttpClient httpClient = HttpClient.create()
+				.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
+				.responseTimeout(Duration.ofMillis(TIMEOUT))
+				.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
+		
+		return WebClient.builder()
+				.clientConnector(new ReactorClientHttpConnector(httpClient))
+				.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+				.defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/Completion.java

@@ -0,0 +1,7 @@
+package de.extio.lmdatasetprep.client;
+
+import java.time.Duration;
+
+public record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, boolean cached) {
+	
+}

src/main/java/de/extio/lmdatasetprep/client/ModelNameLoader.java

@@ -0,0 +1,61 @@
+package de.extio.lmdatasetprep.client;
+
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpMethod;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmdatasetprep.client.dto.Model;
+import de.extio.lmdatasetprep.client.dto.ModelsResponse;
+
+public class ModelNameLoader implements Runnable {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class);
+	
+	private final WebClient webClient;
+	
+	private final Consumer<String> consumer;
+	
+	public ModelNameLoader(final WebClient webClient, final Consumer<String> consumer) {
+		this.consumer = consumer;
+		this.webClient = webClient;
+	}
+	
+	@Override
+	public void run() {
+		String modelName = null;
+		do {
+			try {
+				LOGGER.info("Querying model name");
+				
+				modelName = this.webClient
+						.method(HttpMethod.GET)
+						.uri(uriBuilder -> uriBuilder.path("/v1/models").build())
+						.retrieve()
+						.bodyToMono(ModelsResponse.class)
+						.block()
+						.getData()
+						.stream()
+						.map(Model::getId)
+						.findFirst()
+						.orElseThrow();
+			}
+			catch (final Exception e) {
+				LOGGER.error(e.getMessage(), e);
+				try {
+					Thread.sleep(1000l);
+				}
+				catch (final InterruptedException e1) {
+					throw new RuntimeException("Interrupted", e1);
+				}
+			}
+		}
+		while (modelName == null);
+		
+		LOGGER.info("Model name: {}", modelName);
+		this.consumer.accept(modelName);
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/ModelNameSupplier.java

@@ -0,0 +1,58 @@
+package de.extio.lmdatasetprep.client;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmdatasetprep.client.profile.ModelProfileService;
+
+@Component
+public class ModelNameSupplier implements InitializingBean {
+	
+	@Autowired
+	private WebClient.Builder webClientBuilder;
+	
+	private final Map<String, CompletableFuture<String>> modelNames = new ConcurrentHashMap<>();
+	
+	private final Map<String, CountDownLatch> countDownLatches = new ConcurrentHashMap<>();
+	
+	@Autowired
+	private ModelProfileService modelProfileService;
+	
+	@Override
+	public void afterPropertiesSet() throws Exception {
+		final var urls = this.modelProfileService.getModelProfileUrls();
+		for (final var url : urls) {
+			this.countDownLatches.put(url, new CountDownLatch(1));
+			this.fetchModelName(url);
+		}
+	}
+	
+	private void fetchModelName(final String url) {
+		final var webClient = this.webClientBuilder.baseUrl(url).build();
+		CompletableFuture.runAsync(() -> {
+			final var modelNameLoader = new ModelNameLoader(webClient, modelName -> {
+				this.modelNames.put(url, CompletableFuture.completedFuture(modelName));
+				this.countDownLatches.get(url).countDown();
+			});
+			modelNameLoader.run();
+		});
+	}
+	
+	public String getModelName(final String url) {
+		try {
+			this.countDownLatches.get(url).await();
+		}
+		catch (final InterruptedException e) {
+			throw new RuntimeException("Interrupted", e);
+		}
+		return this.modelNames.get(url).join();
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/Tokenizer.java

@@ -0,0 +1,66 @@
+package de.extio.lmdatasetprep.client;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmdatasetprep.client.profile.ModelProfile;
+
+@Component
+public class Tokenizer {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(Tokenizer.class);
+	
+	@Autowired
+	private WebClient.Builder webClientBuilder;
+	
+	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
+		LOGGER.debug("Requesting tokenizing");
+		final var webClient = this.webClientBuilder.baseUrl(modelProfile.baseUrl()).build();
+		final var request = new TokenizeRequest(txt);
+		final var response = webClient
+				.method(HttpMethod.POST)
+				.uri(uriBuilder -> uriBuilder.path("/tokenize").build())
+				.bodyValue(request)
+				.retrieve()
+				.bodyToMono(TokenizeResponse.class)
+				.block();
+		return response.tokens();
+	}
+	
+	public int count(final String txt, final ModelProfile modelProfile) {
+		return this.tokenize(txt, modelProfile).size();
+	}
+	
+	public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
+		LOGGER.debug("Requesting detokenizing");
+		final var webClient = this.webClientBuilder.baseUrl(modelProfile.baseUrl()).build();
+		final var request = new DetokenizeRequest(tokens);
+		final var response = webClient
+				.method(HttpMethod.POST)
+				.uri(uriBuilder -> uriBuilder.path("/detokenize").build())
+				.bodyValue(request)
+				.retrieve()
+				.bodyToMono(DetokenizeResponse.class)
+				.block();
+		return response.content();
+	}
+	
+	private record TokenizeRequest(String content) {
+	}
+	
+	private record TokenizeResponse(List<Long> tokens) {
+	}
+	
+	private record DetokenizeRequest(List<Long> tokens) {
+	}
+	
+	private record DetokenizeResponse(String content) {
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/dto/Choice.java

@@ -0,0 +1,59 @@
+package de.extio.lmdatasetprep.client.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Choice {
+	
+	public static final String FINISH_REASON_STOP = "stop";
+	
+	public static final String FINISH_REASON_LENGTH = "length";
+	
+	public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
+	
+	private String text;
+	
+	private Double score;
+	
+	private String finishReason;
+	
+	@JsonProperty("text")
+	public String getText() {
+		return this.text;
+	}
+	
+	public void setText(final String text) {
+		this.text = text;
+	}
+	
+	@JsonProperty("score")
+	public Double getScore() {
+		return this.score;
+	}
+	
+	public void setScore(final Double score) {
+		this.score = score;
+	}
+	
+	@JsonProperty("finish_reason")
+	public String getFinishReason() {
+		return this.finishReason;
+	}
+	
+	public void setFinishReason(final String finishReason) {
+		this.finishReason = finishReason;
+	}
+	
+	@Override
+	public String toString() {
+		final StringBuilder builder = new StringBuilder();
+		builder.append("Choice [text=");
+		builder.append(this.text);
+		builder.append(", score=");
+		builder.append(this.score);
+		builder.append(", finishReason=");
+		builder.append(this.finishReason);
+		builder.append("]");
+		return builder.toString();
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/dto/CompletionRequest.java

@@ -0,0 +1,72 @@
+package de.extio.lmdatasetprep.client.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class CompletionRequest {
+	
+	private String model;
+	
+	private String prompt;
+	
+	private Integer maxTokens;
+	
+	private Double temperature;
+	
+	private Double topP;
+	
+	private Boolean stream;
+	
+	@JsonProperty("model")
+	public String getModel() {
+		return this.model;
+	}
+	
+	public void setModel(final String model) {
+		this.model = model;
+	}
+	
+	@JsonProperty("prompt")
+	public String getPrompt() {
+		return this.prompt;
+	}
+	
+	public void setPrompt(final String prompt) {
+		this.prompt = prompt;
+	}
+	
+	@JsonProperty("max_tokens")
+	public Integer getMaxTokens() {
+		return this.maxTokens;
+	}
+	
+	public void setMaxTokens(final Integer maxTokens) {
+		this.maxTokens = maxTokens;
+	}
+	
+	@JsonProperty("temperature")
+	public Double getTemperature() {
+		return this.temperature;
+	}
+	
+	public void setTemperature(final Double temperature) {
+		this.temperature = temperature;
+	}
+	
+	@JsonProperty("top_p")
+	public Double getTopP() {
+		return this.topP;
+	}
+	
+	public void setTopP(final Double topP) {
+		this.topP = topP;
+	}
+	
+	@JsonProperty("stream")
+	public Boolean isStream() {
+		return this.stream;
+	}
+	
+	public void setStream(final Boolean stream) {
+		this.stream = stream;
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/dto/CompletionResponse.java

@@ -0,0 +1,93 @@
+package de.extio.lmdatasetprep.client.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class CompletionResponse {
+	
+	private String id;
+	
+	private String object;
+	
+	private Integer created;
+	
+	private List<Choice> choices;
+	
+	private String content;
+	
+	private boolean isStoppedEos;
+	
+	@JsonProperty("id")
+	public String getId() {
+		return this.id;
+	}
+	
+	public void setId(final String id) {
+		this.id = id;
+	}
+	
+	@JsonProperty("object")
+	public String getObject() {
+		return this.object;
+	}
+	
+	public void setObject(final String object) {
+		this.object = object;
+	}
+	
+	@JsonProperty("created")
+	public Integer getCreated() {
+		return this.created;
+	}
+	
+	public void setCreated(final Integer created) {
+		this.created = created;
+	}
+	
+	@JsonProperty("choices")
+	public List<Choice> getChoices() {
+		return this.choices;
+	}
+	
+	public void setChoices(final List<Choice> choices) {
+		this.choices = choices;
+	}
+	
+	@JsonProperty("content")
+	public String getContent() {
+		return this.content;
+	}
+	
+	public void setContent(final String content) {
+		this.content = content;
+	}
+	
+	@JsonProperty("stopped_eos")
+	public boolean isStoppedEos() {
+		return this.isStoppedEos;
+	}
+	
+	public void setStoppedEos(final boolean isStoppedEos) {
+		this.isStoppedEos = isStoppedEos;
+	}
+	
+	@Override
+	public String toString() {
+		final StringBuilder builder = new StringBuilder();
+		builder.append("CompletionResponse [id=");
+		builder.append(this.id);
+		builder.append(", object=");
+		builder.append(this.object);
+		builder.append(", created=");
+		builder.append(this.created);
+		builder.append(", choices=");
+		builder.append(this.choices);
+		builder.append(", content=");
+		builder.append(this.content);
+		builder.append(", isStoppedEos=");
+		builder.append(this.isStoppedEos);
+		builder.append("]");
+		return builder.toString();
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/dto/Model.java

@@ -0,0 +1,74 @@
+package de.extio.lmdatasetprep.client.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Model {
+	
+	private String id;
+	
+	private String object;
+	
+	private int created;
+	
+	private String ownedBy;
+	
+	private List<String> permission;
+	
+	private String root;
+	
+	@JsonProperty("id")
+	public String getId() {
+		return this.id;
+	}
+	
+	public void setId(final String id) {
+		this.id = id;
+	}
+	
+	@JsonProperty("object")
+	public String getObject() {
+		return this.object;
+	}
+	
+	public void setObject(final String object) {
+		this.object = object;
+	}
+	
+	@JsonProperty("created")
+	public int getCreated() {
+		return this.created;
+	}
+	
+	public void setCreated(final int created) {
+		this.created = created;
+	}
+	
+	@JsonProperty("owned_by")
+	public String getOwned_by() {
+		return this.ownedBy;
+	}
+	
+	public void setOwned_by(final String owned_by) {
+		this.ownedBy = owned_by;
+	}
+	
+	@JsonProperty("permission")
+	public List<String> getPermission() {
+		return this.permission;
+	}
+	
+	public void setPermission(final List<String> permission) {
+		this.permission = permission;
+	}
+	
+	@JsonProperty("root")
+	public String getRoot() {
+		return this.root;
+	}
+	
+	public void setRoot(final String root) {
+		this.root = root;
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/dto/ModelsResponse.java

@@ -0,0 +1,42 @@
+package de.extio.lmdatasetprep.client.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class ModelsResponse {
+	
+	private String object;
+	
+	private List<Model> data;
+	
+	@JsonProperty("object")
+	public String getObject() {
+		return this.object;
+	}
+	
+	public void setObject(final String object) {
+		this.object = object;
+	}
+	
+	@JsonProperty("data")
+	public List<Model> getData() {
+		return this.data;
+	}
+	
+	public void setData(final List<Model> data) {
+		this.data = data;
+	}
+	
+	@Override
+	public String toString() {
+		final StringBuilder builder = new StringBuilder();
+		builder.append("ModelsResponse [object=");
+		builder.append(this.object);
+		builder.append(", data=");
+		builder.append(this.data);
+		builder.append("]");
+		return builder.toString();
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/profile/ModelCategory.java

@@ -0,0 +1,16 @@
+package de.extio.lmdatasetprep.client.profile;
+
+public enum ModelCategory {
+	HOT("profile.hot"),
+	COLD("profile.cold");
+	
+	private final String modelProfile;
+	
+	ModelCategory(final String modelProfile) {
+		this.modelProfile = modelProfile;
+	}
+	
+	public String getModelProfile() {
+		return this.modelProfile;
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/profile/ModelProfile.java

@@ -0,0 +1,11 @@
+package de.extio.lmdatasetprep.client.profile;
+
+public record ModelProfile(
+		String prompt,
+		int maxTokens,
+		int maxContextLength,
+		double temperature,
+		double topP,
+		int maxContinuations,
+		String baseUrl) {
+}

src/main/java/de/extio/lmdatasetprep/client/profile/ModelProfileService.java

@@ -0,0 +1,95 @@
+package de.extio.lmdatasetprep.client.profile;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.bind.Bindable;
+import org.springframework.boot.context.properties.bind.Binder;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.core.env.Environment;
+import org.springframework.core.io.support.ResourcePropertySource;
+import org.springframework.stereotype.Service;
+
+@Service
+public class ModelProfileService {
+	
+	private final Environment environment;
+	
+	private static final String MODEL_PROFILE_PREFIX = "profile";
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class);
+	
+	public ModelProfileService(final Environment environment) {
+		this.environment = environment;
+	}
+	
+	@Cacheable("modelProfiles")
+	public ModelProfile getModelProfile(final String modelProfile) {
+		final var modelName = this.environment.getProperty(modelProfile);
+		if (modelName == null) {
+			throw new IllegalStateException("No model found for profile: " + modelProfile);
+		}
+		try {
+			final var resource = new ResourcePropertySource("classpath:" + modelName + ".properties");
+			final var name = Optional.ofNullable(resource.getProperty("prompts"))
+					.map(Object::toString)
+					.orElseThrow(() -> new IllegalStateException("No model name found for model: " + modelName));
+			final var maxTokens = Optional.ofNullable(resource.getProperty("maxTokens"))
+					.map(Object::toString)
+					.map(Integer::parseInt)
+					.orElseThrow(() -> new IllegalStateException("No model maxTokens found for model: " + modelName));
+			final var maxContextLength = Optional.ofNullable(resource.getProperty("maxContextLength"))
+					.map(Object::toString)
+					.map(Integer::parseInt)
+					.orElseThrow(() -> new IllegalStateException("No model maxContextLength found for model: " + modelName));
+			final var temperature = Optional.ofNullable(resource.getProperty("temperature"))
+					.map(Object::toString)
+					.map(Double::parseDouble)
+					.orElseThrow(() -> new IllegalStateException("No model temperature found for model: " + modelName));
+			final var topP = Optional.ofNullable(resource.getProperty("topP"))
+					.map(Object::toString)
+					.map(Double::parseDouble)
+					.orElseThrow(() -> new IllegalStateException("No model topP found for model: " + modelName));
+			final var maxContinuations = Optional.ofNullable(resource.getProperty("maxContinuations"))
+					.map(Object::toString)
+					.map(Integer::parseInt)
+					.orElseThrow(() -> new IllegalStateException("No model maxContinuations found for model: " + modelName));
+			final var baseUrl = Optional.ofNullable(resource.getProperty("baseUrl"))
+					.map(Object::toString)
+					.orElseThrow(() -> new IllegalStateException("No model baseUrl found for model: " + modelName));
+			
+			return new ModelProfile(name, maxTokens, maxContextLength, temperature, topP, maxContinuations, baseUrl);
+		}
+		catch (final IOException e) {
+			LOGGER.error("Error while reading model profile", e);
+			return null;
+		}
+	}
+	
+	public List<String> getModelProfileUrls() {
+		final var modelProfiles = Binder.get(this.environment)
+				.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
+				.orElse(Collections.emptyMap());
+		
+		return modelProfiles.values().stream()
+				.filter(model -> model != null && !model.isEmpty())
+				.map(model -> {
+					try {
+						final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
+						return resource.getProperty("baseUrl") != null ? resource.getProperty("baseUrl").toString() : null;
+					}
+					catch (final IOException e) {
+						LOGGER.error("Error while reading model profile for: " + model, e);
+						return null;
+					}
+				})
+				.filter(Objects::nonNull)
+				.collect(Collectors.toList());
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/ChatMLPromptStrategy.java

@@ -0,0 +1,45 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class ChatMLPromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String instruction, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<|im_start|>system\n");
+		prompt.append(instruction);
+		prompt.append("<|im_end|>\n<|im_start|>user\n");
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append("<|im_end|>\n<|im_start|>assistant\n");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(assistant);
+		prompt.append("<|im_end|>\n<|im_start|>user\n");
+		prompt.append(user);
+		prompt.append("<|im_end|>\n<|im_start|>assistant\n");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<|im_end|>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "chatml";
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/Gemma2PromptStrategy.java

@@ -0,0 +1,48 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class Gemma2PromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String instruction, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<start_of_turn>user\n");
+		if (!instruction.isEmpty()) {
+			prompt.append(instruction);
+			prompt.append("\n");
+		}
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append("<end_of_turn>\n<start_of_turn>model\n");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(assistant);
+		prompt.append("<end_of_turn>\n<start_of_turn>user\n");
+		prompt.append(user);
+		prompt.append("<end_of_turn>\n<start_of_turn>model\n");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<end_of_turn>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "gemma2";
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/Llama3PromptStrategy.java

@@ -0,0 +1,47 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+/**
+ * @see https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
+ */
+@Component
+public class Llama3PromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String instruction, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n");
+		prompt.append(instruction);
+		prompt.append("<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n");
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n\n");
+		}
+		prompt.append(text);
+		prompt.append("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
+		
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(assistant);
+		prompt.append("<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n" + user + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<|eot_id|>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "llama3";
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/MixtralPromptStrategy.java

@@ -0,0 +1,40 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class MixtralPromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String instruction, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<s>[INST] ");
+		prompt.append(instruction);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append(" ");
+		}
+		prompt.append(question);
+		prompt.append(" [/INST]");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append("\n\"");
+		prompt.append(assistant);
+		prompt.append("\"</s> [INST] ");
+		prompt.append(user);
+		prompt.append(" [/INST]");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "mixtral";
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/Phi3PromptStrategy.java

@@ -0,0 +1,46 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class Phi3PromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String instruction, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<|user|>\n");
+		prompt.append(instruction);
+		prompt.append(" ");
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append(" <|end|>\n<|assistant|>");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append("\n");
+		prompt.append(assistant);
+		prompt.append("<|user|>\n");
+		prompt.append(user);
+		prompt.append("<|end|>\n<|assistant|>");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<|end|>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "phi3";
+	}
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/PromptStrategy.java

@@ -0,0 +1,17 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+public interface PromptStrategy {
+	
+	String getPromptName();
+	
+	StringBuilder start(String instruction, String question, String text);
+	
+	void continue_(StringBuilder prompt, String assistant);
+	
+	void next(StringBuilder prompt, String assistant, String user);
+	
+	default String removeEOT(final String prompt) {
+		return prompt.strip();
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/PromptStrategyFactory.java

@@ -0,0 +1,22 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class PromptStrategyFactory {
+	
+	private final Map<String, PromptStrategy> strategies = new HashMap<>();
+	
+	public PromptStrategyFactory(final List<PromptStrategy> strategies) {
+		strategies.forEach(strategy -> this.strategies.put(strategy.getPromptName(), strategy));
+	}
+	
+	public PromptStrategy getStrategy(final String modelPrompt) {
+		return this.strategies.get(modelPrompt);
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/client/prompt/VicunaPromptStrategy.java

@@ -0,0 +1,46 @@
+package de.extio.lmdatasetprep.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class VicunaPromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String instruction, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append(instruction);
+		prompt.append(" USER: ");
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append(" ASSISTANT:");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(" ");
+		prompt.append(assistant);
+		prompt.append("</s>USER: ");
+		prompt.append(user);
+		prompt.append(" ASSISTANT:");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("</s>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "vicuna";
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/preparer/Rewrite.java

@@ -0,0 +1,112 @@
+package de.extio.lmdatasetprep.preparer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import de.extio.lmdatasetprep.client.Client;
+import de.extio.lmdatasetprep.client.profile.ModelCategory;
+
+@Component
+public class Rewrite implements Consumer<String[]> {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(Rewrite.class);
+	
+	private static final ThreadLocal<Random> RANDOM = ThreadLocal.withInitial(XorShift128Random::new);
+	
+	private static String IMPROVEMENT_PROMPT = "Improve the following paragraph of an existing text without changing the existing sentences, improve the text in a nuanced and understated way. Don't include a preamble and no explanation:";
+	
+	private static String ENHANCE_PROMPT = "Improve the following paragraph of an existing text. %s without changing the existing sentences, focusing on enriching the description of the setting in a nuanced and understated way. Don't include a preamble and no explanation:";
+	
+	private static List<String> ENHANCEMENTS = List.of(
+			"Infuse this paragraph with evocative sensory details, focusing on the texture of objects and the atmosphere of the setting",
+			"Add subtle hints of the character's inner thoughts and emotions within the existing dialogue and actions",
+			"Enhance the paragraph with figurative language like metaphors or similes to create a richer tapestry of imagery",
+			"Develop the subtext of the paragraph by suggesting unspoken tensions or hidden motivations between characters",
+			"Introduce a faint scent or sound that adds another layer of sensory detail to the scene without disrupting the flow",
+			"Imbue the paragraph with a sense of foreboding or anticipation by subtly hinting at future events without revealing specifics",
+			"Weave in a historical or cultural reference relevant to the setting or characters to add depth and context",
+			"Elevate the language of the paragraph by replacing basic verbs and adjectives with more evocative synonyms",
+			"Add a touch of humor or irony to the paragraph through clever wordplay or unexpected observations",
+			"Imagine this paragraph as a still photograph. Describe the composition and lighting to further bring the scene to life");
+	
+	@Autowired
+	private Client client;
+	
+	@Override
+	public void accept(final String[] args) {
+		if (args.length < 5) {
+			LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <Texts per input> <Model suffix>");
+			return;
+		}
+		
+		Utils.transformDirectory(args[1],
+				f -> {
+					final List<Runnable> tasks = new ArrayList<>();
+					tasks.add(() -> this.rewriteFile(args, f, 0, "impr", ModelCategory.COLD, IMPROVEMENT_PROMPT));
+					for (final ModelCategory modelCategory : List.of(ModelCategory.COLD, ModelCategory.HOT)) {
+						for (int i = 0; i < Integer.parseInt(args[3]); i++) {
+							final int fi = i;
+							tasks.add(() -> this.rewriteFile(args, f, fi, "enh", modelCategory, String.format(ENHANCE_PROMPT, ENHANCEMENTS.get(RANDOM.get().nextInt(ENHANCEMENTS.size())))));
+						}
+					}
+					return tasks;
+				});
+	}
+	
+	private void rewriteFile(final String[] args, final Path f, final int i, final String identifier, final ModelCategory modelCategory, final String prompt) {
+		final Path out = Utils.suffixFilename(Path.of(args[2]).resolve(f.getFileName()),
+				"rewr",
+				args[4],
+				identifier,
+				modelCategory.toString().toLowerCase(),
+				String.valueOf(i));
+		if (Files.exists(out)) {
+			LOGGER.info("Skipping " + out);
+			return;
+		}
+		
+		LOGGER.info("Rewriting " + f.getFileName() + " " + identifier + " " + modelCategory.name() + " " + i);
+		
+		final List<String> paragraphs = this.rewrite(f, prompt, modelCategory);
+		try {
+			Files.writeString(out, String.join("\n\n", paragraphs));
+		}
+		catch (final IOException e) {
+			LOGGER.error("Cannot store file", e);
+		}
+	}
+	
+	List<String> rewrite(final Path file, final String prompt, final ModelCategory modelCategory) {
+		String text;
+		try {
+			text = Utils.normalizeText(Files.readString(file));
+		}
+		catch (final IOException e) {
+			throw new RuntimeException("Cannot read file", e);
+		}
+		
+		final List<String> splits = Utils.splitParagraphs(text, Utils.PARAGRAPH_CHUNKS_NORM, Utils.PARAGRAPH_CHUNKS_VAR);
+		
+		final List<String> paragraphs = new ArrayList<>(splits.size());
+		for (final String split : splits) {
+			final var completion = this.client.completion("You are a helpful assistant with great authoring skills.",
+					prompt,
+					split,
+					modelCategory);
+			paragraphs.add(completion.response());
+		}
+		
+		return paragraphs;
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/preparer/Text2Jsonl.java

@@ -0,0 +1,78 @@
+package de.extio.lmdatasetprep.preparer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+@Component
+public class Text2Jsonl implements Consumer<String[]> {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(Text2Jsonl.class);
+	
+	@Override
+	public void accept(final String[] args) {
+		if (args.length < 5) {
+			LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir> <chunk norm> <chunk var>");
+			return;
+		}
+		
+		final List<String> lines = Collections.synchronizedList(new ArrayList<>());
+		
+		Utils.transformDirectory(args[1], p -> {
+			return List.of(() -> {
+				lines.addAll(this.splitToJsonl(p, Integer.parseInt(args[3]), Integer.parseInt(args[4])));
+			});
+		});
+		
+		final Path out = Path.of(args[2], "dataset.jsonl");
+		try {
+			Files.writeString(out, String.join("\n", lines));
+		}
+		catch (final IOException e) {
+			LOGGER.error("Cannot store dataset", e);
+		}
+	}
+	
+	List<String> splitToJsonl(final Path file, final int chunkNorm, final int chunkVar) {
+		LOGGER.info("Splitting to jsonl " + file);
+		
+		String text;
+		try {
+			text = Utils.normalizeText(Files.readString(file));
+		}
+		catch (final IOException e) {
+			throw new RuntimeException("Cannot read file", e);
+		}
+		
+		final ObjectMapper mapper = new ObjectMapper();
+		
+		return Utils
+				.splitParagraphs(text, chunkNorm, chunkVar)
+				.stream()
+				.map(split -> {
+					try {
+						return mapper.writeValueAsString(new TextLine(split));
+					}
+					catch (final JsonProcessingException e) {
+						LOGGER.error("Cannot convert split to json", e);
+						return "";
+					}
+				})
+				.toList();
+	}
+	
+	record TextLine(String text) {
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/preparer/Translate.java

@@ -0,0 +1,77 @@
+package de.extio.lmdatasetprep.preparer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import de.extio.lmdatasetprep.client.Client;
+import de.extio.lmdatasetprep.client.profile.ModelCategory;
+
+@Component
+public class Translate implements Consumer<String[]> {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(Translate.class);
+	
+	@Autowired
+	private Client client;
+	
+	@Override
+	public void accept(final String[] args) {
+		if (args.length < 3) {
+			LOGGER.error("Arguments are missing. <Path to text files> <Path to output dir>");
+			return;
+		}
+		
+		Utils.transformDirectory(args[1], p -> {
+			return List.of(() -> {
+				try {
+					Path out = Utils.suffixFilename(Path.of(args[2]).resolve(p.getFileName()), "_en");
+					if (Files.exists(out)) {
+						LOGGER.info("Skipping " + out);
+					}
+					else {
+						final var translations = this.translate(p);
+						Files.writeString(out, String.join("\n\n", translations));
+					}
+				}
+				catch (final IOException e) {
+					LOGGER.error("Cannot store file", e);
+				}
+			});
+		});
+	}
+	
+	List<String> translate(final Path file) {
+		LOGGER.info("Translating " + file);
+		
+		String text;
+		try {
+			text = Utils.normalizeText(Files.readString(file));
+		}
+		catch (final IOException e) {
+			throw new RuntimeException("Cannot read file", e);
+		}
+		
+		final List<String> splits = Utils.splitParagraphs(text, Utils.PARAGRAPH_CHUNKS_NORM, Utils.PARAGRAPH_CHUNKS_VAR);
+		
+		final List<String> translations = new ArrayList<>(splits.size());
+		for (final String split : splits) {
+			final var completion = this.client.completion("You are a helpful assistant with great language translation and authoring skills.",
+					"Translate the following text to English. Don't include a preamble and no explanation:",
+					split,
+					ModelCategory.COLD);
+			translations.add(completion.response());
+		}
+		
+		return translations;
+	}
+	
+}

src/main/java/de/extio/lmdatasetprep/preparer/Utils.java

@@ -0,0 +1,139 @@
+package de.extio.lmdatasetprep.preparer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Utils {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
+	
+	static final int PARAGRAPH_CHUNKS_NORM = 1250;
+	
+	static final int PARAGRAPH_CHUNKS_VAR = 350;
+	
+	static String normalizeText(final String text) {
+		String result = text;
+		
+		// Remove excessive newlines, spaces and tabs
+		int cnt = 0;
+		do {
+			cnt = result.length();
+			result = result.replace("\r", "\n");
+			result = result.replace("\n\n\n", "\n\n");
+			result = result.replace("\t", "");
+			result = result.replace("  ", " ");
+			result = result.replace(" .", ".");
+			result = result.replace(".  ", ". ");
+			result = result.replace("\n.", ".");
+			result = result.replace(" !", "!");
+			result = result.replace("!  ", "! ");
+			result = result.replace("\n!", "!");
+			result = result.replace(" ?", "?");
+			result = result.replace("?  ", "? ");
+			result = result.replace("\n?", "?");
+			result = result.replace(" \n", "\n");
+		} while (result.length() != cnt);
+		
+		// Remove newlines mid of the sentence
+		final StringBuilder sb = new StringBuilder(result);
+		for (int i = 1; i < sb.length(); i++) {
+			final char c = sb.charAt(i);
+			if (c == '\n' && sb.charAt(i - 1) != '.' && sb.charAt(i - 1) != '?' && sb.charAt(i - 1) != '!' && sb.charAt(i - 1) != '\n') {
+				sb.setCharAt(i, ' ');
+			}
+		}
+		result = sb.toString();
+		
+		LOGGER.info("Normalizer stats: Before: {} After {}", text.length(), result.length());
+		
+		return result;
+	}
+	
+	static List<String> splitParagraphs(final String text, final int chunks_norm, final int chunks_var) {
+		final int CHUNKS_MIN = chunks_norm - chunks_var;
+		final int CHUNKS_MAX = chunks_norm + chunks_var;
+		
+		if (text.length() <= chunks_norm + chunks_var) {
+			return Collections.singletonList(text);
+		}
+		
+		final List<String> splits = new ArrayList<>();
+		
+		int pos = 0;
+		paragraphs: do {
+			if (text.length() - pos > chunks_norm + chunks_var) {
+				delimiters: for (final String delimiter : List.of("\n\n", "\n", ".")) {
+					int next = text.indexOf(delimiter, pos);
+					while (next > -1) {
+						if (next < pos + CHUNKS_MIN) {
+							next = text.indexOf(delimiter, next + 1);
+						}
+						else if (next > pos + CHUNKS_MAX) {
+							continue delimiters;
+						}
+						else {
+							String split = text.substring(pos, next).trim();
+							if (!split.endsWith(".")) {
+								split = split + ".";
+							}
+							splits.add(split);
+							pos = next + delimiter.length();
+							continue paragraphs;
+						}
+					}
+				}
+			}
+			
+			final int next = Math.min(pos + chunks_norm, text.length());
+			final String split = text.substring(pos, next).trim();
+			splits.add(split);
+			pos = next + 1;
+		} while (pos < text.length());
+		
+		LOGGER.info("Created {} splits", splits.size());
+		
+		return splits;
+	}
+	
+	static void transformDirectory(final String path, final Function<Path, List<Runnable>> createTasks) {
+		try (final ExecutorService executor = Executors.newFixedThreadPool(4);
+				Stream<Path> stream = Files.list(Paths.get(path))) {
+			final List<CompletableFuture<?>> tasks = new ArrayList<>();
+			stream
+					.filter(p -> Files.isRegularFile(p))
+					.forEach(p -> {
+						for (final Runnable r : createTasks.apply(p)) {
+							tasks.add(CompletableFuture.runAsync(r, executor));
+						}
+					});
+			CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).join();
+		}
+		catch (final IOException e) {
+			LOGGER.error("Error", e);
+		}
+	}
+	
+	static Path suffixFilename(final Path f, final String... suffixes) {
+		final String[] parts = f.getFileName().toString().split("[.]");
+		final StringBuilder sb = new StringBuilder();
+		for (final String suffix : suffixes) {
+			sb.append('_');
+			sb.append(suffix);
+		}
+		sb.append('.');
+		return f.getParent().resolve(parts[0] + sb.toString() + parts[1]);
+	}
+}

src/main/java/de/extio/lmdatasetprep/preparer/XorShift128Random.java

@@ -0,0 +1,62 @@
+package de.extio.lmdatasetprep.preparer;
+
+import java.util.Random;
+import java.util.UUID;
+
+@SuppressWarnings("serial")
+public final class XorShift128Random extends Random {
+	
+	private long state0;
+	
+	private long state1;
+	
+	public XorShift128Random() {
+		super();
+	}
+	
+	public XorShift128Random(final long seed) {
+		this.setSeed(seed);
+	}
+	
+	public XorShift128Random(final String state) {
+		final UUID uuid = UUID.fromString(state);
+		this.state0 = uuid.getMostSignificantBits();
+		this.state1 = uuid.getLeastSignificantBits();
+	}
+	
+	@Override
+	public synchronized void setSeed(final long seed) {
+		this.state0 = avalanche(seed == 0 ? -1 : seed);
+		this.state1 = avalanche(this.state0);
+		this.state0 = avalanche(this.state1);
+	}
+	
+	@Override
+	protected int next(final int bits) {
+		return (int) (this.nextLong() & (1L << bits) - 1);
+	}
+	
+	@Override
+	public long nextLong() {
+		long s1 = this.state0;
+		final long s0 = this.state1;
+		this.state0 = s0;
+		s1 ^= s1 << 23;
+		return (this.state1 = s1 ^ s0 ^ (s1 >>> 17) ^ (s0 >>> 26)) + s0;
+	}
+	
+	@Override
+	public String toString() {
+		return new UUID(this.state0, this.state1).toString();
+	}
+	
+	private static long avalanche(long k) {
+		k ^= k >>> 33;
+		k *= 0xff51afd7ed558ccdL;
+		k ^= k >>> 33;
+		k *= 0xc4ceb9fe1a85ec53L;
+		k ^= k >>> 33;
+		
+		return k;
+	}
+}