lm-lib first commit;

git-svn-id: svn://home0/lm-lib@3603 71dcc7fd-b1f2-4e5d-bb8e-274146e81173

src/main/java/de/extio/lmlib/client/Client.java

@@ -0,0 +1,13 @@
+package de.extio.lmlib.client;
+
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+
+public interface Client {
+	
+	Completion completion(ModelCategory modelCategory, String system, String question, String fullText);
+	
+	Completion conversation(ModelCategory modelCategory, Conversation conversation);
+	
+	ModelProvider getModelProvider();
+}

src/main/java/de/extio/lmlib/client/ClientService.java

@@ -0,0 +1,28 @@
+package de.extio.lmlib.client;
+
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfileService;
+
+@Service
+public class ClientService {
+	
+	@Autowired
+	private List<Client> clients;
+	
+	@Autowired
+	private ModelProfileService modelProfileService;
+	
+	public Client getClient(final ModelCategory category) {
+		final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile());
+		return this.clients.stream()
+				.filter(client -> client.getModelProvider() == modelProfile.modelProvider())
+				.findFirst()
+				.orElseThrow(() -> new IllegalStateException("No client found for model profile " + category));
+	}
+	
+}

src/main/java/de/extio/lmlib/client/Completion.java

@@ -0,0 +1,7 @@
+package de.extio.lmlib.client;
+
+import java.time.Duration;
+
+public final record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, boolean cached) {
+	
+}

src/main/java/de/extio/lmlib/client/Conversation.java

@@ -0,0 +1,78 @@
+package de.extio.lmlib.client;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public final class Conversation {
+	
+	public static Conversation create(final String system, final String user) {
+		final Conversation conversation = new Conversation();
+		if (system != null && !system.isEmpty()) {
+			conversation.addTurn(new Turn(TurnType.SYSTEM, system));
+		}
+		if (user != null && !user.isEmpty()) {
+			conversation.addTurn(new Turn(TurnType.USER, user));
+		}
+		return conversation;
+	}
+	
+	public static Conversation create(final String user) {
+		final Conversation conversation = new Conversation();
+		if (user != null && !user.isEmpty()) {
+			conversation.addTurn(new Turn(TurnType.USER, user));
+		}
+		return conversation;
+	}
+	
+	public static Conversation copy(final Conversation conversation) {
+		final Conversation copy = new Conversation();
+		if (conversation != null) {
+			synchronized (conversation.conversation) {
+				copy.conversation.addAll(conversation.conversation);
+			}
+		}
+		return copy;
+	}
+	
+	private final List<Turn> conversation = Collections.synchronizedList(new ArrayList<>());
+	
+	private Conversation() {
+	}
+	
+	public void addTurn(final Turn turn) {
+		this.conversation.add(turn);
+	}
+	
+	public void replaceTurn(final Turn turn) {
+		synchronized (this.conversation) {
+			if (!this.conversation.isEmpty()) {
+				this.conversation.set(this.conversation.size() - 1, turn);
+			}
+			else {
+				this.addTurn(turn);
+			}
+		}
+	}
+	
+	public List<Turn> getConversation() {
+		return Collections.unmodifiableList(this.conversation);
+	}
+	
+	@Override
+	public String toString() {
+		final StringBuilder builder = new StringBuilder();
+		builder.append("Conversation [conversation=");
+		builder.append(this.conversation);
+		builder.append("]");
+		return builder.toString();
+	}
+	
+	public enum TurnType {
+		SYSTEM, USER, ASSISTANT
+	}
+	
+	public record Turn(TurnType type, String text) {
+	}
+	
+}

src/main/java/de/extio/lmlib/client/WebClientConfiguration.java

@@ -0,0 +1,40 @@
+package de.extio.lmlib.client;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import io.netty.channel.ChannelOption;
+import io.netty.handler.timeout.ReadTimeoutHandler;
+import io.netty.handler.timeout.WriteTimeoutHandler;
+import reactor.netty.http.client.HttpClient;
+
+@Configuration
+public class WebClientConfiguration {
+	
+	private static final int TIMEOUT = 900000;
+	
+	private static final int CONNECT_TIMEOUT = 10000;
+	
+	@Bean
+	@Qualifier("lmLibWebClientBuilder")
+	WebClient.Builder lmLibWebClientBuilder() {
+		final HttpClient httpClient = HttpClient.create()
+				.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
+				.responseTimeout(Duration.ofMillis(TIMEOUT))
+				.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
+		
+		return WebClient.builder()
+				.clientConnector(new ReactorClientHttpConnector(httpClient))
+				.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+				.defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
+	}
+	
+}

src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java

@@ -0,0 +1,258 @@
+package de.extio.lmlib.client.azureai;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.azure.ai.inference.ChatCompletionsClient;
+import com.azure.ai.inference.ChatCompletionsClientBuilder;
+import com.azure.ai.inference.models.ChatCompletions;
+import com.azure.ai.inference.models.ChatCompletionsOptions;
+import com.azure.ai.inference.models.ChatRequestAssistantMessage;
+import com.azure.ai.inference.models.ChatRequestMessage;
+import com.azure.ai.inference.models.ChatRequestSystemMessage;
+import com.azure.ai.inference.models.ChatRequestUserMessage;
+import com.azure.ai.inference.models.CompletionsFinishReason;
+import com.azure.core.credential.AzureKeyCredential;
+import com.azure.core.http.policy.ExponentialBackoffOptions;
+import com.azure.core.http.policy.RetryOptions;
+
+import de.extio.lmlib.client.Client;
+import de.extio.lmlib.client.Completion;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfile;
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+import de.extio.lmlib.client.profile.ModelProfileService;
+import de.extio.lmlib.client.token.Tokenizer;
+
+public final class AzureAiClient implements Client {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClient.class);
+	
+	private final ConcurrentMap<ModelCategory, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
+	
+	private final ModelProfileService modelProfileService;
+	
+	private final Tokenizer tokenizer;
+	
+	AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
+		this.modelProfileService = modelProfileService;
+		this.tokenizer = tokenizer;
+	}
+	
+	@Override
+	public ModelProvider getModelProvider() {
+		return ModelProvider.AZURE_AI;
+	}
+	
+	@Override
+	public Completion completion(final ModelCategory modelCategory, final String system, final String question, final String fullText) {
+		String text = "";
+		if (!question.isEmpty() && !fullText.isEmpty()) {
+			text = question + "\n" + fullText;
+		}
+		else if (!question.isEmpty()) {
+			text = question;
+		}
+		else if (!fullText.isEmpty()) {
+			text = fullText;
+		}
+		else {
+			throw new IllegalArgumentException("Either question or fullText must be set");
+		}
+		
+		return this.conversation(modelCategory, Conversation.create(system, text));
+	}
+	
+	@Override
+	public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
+		if (modelCategory == null) {
+			modelCategory = ModelCategory.MEDIUM;
+		}
+		final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
+		if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
+			throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
+		}
+		
+		final List<List<ChatRequestMessage>> chats = this.createChats(conversation, modelProfile);
+		
+		final List<StringBuilder> answers = new ArrayList<>();
+		final CompletionStatistics statistics = new CompletionStatistics();
+		for (final List<ChatRequestMessage> chat : chats) {
+			final StringBuilder answer = new StringBuilder();
+			this.requestCompletionsContinuations(chat, answer, statistics, modelCategory, modelProfile);
+			answers.add(answer);
+		}
+		
+		// TODO: Reactivate when multiple (split) prompts have been implemented again
+		//		if (answers.size() > 1) {
+		//			this.summarize(answers, modelProfile, promptStrategy, statistics);
+		//		}
+		
+		return new Completion(
+				answers.stream().collect(Collectors.joining()),
+				statistics.requests,
+				statistics.duration,
+				statistics.inTokens,
+				statistics.outTokens,
+				false);
+	}
+	
+	private List<List<ChatRequestMessage>> createChats(final Conversation conversation, final ModelProfile modelProfile) {
+		final List<ChatRequestMessage> chat = new ArrayList<>();
+		
+		final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
+		Turn systemTurn = null;
+		Turn userTurn = null;
+		Turn assistantTurn = null;
+		boolean first = true;
+		
+		final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 50; // 50 is a margin for EOT and other special tokens
+		final AtomicInteger tokens = new AtomicInteger();
+		
+		while (!turns.isEmpty()) {
+			final Turn turn = turns.pop();
+			switch (turn.type()) {
+				case SYSTEM:
+					systemTurn = turn;
+					break;
+				case USER:
+					userTurn = turn;
+					break;
+				case ASSISTANT:
+					assistantTurn = turn;
+					break;
+				default:
+					throw new IllegalArgumentException("Unknown turn type: " + turn.type());
+			}
+			
+			if (first && userTurn != null) {
+				if (systemTurn != null) {
+					chat.add(new ChatRequestSystemMessage(systemTurn.text()));
+				}
+				if (this.addChatRequestMessage(userTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
+					break;
+				}
+				userTurn = null;
+				first = false;
+			}
+			else if (!first && assistantTurn != null && userTurn != null) {
+				if (this.addChatRequestMessage(assistantTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestAssistantMessage::new)) {
+					break;
+				}
+				if (this.addChatRequestMessage(userTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
+					break;
+				}
+				userTurn = null;
+				assistantTurn = null;
+			}
+		}
+		
+		return List.of(chat);
+	}
+	
+	private boolean addChatRequestMessage(final String text, final int maxLength, final AtomicInteger tokens, final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Function<String, ChatRequestMessage> messageFactory) {
+		final int textTokens = this.tokenizer.count(text, modelProfile);
+		
+		String trimmedText;
+		boolean trimmed = false;
+		if (tokens.addAndGet(textTokens) > maxLength) {
+			// TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
+			LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokens.get(), maxLength);
+			trimmedText = text.substring(0, text.length() - (tokens.get() - maxLength) / 3);
+			trimmed = true;
+		}
+		else {
+			trimmedText = text;
+		}
+		
+		chat.add(messageFactory.apply(trimmedText));
+		
+		return trimmed;
+	}
+	
+	private void requestCompletionsContinuations(final List<ChatRequestMessage> chat, final StringBuilder answer, final CompletionStatistics statistics, final ModelCategory modelCategory, final ModelProfile modelProfile) {
+		for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
+			final LocalDateTime start = LocalDateTime.now();
+			
+			final var response = this.requestCompletions(chat, modelCategory, modelProfile);
+			
+			final var choice = response.getChoice();
+			
+			statistics.add(Duration.between(start, LocalDateTime.now()), response.getUsage().getPromptTokens(), response.getUsage().getCompletionTokens());
+			
+			answer.append(choice.getMessage().getContent());
+			
+			final var complete = !(choice.getFinishReason() == CompletionsFinishReason.TOKEN_LIMIT_REACHED);
+			if (complete) {
+				LOGGER.debug("Prompt response complete");
+				break;
+			}
+			
+			answer.append('\n');
+			chat.add(new ChatRequestAssistantMessage(choice.getMessage().getContent()));
+			chat.add(new ChatRequestUserMessage("continue"));
+		}
+	}
+	
+	private ChatCompletions requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
+		LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
+		
+		final var client = this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
+			final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
+					.setMaxRetries(99)
+					.setBaseDelay(Duration.ofSeconds(1))
+					.setMaxDelay(Duration.ofSeconds(10));
+			final RetryOptions retryOptions = new RetryOptions(exponentialOptions);
+			
+			final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
+					.credential(new AzureKeyCredential(modelProfile.apiKey()))
+					.endpoint(modelProfile.url())
+					.retryOptions(retryOptions)
+					.buildClient();
+			return chatCompletionsClient;
+		});
+		
+		final var options = new ChatCompletionsOptions(chat);
+		options.setModel(modelProfile.modelName());
+		options.setMaxTokens(modelProfile.maxTokens());
+		options.setTemperature(modelProfile.temperature());
+		options.setTopP(modelProfile.topP());
+		
+		return client.complete(options);
+	}
+	
+	static class CompletionStatistics {
+		
+		int requests;
+		
+		Duration duration = Duration.ofMillis(0l);
+		
+		long inTokens;
+		
+		long outTokens;
+		
+		void add(final Duration duration, final int inTokens, final int outTokens) {
+			LOGGER.info("Request duration: {}; in: {}; out: {}; tps: {}", duration, inTokens, outTokens, String.format("%.2f", (inTokens + outTokens) / (double) duration.toMillis() * 1000.0));
+			
+			this.requests++;
+			this.duration = this.duration.plus(duration);
+			this.inTokens += inTokens;
+			this.outTokens += outTokens;
+		}
+	}
+	
+}

src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java

@@ -0,0 +1,19 @@
+package de.extio.lmlib.client.azureai;
+
+import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.context.annotation.Bean;
+
+import de.extio.lmlib.client.profile.ModelProfileService;
+import de.extio.lmlib.client.token.Tokenizer;
+
+@AutoConfiguration
+@ConditionalOnClass(name = "com.azure.ai.inference.ChatCompletionsClient")
+public class AzureAiClientAutoConfiguration {
+	
+	@Bean
+	AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
+		return new AzureAiClient(modelProfileService, tokenizer);
+	}
+	
+}

src/main/java/de/extio/lmlib/client/profile/ModelCategory.java

@@ -0,0 +1,27 @@
+package de.extio.lmlib.client.profile;
+
+public enum ModelCategory {
+	
+	SMALL("profile.model.s", "S"),
+	MEDIUM("profile.model.m", "M"),
+	LARGE("profile.model.l", "L"),
+	HOT("profile.model.hot", "HOT"),
+	COLD("profile.model.cold", "COLD");
+	
+	private final String modelProfile;
+	
+	private final String shortName;
+	
+	ModelCategory(final String modelProfile, final String shortName) {
+		this.modelProfile = modelProfile;
+		this.shortName = shortName;
+	}
+	
+	public String getModelProfile() {
+		return this.modelProfile;
+	}
+	
+	public String getShortName() {
+		return this.shortName;
+	}
+}

src/main/java/de/extio/lmlib/client/profile/ModelProfile.java

@@ -0,0 +1,19 @@
+package de.extio.lmlib.client.profile;
+
+public record ModelProfile(
+		String prompt,
+		String tokenEncoding,
+		int maxTokens,
+		int maxContextLength,
+		double temperature,
+		double topP,
+		int maxContinuations,
+		ModelProvider modelProvider,
+		String modelName,
+		String url,
+		String apiKey) {
+	
+	public static enum ModelProvider {
+		AZURE_AI, OAI_TEXT_COMPLETION;
+	}
+}

src/main/java/de/extio/lmlib/client/profile/ModelProfileService.java

@@ -0,0 +1,113 @@
+package de.extio.lmlib.client.profile;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.bind.Bindable;
+import org.springframework.boot.context.properties.bind.Binder;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.core.env.Environment;
+import org.springframework.core.io.support.ResourcePropertySource;
+import org.springframework.stereotype.Service;
+
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+
+@Service
+public class ModelProfileService {
+	
+	private final Environment environment;
+	
+	private static final String MODEL_PROFILE_PREFIX = "profile";
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class);
+	
+	public ModelProfileService(final Environment environment) {
+		this.environment = environment;
+	}
+	
+	@Cacheable("modelProfiles")
+	public ModelProfile getModelProfile(final String modelProfile) {
+		final var modelName = this.environment.getProperty(modelProfile);
+		if (modelName == null) {
+			throw new IllegalStateException("No model found for profile: " + modelProfile);
+		}
+		try {
+			final var resource = new ResourcePropertySource("classpath:" + modelName + ".properties");
+			final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts"))
+					.map(Object::toString)
+					.orElseThrow(() -> new IllegalStateException("No model name found for model: " + modelName));
+			final var tokenEncoding = Optional.ofNullable(resource.getProperty("tokenEncoding"))
+					.map(Object::toString)
+					.orElse("none");
+			final var maxTokens = Optional.ofNullable(resource.getProperty("maxTokens"))
+					.map(Object::toString)
+					.map(Integer::parseInt)
+					.orElseThrow(() -> new IllegalStateException("No model maxTokens found for model: " + modelName));
+			final var maxContextLength = Optional.ofNullable(resource.getProperty("maxContextLength"))
+					.map(Object::toString)
+					.map(Integer::parseInt)
+					.orElseThrow(() -> new IllegalStateException("No model maxContextLength found for model: " + modelName));
+			final var temperature = Optional.ofNullable(resource.getProperty("temperature"))
+					.map(Object::toString)
+					.map(Double::parseDouble)
+					.orElseThrow(() -> new IllegalStateException("No model temperature found for model: " + modelName));
+			final var topP = Optional.ofNullable(resource.getProperty("topP"))
+					.map(Object::toString)
+					.map(Double::parseDouble)
+					.orElseThrow(() -> new IllegalStateException("No model topP found for model: " + modelName));
+			final var maxContinuations = Optional.ofNullable(resource.getProperty("maxContinuations"))
+					.map(Object::toString)
+					.map(Integer::parseInt)
+					.orElseThrow(() -> new IllegalStateException("No model maxContinuations found for model: " + modelName));
+			final var modelProvider = Optional.ofNullable(resource.getProperty("modelProvider"))
+					.map(Object::toString)
+					.map(ModelProvider::valueOf)
+					.orElseThrow(() -> new IllegalStateException("No model modelProvider found for model: " + modelName));
+			final var modelNameCfg = Optional.ofNullable(resource.getProperty("modelName"))
+					.map(Object::toString)
+					.orElseThrow(() -> new IllegalStateException("No model modelName found for model: " + modelName));
+			final var url = Optional.ofNullable(resource.getProperty("url"))
+					.map(Object::toString)
+					.orElseThrow(() -> new IllegalStateException("No model url found for model: " + modelName));
+			final var apiKey = Optional.ofNullable(resource.getProperty("apiKey"))
+					.map(Object::toString)
+					.orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
+			
+			return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, maxContinuations, modelProvider, modelNameCfg, url, apiKey);
+		}
+		catch (final IOException e) {
+			LOGGER.error("Error while reading model profile", e);
+			return null;
+		}
+	}
+	
+	public List<String> getModelProfileUrls(final ModelProvider modelProvider) {
+		final var modelProfiles = Binder.get(this.environment)
+				.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
+				.orElse(Collections.emptyMap());
+		
+		return modelProfiles.values().stream()
+				.filter(model -> model != null && !model.isEmpty())
+				.map(model -> {
+					try {
+						final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
+						if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
+							return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
+						}
+						return null;
+					}
+					catch (final IOException e) {
+						LOGGER.error("Error while reading model profile for: " + model, e);
+						return null;
+					}
+				})
+				.filter(Objects::nonNull)
+				.collect(Collectors.toList());
+	}
+}

src/main/java/de/extio/lmlib/client/prompt/ChatMLPromptStrategy.java

@@ -0,0 +1,49 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class ChatMLPromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String system, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<|im_start|>");
+		if (!system.isEmpty()) {
+			prompt.append("system\n");
+			prompt.append(system);
+			prompt.append("<|im_end|>\n");
+		}
+		prompt.append("<|im_start|>user\n");
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append("<|im_end|>\n<|im_start|>assistant\n");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(assistant);
+		prompt.append("<|im_end|>\n<|im_start|>user\n");
+		prompt.append(user);
+		prompt.append("<|im_end|>\n<|im_start|>assistant\n");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<|im_end|>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "chatml";
+	}
+}

src/main/java/de/extio/lmlib/client/prompt/Gemma2PromptStrategy.java

@@ -0,0 +1,48 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class Gemma2PromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String system, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<start_of_turn>user\n");
+		if (!system.isEmpty()) {
+			prompt.append(system);
+			prompt.append("\n");
+		}
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append("<end_of_turn>\n<start_of_turn>model\n");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(assistant);
+		prompt.append("<end_of_turn>\n<start_of_turn>user\n");
+		prompt.append(user);
+		prompt.append("<end_of_turn>\n<start_of_turn>model\n");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<end_of_turn>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "gemma2";
+	}
+	
+}

src/main/java/de/extio/lmlib/client/prompt/Llama3PromptStrategy.java

@@ -0,0 +1,51 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+/**
+ * @see https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
+ */
+@Component
+public class Llama3PromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String system, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<|begin_of_text|>");
+		if (!system.isEmpty()) {
+			prompt.append("<|start_header_id|>system<|end_header_id|>\n\n");
+			prompt.append(system);
+			prompt.append("<|eot_id|>");
+		}
+		prompt.append("<|start_header_id|>user<|end_header_id|>\n\n");
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n\n");
+		}
+		prompt.append(text);
+		prompt.append("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
+		
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(assistant);
+		prompt.append("<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n" + user + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<|eot_id|>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "llama3";
+	}
+}

src/main/java/de/extio/lmlib/client/prompt/MixtralPromptStrategy.java

@@ -0,0 +1,48 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class MixtralPromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String system, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<s>[INST] ");
+		if (!system.isEmpty()) {
+			prompt.append(system);
+			if (!question.isEmpty() || !text.isEmpty()) {
+				prompt.append("\n");
+			}
+		}
+		if (!question.isEmpty()) {
+			prompt.append(question);
+			if (!text.isEmpty()) {
+				prompt.append("\n");
+			}
+		}
+		prompt.append(question);
+		prompt.append(" [/INST]");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append("\n\"");
+		prompt.append(assistant);
+		prompt.append("\"</s> [INST] ");
+		prompt.append(user);
+		prompt.append(" [/INST]");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "mixtral";
+	}
+	
+}

src/main/java/de/extio/lmlib/client/prompt/Phi3PromptStrategy.java

@@ -0,0 +1,48 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class Phi3PromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String system, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append("<|user|>\n");
+		prompt.append(system);
+		if (!system.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append(" <|end|>\n<|assistant|>");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append("\n");
+		prompt.append(assistant);
+		prompt.append("<|user|>\n");
+		prompt.append(user);
+		prompt.append("<|end|>\n<|assistant|>");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("<|end|>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "phi3";
+	}
+}

src/main/java/de/extio/lmlib/client/prompt/PromptStrategy.java

@@ -0,0 +1,17 @@
+package de.extio.lmlib.client.prompt;
+
+public interface PromptStrategy {
+	
+	String getPromptName();
+	
+	StringBuilder start(String system, String question, String text);
+	
+	void continue_(StringBuilder prompt, String assistant);
+	
+	void next(StringBuilder prompt, String assistant, String user);
+	
+	default String removeEOT(final String prompt) {
+		return prompt.strip();
+	}
+	
+}

src/main/java/de/extio/lmlib/client/prompt/PromptStrategyFactory.java

@@ -0,0 +1,22 @@
+package de.extio.lmlib.client.prompt;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class PromptStrategyFactory {
+	
+	private final Map<String, PromptStrategy> strategies = new HashMap<>();
+	
+	public PromptStrategyFactory(final List<PromptStrategy> strategies) {
+		strategies.forEach(strategy -> this.strategies.put(strategy.getPromptName(), strategy));
+	}
+	
+	public PromptStrategy getStrategy(final String modelPrompt) {
+		return this.strategies.get(modelPrompt);
+	}
+	
+}

src/main/java/de/extio/lmlib/client/prompt/VicunaPromptStrategy.java

@@ -0,0 +1,46 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class VicunaPromptStrategy implements PromptStrategy {
+	
+	@Override
+	public StringBuilder start(final String system, final String question, final String text) {
+		final StringBuilder prompt = new StringBuilder();
+		prompt.append(system);
+		prompt.append(" USER: ");
+		prompt.append(question);
+		if (!question.isEmpty() && !text.isEmpty()) {
+			prompt.append("\n");
+		}
+		prompt.append(text);
+		prompt.append(" ASSISTANT:");
+		return prompt;
+	}
+	
+	@Override
+	public void continue_(final StringBuilder prompt, final String assistant) {
+		this.next(prompt, assistant, "Continue");
+	}
+	
+	@Override
+	public void next(final StringBuilder prompt, final String assistant, final String user) {
+		prompt.append(" ");
+		prompt.append(assistant);
+		prompt.append("</s>USER: ");
+		prompt.append(user);
+		prompt.append(" ASSISTANT:");
+	}
+	
+	@Override
+	public String removeEOT(final String prompt) {
+		return prompt.strip().replace("</s>", "");
+	}
+	
+	@Override
+	public String getPromptName() {
+		return "vicuna";
+	}
+	
+}

src/main/java/de/extio/lmlib/client/textcompletion/ModelNameLoader.java

@@ -0,0 +1,65 @@
+package de.extio.lmlib.client.textcompletion;
+
+import java.time.Duration;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.http.HttpMethod;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmlib.client.textcompletion.dto.Model;
+import de.extio.lmlib.client.textcompletion.dto.ModelsResponse;
+
+import reactor.util.retry.Retry;
+
+class ModelNameLoader implements Runnable {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class);
+	
+	private final WebClient webClient;
+	
+	private final Consumer<String> consumer;
+	
+	public ModelNameLoader(@Qualifier("textCompletionWebClientBuilder") final WebClient webClient, final Consumer<String> consumer) {
+		this.consumer = consumer;
+		this.webClient = webClient;
+	}
+	
+	@Override
+	public void run() {
+		String modelName = null;
+		do {
+			try {
+				LOGGER.info("Querying model name");
+				
+				modelName = this.webClient
+						.method(HttpMethod.GET)
+						.uri(uriBuilder -> uriBuilder.path("/v1/models").build())
+						.retrieve()
+						.bodyToMono(ModelsResponse.class)
+						.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+						.block()
+						.getData()
+						.stream()
+						.map(Model::getId)
+						.findFirst()
+						.orElseThrow();
+			}
+			catch (final Exception e) {
+				LOGGER.error(e.getMessage(), e);
+				try {
+					Thread.sleep(1000l);
+				}
+				catch (final InterruptedException e1) {
+					throw new RuntimeException("Interrupted", e1);
+				}
+			}
+		} while (modelName == null);
+		
+		LOGGER.info("Model name: {}", modelName);
+		this.consumer.accept(modelName);
+	}
+	
+}

src/main/java/de/extio/lmlib/client/textcompletion/ModelNameSupplier.java

@@ -0,0 +1,61 @@
+package de.extio.lmlib.client.textcompletion;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+import de.extio.lmlib.client.profile.ModelProfileService;
+
+@Component
+class ModelNameSupplier implements InitializingBean {
+	
+	@Autowired
+	@Qualifier("lmLibWebClientBuilder")
+	private WebClient.Builder webClientBuilder;
+	
+	private final Map<String, CompletableFuture<String>> modelNames = new ConcurrentHashMap<>();
+	
+	private final Map<String, CountDownLatch> countDownLatches = new ConcurrentHashMap<>();
+	
+	@Autowired
+	private ModelProfileService modelProfileService;
+	
+	@Override
+	public void afterPropertiesSet() throws Exception {
+		final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION);
+		for (final var url : urls) {
+			this.countDownLatches.put(url, new CountDownLatch(1));
+			this.fetchModelName(url);
+		}
+	}
+	
+	private void fetchModelName(final String url) {
+		final var webClient = this.webClientBuilder.baseUrl(url).build();
+		CompletableFuture.runAsync(() -> {
+			final var modelNameLoader = new ModelNameLoader(webClient, modelName -> {
+				this.modelNames.put(url, CompletableFuture.completedFuture(modelName));
+				this.countDownLatches.get(url).countDown();
+			});
+			modelNameLoader.run();
+		});
+	}
+	
+	public String getModelName(final String url) {
+		try {
+			this.countDownLatches.get(url).await();
+		}
+		catch (final InterruptedException e) {
+			throw new RuntimeException("Interrupted", e);
+		}
+		return this.modelNames.get(url).join();
+	}
+	
+}

src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java

@@ -0,0 +1,304 @@
+package de.extio.lmlib.client.textcompletion;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmlib.client.Client;
+import de.extio.lmlib.client.Completion;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfile;
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+import de.extio.lmlib.client.profile.ModelProfileService;
+import de.extio.lmlib.client.prompt.PromptStrategy;
+import de.extio.lmlib.client.prompt.PromptStrategyFactory;
+import de.extio.lmlib.client.textcompletion.dto.Choice;
+import de.extio.lmlib.client.textcompletion.dto.CompletionRequest;
+import de.extio.lmlib.client.textcompletion.dto.CompletionResponse;
+import de.extio.lmlib.client.token.Tokenizer;
+
+import reactor.util.retry.Retry;
+
+@Component
+public class TextCompletionClient implements Client, DisposableBean {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
+	
+	@Autowired
+	@Qualifier("lmLibWebClientBuilder")
+	private WebClient.Builder webClientBuilder;
+	
+	@Autowired
+	private PromptStrategyFactory promptStrategyFactory;
+	
+	@Autowired
+	private ModelNameSupplier modelNameSupplier;
+	
+	@Autowired
+	private Tokenizer tokenizer;
+	
+	@Autowired
+	private ModelProfileService modelProfileService;
+	
+	@Value("${client.collectStatistics:false}")
+	private boolean collectStatistics;
+	
+	private final CompletionStatistics totalStatistics = new CompletionStatistics();
+	
+	@Override
+	public ModelProvider getModelProvider() {
+		return ModelProvider.OAI_TEXT_COMPLETION;
+	}
+	
+	@Override
+	public Completion completion(final ModelCategory modelCategory, final String system, final String question, final String fullText) {
+		String text = "";
+		if (!question.isEmpty() && !fullText.isEmpty()) {
+			text = question + "\n" + fullText;
+		}
+		else if (!question.isEmpty()) {
+			text = question;
+		}
+		else if (!fullText.isEmpty()) {
+			text = fullText;
+		}
+		else {
+			throw new IllegalArgumentException("Either question or fullText must be set");
+		}
+		
+		return this.conversation(modelCategory, Conversation.create(system, text));
+	}
+	
+	@Override
+	public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
+		if (modelCategory == null) {
+			modelCategory = ModelCategory.MEDIUM;
+		}
+		final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
+		if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION) {
+			throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
+		}
+		
+		final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
+		if (promptStrategy == null) {
+			throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
+		}
+		
+		final List<StringBuilder> prompts = this.createPrompts(conversation, modelCategory, modelProfile, promptStrategy);
+		
+		final List<StringBuilder> answers = new ArrayList<>();
+		final CompletionStatistics statistics = new CompletionStatistics();
+		for (final StringBuilder prompt : prompts) {
+			final StringBuilder answer = new StringBuilder();
+			this.requestCompletionsContinuations(prompt, answer, statistics, modelProfile, promptStrategy);
+			answers.add(answer);
+		}
+		
+		if (answers.size() > 1) {
+			this.summarize(answers, modelProfile, promptStrategy, statistics);
+		}
+		
+		return new Completion(
+				answers.stream().collect(Collectors.joining()),
+				statistics.requests,
+				statistics.duration,
+				statistics.inTokens,
+				statistics.outTokens,
+				false);
+	}
+	
+	private List<StringBuilder> createPrompts(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
+		StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
+		
+		List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
+		final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
+		if (tokenized.size() > maxLength) {
+			// TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
+			LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
+			tokenized = tokenized.subList(0, maxLength);
+			prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
+		}
+		
+		LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
+		
+		return List.of(prompt);
+	}
+	
+	private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
+		final StringBuilder prompt = new StringBuilder();
+		
+		final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
+		Turn systemTurn = null;
+		Turn userTurn = null;
+		Turn assistantTurn = null;
+		boolean first = true;
+		
+		while (!turns.isEmpty()) {
+			final Turn turn = turns.pop();
+			switch (turn.type()) {
+				case SYSTEM:
+					systemTurn = turn;
+					break;
+				case USER:
+					userTurn = turn;
+					break;
+				case ASSISTANT:
+					assistantTurn = turn;
+					break;
+				default:
+					throw new IllegalArgumentException("Unknown turn type: " + turn.type());
+			}
+			
+			if (first && userTurn != null) {
+				prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", "", userTurn.text()));
+				userTurn = null;
+				first = false;
+			}
+			else if (!first && assistantTurn != null && userTurn != null) {
+				promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
+				userTurn = null;
+				assistantTurn = null;
+			}
+		}
+		
+		return prompt;
+	}
+	
+	private void requestCompletionsContinuations(final StringBuilder prompt, final StringBuilder answer, final CompletionStatistics statistics,
+			final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
+		for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
+			final LocalDateTime start = LocalDateTime.now();
+			
+			final String promptStr = prompt.toString();
+			
+			final var response = this.requestCompletions(promptStr, modelProfile);
+			
+			String content = null;
+			var complete = false;
+			if (response.getContent() != null) {
+				content = promptStrategy.removeEOT(response.getContent());
+				complete = response.isStoppedEos();
+			}
+			else {
+				final var choice = response.getChoices().getFirst();
+				content = promptStrategy.removeEOT(choice.getText());
+				complete = !Choice.FINISH_REASON_LENGTH.equals(choice.getFinishReason());
+			}
+			
+			if (this.collectStatistics) {
+				final var dur = Duration.between(start, LocalDateTime.now());
+				final var in = this.tokenizer.count(promptStr, modelProfile);
+				final var out = this.tokenizer.count(content, modelProfile);
+				LOGGER.debug("Request duration: " + dur + "; in tokens: " + in + "; out tokens: " + out);
+				statistics.add(dur, in, out);
+				this.totalStatistics.add(dur, in, out);
+			}
+			
+			answer.append(content);
+			
+			if (complete) {
+				LOGGER.debug("Prompt response complete");
+				break;
+			}
+			
+			answer.append('\n');
+			promptStrategy.continue_(prompt, content);
+		}
+	}
+	
+	private CompletionResponse requestCompletions(final String prompt, final ModelProfile modelProfile) {
+		LOGGER.debug("Requesting completion at {}", modelProfile.url());
+		
+		final var request = new CompletionRequest();
+		request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
+		request.setPrompt(prompt);
+		request.setMaxTokens(modelProfile.maxTokens());
+		request.setTemperature(modelProfile.temperature());
+		request.setTopP(modelProfile.topP());
+		request.setStream(false);
+		
+		final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
+		return webClient
+				.method(HttpMethod.POST)
+				.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
+				.bodyValue(request)
+				.retrieve()
+				.bodyToMono(CompletionResponse.class)
+				.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+				.block();
+	}
+	
+	private void summarize(final List<StringBuilder> answers, final ModelProfile modelProfile, final PromptStrategy promptStrategy, final CompletionStatistics statistics) {
+		// TODO: Reactivate when multiple (split) prompts have been implemented again
+		//		LOGGER.debug("Summarizing answers");
+		//		
+		//		final StringBuilder summary = new StringBuilder();
+		//		
+		//		final StringBuilder prompt = promptStrategy.start(instruction, question, "");
+		//		answers.forEach(answer -> promptStrategy.next(prompt, answer.toString(), "Continue"));
+		//		promptStrategy.next(prompt, "", "Generate now a full summary");
+		//		
+		//		this.requestCompletionsContinuations(prompt, summary, statistics, modelProfile, promptStrategy);
+		//		
+		//		answers.clear();
+		//		answers.add(summary);
+	}
+	
+	@Override
+	public void destroy() throws Exception {
+		if (this.collectStatistics) {
+			LOGGER.info("Client Statistics: {}", this.totalStatistics);
+		}
+	}
+	
+	static class CompletionStatistics {
+		
+		volatile int requests;
+		
+		volatile Duration duration = Duration.ofMillis(0l);
+		
+		volatile long inTokens;
+		
+		volatile long outTokens;
+		
+		synchronized void add(final Duration duration, final int inTokens, final int outTokens) {
+			this.requests++;
+			this.duration = this.duration.plus(duration);
+			this.inTokens += inTokens;
+			this.outTokens += outTokens;
+		}
+		
+		@Override
+		public String toString() {
+			final StringBuilder builder = new StringBuilder();
+			builder.append("CompletionStatistics [requests=");
+			builder.append(this.requests);
+			builder.append(", duration=");
+			builder.append(this.duration);
+			builder.append(", inTokens=");
+			builder.append(this.inTokens);
+			builder.append(", outTokens=");
+			builder.append(this.outTokens);
+			builder.append("]");
+			return builder.toString();
+		}
+		
+	}
+	
+}

src/main/java/de/extio/lmlib/client/textcompletion/dto/Choice.java

@@ -0,0 +1,59 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class Choice {
+	
+	public static final String FINISH_REASON_STOP = "stop";
+	
+	public static final String FINISH_REASON_LENGTH = "length";
+	
+	public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
+	
+	private String text;
+	
+	private Double score;
+	
+	private String finishReason;
+	
+	@JsonProperty("text")
+	public String getText() {
+		return this.text;
+	}
+	
+	public void setText(final String text) {
+		this.text = text;
+	}
+	
+	@JsonProperty("score")
+	public Double getScore() {
+		return this.score;
+	}
+	
+	public void setScore(final Double score) {
+		this.score = score;
+	}
+	
+	@JsonProperty("finish_reason")
+	public String getFinishReason() {
+		return this.finishReason;
+	}
+	
+	public void setFinishReason(final String finishReason) {
+		this.finishReason = finishReason;
+	}
+	
+	@Override
+	public String toString() {
+		final StringBuilder builder = new StringBuilder();
+		builder.append("Choice [text=");
+		builder.append(this.text);
+		builder.append(", score=");
+		builder.append(this.score);
+		builder.append(", finishReason=");
+		builder.append(this.finishReason);
+		builder.append("]");
+		return builder.toString();
+	}
+	
+}

src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionRequest.java

@@ -0,0 +1,72 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class CompletionRequest {
+	
+	private String model;
+	
+	private String prompt;
+	
+	private Integer maxTokens;
+	
+	private Double temperature;
+	
+	private Double topP;
+	
+	private Boolean stream;
+	
+	@JsonProperty("model")
+	public String getModel() {
+		return this.model;
+	}
+	
+	public void setModel(final String model) {
+		this.model = model;
+	}
+	
+	@JsonProperty("prompt")
+	public String getPrompt() {
+		return this.prompt;
+	}
+	
+	public void setPrompt(final String prompt) {
+		this.prompt = prompt;
+	}
+	
+	@JsonProperty("max_tokens")
+	public Integer getMaxTokens() {
+		return this.maxTokens;
+	}
+	
+	public void setMaxTokens(final Integer maxTokens) {
+		this.maxTokens = maxTokens;
+	}
+	
+	@JsonProperty("temperature")
+	public Double getTemperature() {
+		return this.temperature;
+	}
+	
+	public void setTemperature(final Double temperature) {
+		this.temperature = temperature;
+	}
+	
+	@JsonProperty("top_p")
+	public Double getTopP() {
+		return this.topP;
+	}
+	
+	public void setTopP(final Double topP) {
+		this.topP = topP;
+	}
+	
+	@JsonProperty("stream")
+	public Boolean isStream() {
+		return this.stream;
+	}
+	
+	public void setStream(final Boolean stream) {
+		this.stream = stream;
+	}
+}

src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionResponse.java

@@ -0,0 +1,93 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class CompletionResponse {
+	
+	private String id;
+	
+	private String object;
+	
+	private Integer created;
+	
+	private List<Choice> choices;
+	
+	private String content;
+	
+	private boolean isStoppedEos;
+	
+	@JsonProperty("id")
+	public String getId() {
+		return this.id;
+	}
+	
+	public void setId(final String id) {
+		this.id = id;
+	}
+	
+	@JsonProperty("object")
+	public String getObject() {
+		return this.object;
+	}
+	
+	public void setObject(final String object) {
+		this.object = object;
+	}
+	
+	@JsonProperty("created")
+	public Integer getCreated() {
+		return this.created;
+	}
+	
+	public void setCreated(final Integer created) {
+		this.created = created;
+	}
+	
+	@JsonProperty("choices")
+	public List<Choice> getChoices() {
+		return this.choices;
+	}
+	
+	public void setChoices(final List<Choice> choices) {
+		this.choices = choices;
+	}
+	
+	@JsonProperty("content")
+	public String getContent() {
+		return this.content;
+	}
+	
+	public void setContent(final String content) {
+		this.content = content;
+	}
+	
+	@JsonProperty("stopped_eos")
+	public boolean isStoppedEos() {
+		return this.isStoppedEos;
+	}
+	
+	public void setStoppedEos(final boolean isStoppedEos) {
+		this.isStoppedEos = isStoppedEos;
+	}
+	
+	@Override
+	public String toString() {
+		final StringBuilder builder = new StringBuilder();
+		builder.append("CompletionResponse [id=");
+		builder.append(this.id);
+		builder.append(", object=");
+		builder.append(this.object);
+		builder.append(", created=");
+		builder.append(this.created);
+		builder.append(", choices=");
+		builder.append(this.choices);
+		builder.append(", content=");
+		builder.append(this.content);
+		builder.append(", isStoppedEos=");
+		builder.append(this.isStoppedEos);
+		builder.append("]");
+		return builder.toString();
+	}
+}

src/main/java/de/extio/lmlib/client/textcompletion/dto/Model.java

@@ -0,0 +1,74 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class Model {
+	
+	private String id;
+	
+	private String object;
+	
+	private int created;
+	
+	private String ownedBy;
+	
+	private List<String> permission;
+	
+	private String root;
+	
+	@JsonProperty("id")
+	public String getId() {
+		return this.id;
+	}
+	
+	public void setId(final String id) {
+		this.id = id;
+	}
+	
+	@JsonProperty("object")
+	public String getObject() {
+		return this.object;
+	}
+	
+	public void setObject(final String object) {
+		this.object = object;
+	}
+	
+	@JsonProperty("created")
+	public int getCreated() {
+		return this.created;
+	}
+	
+	public void setCreated(final int created) {
+		this.created = created;
+	}
+	
+	@JsonProperty("owned_by")
+	public String getOwned_by() {
+		return this.ownedBy;
+	}
+	
+	public void setOwned_by(final String owned_by) {
+		this.ownedBy = owned_by;
+	}
+	
+	@JsonProperty("permission")
+	public List<String> getPermission() {
+		return this.permission;
+	}
+	
+	public void setPermission(final List<String> permission) {
+		this.permission = permission;
+	}
+	
+	@JsonProperty("root")
+	public String getRoot() {
+		return this.root;
+	}
+	
+	public void setRoot(final String root) {
+		this.root = root;
+	}
+}

src/main/java/de/extio/lmlib/client/textcompletion/dto/ModelsResponse.java

@@ -0,0 +1,42 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class ModelsResponse {
+	
+	private String object;
+	
+	private List<Model> data;
+	
+	@JsonProperty("object")
+	public String getObject() {
+		return this.object;
+	}
+	
+	public void setObject(final String object) {
+		this.object = object;
+	}
+	
+	@JsonProperty("data")
+	public List<Model> getData() {
+		return this.data;
+	}
+	
+	public void setData(final List<Model> data) {
+		this.data = data;
+	}
+	
+	@Override
+	public String toString() {
+		final StringBuilder builder = new StringBuilder();
+		builder.append("ModelsResponse [object=");
+		builder.append(this.object);
+		builder.append(", data=");
+		builder.append(this.data);
+		builder.append("]");
+		return builder.toString();
+	}
+	
+}

src/main/java/de/extio/lmlib/client/token/JTokkitTokenizer.java

@@ -0,0 +1,53 @@
+package de.extio.lmlib.client.token;
+
+import java.util.List;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.knuddels.jtokkit.Encodings;
+import com.knuddels.jtokkit.api.Encoding;
+import com.knuddels.jtokkit.api.EncodingRegistry;
+import com.knuddels.jtokkit.api.EncodingType;
+import com.knuddels.jtokkit.api.IntArrayList;
+
+import de.extio.lmlib.client.profile.ModelProfile;
+
+final class JTokkitTokenizer implements Tokenizer {
+	
+	private static final EncodingRegistry registry;
+	
+	static {
+		registry = Encodings.newDefaultEncodingRegistry();
+	}
+	
+	private final LoadingCache<String, Encoding> encodings = Caffeine
+			.newBuilder()
+			.build(k -> {
+				return registry.getEncoding(EncodingType.fromName(k).orElseThrow(() -> new IllegalArgumentException("Token encoding " + k)));
+			});
+	
+	@Override
+	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
+		return this.encodings
+				.get(modelProfile.tokenEncoding())
+				.encodeOrdinary(txt)
+				.boxed()
+				.stream()
+				.map(Long::valueOf)
+				.toList();
+	}
+	
+	@Override
+	public int count(final String txt, final ModelProfile modelProfile) {
+		return this.encodings.get(modelProfile.tokenEncoding()).countTokensOrdinary(txt);
+	}
+	
+	@Override
+	public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
+		final var ial = new IntArrayList(tokens.size());
+		tokens.forEach(t -> ial.add(t.intValue()));
+		return this.encodings
+				.get(modelProfile.tokenEncoding())
+				.decode(ial);
+	}
+}

src/main/java/de/extio/lmlib/client/token/LlamaServerTokenizer.java

@@ -0,0 +1,75 @@
+package de.extio.lmlib.client.token;
+
+import java.time.Duration;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpMethod;
+import org.springframework.web.reactive.function.client.WebClient;
+import org.springframework.web.reactive.function.client.WebClient.Builder;
+
+import de.extio.lmlib.client.profile.ModelProfile;
+
+import reactor.util.retry.Retry;
+
+final class LlamaServerTokenizer implements Tokenizer {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class);
+	
+	private final WebClient.Builder webClientBuilder;
+	
+	public LlamaServerTokenizer(final Builder webClientBuilder) {
+		this.webClientBuilder = webClientBuilder;
+	}
+	
+	@Override
+	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
+		LOGGER.debug("Requesting tokenizing");
+		final var request = new TokenizeRequest(txt);
+		final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
+		final var response = webClient
+				.method(HttpMethod.POST)
+				.uri(uriBuilder -> uriBuilder.path("/tokenize").build())
+				.bodyValue(request)
+				.retrieve()
+				.bodyToMono(TokenizeResponse.class)
+				.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+				.block();
+		return response.tokens();
+	}
+	
+	@Override
+	public int count(final String txt, final ModelProfile modelProfile) {
+		return this.tokenize(txt, modelProfile).size();
+	}
+	
+	@Override
+	public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
+		LOGGER.debug("Requesting detokenizing");
+		final var request = new DetokenizeRequest(tokens);
+		final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
+		final var response = webClient
+				.method(HttpMethod.POST)
+				.uri(uriBuilder -> uriBuilder.path("/detokenize").build())
+				.bodyValue(request)
+				.retrieve()
+				.bodyToMono(DetokenizeResponse.class)
+				.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+				.block();
+		return response.content();
+	}
+	
+	private record TokenizeRequest(String content) {
+	}
+	
+	private record TokenizeResponse(List<Long> tokens) {
+	}
+	
+	private record DetokenizeRequest(List<Long> tokens) {
+	}
+	
+	private record DetokenizeResponse(String content) {
+	}
+	
+}

src/main/java/de/extio/lmlib/client/token/Tokenizer.java

@@ -0,0 +1,15 @@
+package de.extio.lmlib.client.token;
+
+import java.util.List;
+
+import de.extio.lmlib.client.profile.ModelProfile;
+
+public interface Tokenizer {
+	
+	List<Long> tokenize(String txt, ModelProfile modelProfile);
+	
+	int count(String txt, ModelProfile modelProfile);
+	
+	String detokenize(List<Long> tokens, ModelProfile modelProfile);
+	
+}

src/main/java/de/extio/lmlib/client/token/TokenizerAutoConfiguration.java

@@ -0,0 +1,35 @@
+package de.extio.lmlib.client.token;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.reactive.function.client.WebClient;
+
+@AutoConfiguration
+public class TokenizerAutoConfiguration {
+	
+	@Bean
+	@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer")
+	@ConditionalOnMissingBean
+	Tokenizer llamaServerTokenizer(@Qualifier("lmLibWebClientBuilder") final WebClient.Builder webClientBuilder) {
+		return new LlamaServerTokenizer(webClientBuilder);
+	}
+	
+	@Configuration(proxyBeanMethods = false)
+	@ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings")
+	@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "jTokkit")
+	@ConditionalOnMissingBean
+	public static class SomeServiceConfiguration {
+		
+		@Bean
+		Tokenizer jTokkitTokenizer() {
+			return new JTokkitTokenizer();
+		}
+		
+	}
+	
+}