commit c1a283305d0967cc7eb682e6eee84f96d58fa0ce
Author: (no author) <(no author)@71dcc7fd-b1f2-4e5d-bb8e-274146e81173>
Date: Mon Dec 16 20:58:04 2024 +0000
lm-lib first commit;
git-svn-id: svn://home0/lm-lib@3603 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e6a822e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,52 @@
+This suite of Java packages provides a comprehensive framework for building, deploying, and interacting with conversational AI agents powered by language models.
+
+**Core Functionality:**
+
+* **Agent Framework:** Enables the creation of modular and scalable agents with complex branching logic and concurrent execution capabilities.
+* **Language Model Interaction:** Provides tools for accessing and interacting with various language models, including Azure AI's text generation service. This includes retrieving model information, managing conversations, generating completions, and customizing parameters.
+* **Prompt Engineering:** Offers functionalities for constructing and manipulating prompts tailored to specific language models, supporting various formatting strategies.
+* **Text Processing:** Includes tools for tokenizing and detokenizing text, essential for preparing input for language models and interpreting their output.
+
+**Purpose:**
+
+The primary purpose of this program is to simplify the development and deployment of sophisticated conversational AI agents. By providing a robust set of tools for agent management, language model interaction, prompt engineering, and text processing, the program empowers developers to create powerful and engaging AI-driven conversational experiences.
+
+*gemma2-27b requests=49, requestDuration=PT7M4.534240136S, inTokens=26243, outTokens=2159, tps=5.091981132075472*
+
+## Introducing a Powerful Framework for Building Intelligent Agents
+
+We're excited to announce the development of a new Java codebase designed to build intelligent agents capable of interacting with language models like GPT. This framework offers a flexible and extensible architecture, empowering developers to create sophisticated conversational agents for a wide range of applications.
+
+**Key Features:**
+
+* **Concurrent Text Processing:** Leveraging Java's `CompletableFuture` and `ExecutorService`, the framework enables parallel processing of text chunks, significantly speeding up agent execution.
+
+* **Response-Driven Agent Selection:** The `chooseNextAgent` method intelligently selects the next agent in the chain based on the response received from the language model, allowing for dynamic and adaptive conversations.
+
+* **Customizable Response Handling:** Each agent can be equipped with a custom `responseHandler` to parse and interpret responses from the language model, enabling tailored logic for specific tasks.
+
+* **Dynamic Agent Chaining:** Agents can be chained together to create complex, multi-step interactions, with each agent specializing in a particular task and passing context to the next.
+
+* **Contextualized Agent Execution:** The framework utilizes an `AgentContext` object to store and share information between agents, ensuring each agent has access to the relevant context for its task.
+
+**How it Works:**
+
+The framework revolves around the `Agent` class, which defines an agent's behavior and interaction with the language model. Each agent receives an `AgentContext` containing relevant information, processes text using its defined template, interacts with the language model, and then uses its `responseHandler` to interpret the response. Based on the response, the `chooseNextAgent` method determines the next agent in the chain.
+
+**Benefits:**
+
+This framework offers several advantages for developers:
+
+* **Flexibility:** Easily customize agent behavior through configurable parameters and custom response handlers.
+* **Extensibility:** Add new agent types and functionalities as needed.
+* **Efficiency:** Parallel processing and dynamic agent chaining optimize performance.
+* **Adaptability:** Agents can respond intelligently to changing conversation contexts.
+
+**Looking Ahead:**
+
+We believe this framework has the potential to revolutionize the way we build conversational agents. We're actively developing new features and functionalities, and we encourage the community to contribute and explore the possibilities.
+
+Stay tuned for more updates and documentation!
+
+*Graph: M, CodeSummarizer, ↴, M, FeatureIdentifier, →, ⇉ 3/5, M, FeatureAnalyzer, →, 5↣1, M, AnswerGenerator*
+*gemma2-27b requests=8, inTokens=19646, outTokens=2784, requestDuration=PT7M38.309006603S, getAbsoluteDuration()=PT3M54.653718763S, getTps()=95.59, getOutTps()=11.86*
diff --git a/pom.xml b/pom.xml
new file mode 100755
index 0000000..dd488f7
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,60 @@
+
+
+ 4.0.0
+ lm-lib
+ de.extio
+ lm-lib
+ 1.0.0-SNAPSHOT
+
+
+ 21
+ 21
+ 21
+ 3.4.0
+ 1.0.0-beta.2
+ 1.1.0
+
+
+
+
+
+ org.springframework.boot
+ spring-boot-dependencies
+ ${spring-boot.version}
+ pom
+ import
+
+
+
+
+
+
+ org.springframework.boot
+ spring-boot-starter-webflux
+
+
+
+ com.github.ben-manes.caffeine
+ caffeine
+
+
+
+ com.azure
+ azure-ai-inference
+ ${azureai.version}
+ provided
+
+
+ com.knuddels
+ jtokkit
+ ${jtokkit.version}
+ provided
+
+
+
+ org.springframework.boot
+ spring-boot-starter-test
+ test
+
+
+
diff --git a/src/main/java/de/extio/lmlib/agent/Agent.java b/src/main/java/de/extio/lmlib/agent/Agent.java
new file mode 100755
index 0000000..25dc2d1
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/Agent.java
@@ -0,0 +1,238 @@
+package de.extio.lmlib.agent;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.extio.lmlib.client.Client;
+import de.extio.lmlib.client.Completion;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.TurnType;
+import de.extio.lmlib.client.profile.ModelCategory;
+
+public record Agent(String name, AgentType agentType, ModelCategory modelCategory, String systemPrompt, String textTemplate, AgentResponseHandler responseHandler, Consumer preProcessor, Consumer postProcessor, Function, List> merger, Function chooseNext) {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(Agent.class);
+
+ public List execute(final Client client, final ExecutorService agentExecutorService, final AgentContext context) {
+ final var retContexts = Collections.synchronizedList(new ArrayList());
+
+ context.setNextAgent(null);
+
+ if (this.preProcessor != null) {
+ this.preProcessor.accept(context);
+ }
+
+ final var splits = this.applyTemplate(context);
+ final var tasks = new ArrayList>(splits.size());
+ for (final var split : splits) {
+ tasks.add(CompletableFuture.runAsync(() -> {
+ split.context().getGraph().add(this.modelCategory.getShortName());
+ split.context().getGraph().add(this.name());
+
+ final var conversation = this.setupConversation(split);
+ LOGGER.debug("Conversation: {}", conversation);
+
+ Completion completion = null;
+ boolean parseable = false;
+ for (int i = 0; i < 2; i++) {
+ completion = client.conversation(this.modelCategory, conversation);
+ split.context().getRequestStatistic().add(completion);
+
+ try {
+ if (!this.responseHandler.handle(split, completion)) {
+ context.getGraph().add("⚠");
+ LOGGER.warn("{} Cannot parse response: {}", this.name(), completion.response());
+ continue;
+ }
+
+ parseable = true;
+ break;
+ }
+ catch (final Exception e) {
+ context.getGraph().add("⚠");
+ LOGGER.warn("{} Cannot parse response: {}", this.name(), completion.response(), e);
+ continue;
+ }
+ }
+ if (!parseable) {
+ LOGGER.warn("{} Response is still not parseable after last attempt", this.name());
+ context.getGraph().add("☢");
+ return;
+ }
+
+ if (this.postProcessor != null) {
+ this.postProcessor.accept(split.context());
+ }
+
+ this.chooseNextAgent(conversation, completion, split.context());
+
+ retContexts.add(split.context());
+ }, agentExecutorService));
+ }
+
+ CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).join();
+
+ if (this.merger != null) {
+ final var mergedContexts = this.merger.apply(retContexts);
+ mergedContexts.forEach(c -> c.getGraph().add(retContexts.size() + "↣" + mergedContexts.size()));
+ return mergedContexts;
+ }
+ return retContexts;
+ }
+
+ public static List mergeContexts(final List keys, final List contexts) {
+ final var first = contexts.getFirst();
+ for (final var key : keys) {
+ first.getContext().compute(key, (k, v) -> {
+ final var al = new ArrayList<>();
+ if (v != null) {
+ al.addAll(v);
+ }
+ for (int i = 1; i < contexts.size(); i++) {
+ final var ctx = contexts.get(i);
+ final var val = ctx.getContext().get(key);
+ if (val != null) {
+ al.addAll(val);
+ }
+ }
+ return al;
+ });
+ }
+ return List.of(first);
+ }
+
+ private List applyTemplate(final AgentContext context) {
+ String text = this.textTemplate;
+ for (final var entry : context.getContext().entrySet()) {
+ final var k = "{{" + entry.getKey() + "}}";
+ if (text.contains(k)) {
+ text = text.replace(k, entry.getValue().stream().map(Object::toString).collect(Collectors.joining("\n\n")));
+ }
+ }
+
+ for (final var entry : context.getContext().entrySet()) {
+ final var k = "[[" + entry.getKey() + "]]";
+ if (text.contains(k)) {
+ final var result = new ArrayList();
+
+ int i = 0;
+ for (final var value : entry.getValue()) {
+ final var nextText = text.replace(k, value.toString());
+
+ final var nextContext = new HashMap<>(context.getContext());
+ nextContext.put(entry.getKey(), List.of(value));
+
+ final var nextAgentContext = new AgentContext(context, nextContext);
+ if (entry.getValue().size() > 1) {
+ nextAgentContext.getGraph().add("⇉ " + ++i + "/" + entry.getValue().size());
+ }
+
+ result.add(new Split(nextText, nextAgentContext));
+ }
+
+ return result; // TODO: Multiple splits not supported yet
+ }
+ }
+
+ return List.of(new Split(text, context));
+ }
+
+ private Conversation setupConversation(final Split split) {
+ var conversation = split.context().getConversation();
+ if (conversation == null || conversation.getConversation().isEmpty()) {
+ conversation = Conversation.create(this.systemPrompt(), split.text());
+ split.context().setConversation(conversation);
+ }
+ else if (this.agentType() == AgentType.START_CONVERSATION) {
+ conversation = Conversation.create(this.systemPrompt() + "\n" + split.text());
+ split.context().setConversation(conversation);
+ }
+ else {
+ if (conversation.getConversation().get(0).type() == TurnType.SYSTEM) {
+ final var mergedSysAndUser = conversation.getConversation().get(0).text() + "\n" + conversation.getConversation().get(1).text();
+
+ final var newConversation = Conversation.create(mergedSysAndUser);
+ for (int n = 2; n < conversation.getConversation().size(); n++) {
+ newConversation.addTurn(conversation.getConversation().get(n));
+ }
+
+ split.context().setConversation(newConversation);
+ conversation = newConversation;
+ }
+ if (conversation.getConversation().get(conversation.getConversation().size() - 1).type() == TurnType.ASSISTANT) {
+ conversation.addTurn(new Conversation.Turn(TurnType.USER, String.join("\n", this.systemPrompt, split.text())));
+ }
+ }
+ return conversation;
+ }
+
+ private void chooseNextAgent(final Conversation conversation, final Completion completion, final AgentContext context) {
+ context.setNextAgent(this.chooseNext.apply(context));
+
+ if (context.getNextAgent() != null) {
+ if (context.getNextAgent().gradingPassed() != null) {
+ if (context.getNextAgent().gradingPassed()) {
+ context.getGraph().add("✓");
+ }
+ else {
+ context.getGraph().add("✗");
+ }
+ }
+
+ if (context.getNextAgent().name() == null) {
+ context.setNextAgent(null);
+ return;
+ }
+
+ String answer = completion.response();
+ if (context.getNextAgent().transformResponse() != null) {
+ answer = context.getNextAgent().transformResponse().apply(answer);
+ }
+
+ final Agent nextAgent = context.getAgents().get(context.getNextAgent().name());
+ switch (nextAgent) {
+ case null -> {
+ context.getGraph().add("☢");
+ LOGGER.error("Agent not found: {}", context.getNextAgent().name());
+ context.setNextAgent(null);
+ }
+
+ case final Agent a when context.getNextAgent().responseAsUserPrompt() == Boolean.TRUE -> {
+ context.getGraph().add("↺");
+ conversation.replaceTurn(new Conversation.Turn(TurnType.USER, answer));
+ }
+
+ case final Agent a when a.agentType() == AgentType.COMPLETION -> {
+ context.getGraph().add("→");
+ context.setConversation(null);
+ }
+
+ case final Agent a when a.agentType() == AgentType.START_CONVERSATION -> {
+ context.getGraph().add("🗨");
+ context.setConversation(null);
+ }
+
+ case final Agent a when a.agentType() == AgentType.CONVERSATION -> {
+ context.getGraph().add("↴");
+ conversation.addTurn(new Conversation.Turn(TurnType.ASSISTANT, answer));
+ }
+
+ default -> {
+ context.getGraph().add("☢");
+ LOGGER.error("Unknown agent type: {}", nextAgent.agentType());
+ context.setNextAgent(null);
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/agent/AgentContext.java b/src/main/java/de/extio/lmlib/agent/AgentContext.java
new file mode 100755
index 0000000..418d464
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/AgentContext.java
@@ -0,0 +1,78 @@
+package de.extio.lmlib.agent;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import de.extio.lmlib.client.Conversation;
+
+public final class AgentContext {
+
+ private final Map agents;
+
+ private final Map> context;
+
+ private final AgentRequestStatistic requestStatistic;
+
+ private final List graph = Collections.synchronizedList(new ArrayList<>());
+
+ private volatile Conversation conversation;
+
+ private volatile AgentNext nextAgent;
+
+ public AgentContext(final Map agents) {
+ this.agents = agents;
+ this.context = new ConcurrentHashMap<>();
+ this.requestStatistic = new AgentRequestStatistic();
+ }
+
+ public AgentContext(final AgentContext other) {
+ this(other, new ConcurrentHashMap<>(other.context));
+ }
+
+ public AgentContext(final AgentContext other, final Map> context) {
+ this.agents = other.agents;
+ this.context = context;
+ this.conversation = Conversation.copy(other.conversation);
+ this.nextAgent = other.nextAgent;
+ this.requestStatistic = other.requestStatistic;
+ synchronized (other.graph) {
+ this.graph.addAll(other.graph);
+ }
+ }
+
+ public Map> getContext() {
+ return this.context;
+ }
+
+ public Map getAgents() {
+ return this.agents;
+ }
+
+ public AgentRequestStatistic getRequestStatistic() {
+ return this.requestStatistic;
+ }
+
+ public Conversation getConversation() {
+ return this.conversation;
+ }
+
+ public void setConversation(final Conversation conversation) {
+ this.conversation = conversation;
+ }
+
+ public AgentNext getNextAgent() {
+ return this.nextAgent;
+ }
+
+ public void setNextAgent(final AgentNext next) {
+ this.nextAgent = next;
+ }
+
+ public List getGraph() {
+ return this.graph;
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/agent/AgentExecutorService.java b/src/main/java/de/extio/lmlib/agent/AgentExecutorService.java
new file mode 100755
index 0000000..692e151
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/AgentExecutorService.java
@@ -0,0 +1,78 @@
+package de.extio.lmlib.agent;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import de.extio.lmlib.client.ClientService;
+
+@Service
+public class AgentExecutorService implements InitializingBean, DisposableBean {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AgentExecutorService.class);
+
+ @Value("${agent.threads:2}")
+ private int threads;
+
+ @Autowired
+ private ClientService clientService;
+
+ private ExecutorService branchExecutorService;
+
+ private ExecutorService agentExecutorService;
+
+ @Override
+ public void afterPropertiesSet() throws Exception {
+ this.branchExecutorService = Executors.newVirtualThreadPerTaskExecutor();
+ this.agentExecutorService = Executors.newFixedThreadPool(this.threads);
+ }
+
+ @Override
+ public void destroy() throws Exception {
+ this.branchExecutorService.close();
+ this.agentExecutorService.close();
+ }
+
+ public List walkGraph(final Agent agent, final AgentContext context) {
+ LOGGER.info("Agent: {}", agent.name());
+
+ final var client = this.clientService.getClient(agent.modelCategory());
+ final var responses = Collections.synchronizedList(new ArrayList());
+
+ final var branches = agent.execute(client, this.agentExecutorService, context);
+ final var branchTasks = new ArrayList>(branches.size());
+ for (final var branchContext : branches) {
+ if (branchContext.getNextAgent() != null) {
+ final var branchAgent = context.getAgents().get(branchContext.getNextAgent().name());
+ if (branchAgent == null) {
+ throw new IllegalArgumentException("Agent not found: " + branchContext.getNextAgent().name());
+ }
+
+ branchTasks.add(CompletableFuture.runAsync(() -> {
+ responses.addAll(this.walkGraph(branchAgent, branchContext));
+ }, this.branchExecutorService));
+ }
+ else {
+ responses.add(branchContext);
+ }
+ }
+
+ if (!branchTasks.isEmpty()) {
+ CompletableFuture.allOf(branchTasks.toArray(new CompletableFuture[0])).join();
+ }
+
+ return responses;
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/agent/AgentNext.java b/src/main/java/de/extio/lmlib/agent/AgentNext.java
new file mode 100755
index 0000000..03b12ed
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/AgentNext.java
@@ -0,0 +1,7 @@
+package de.extio.lmlib.agent;
+
+import java.util.function.Function;
+
+public final record AgentNext(String name, Boolean gradingPassed, Boolean responseAsUserPrompt, Function transformResponse) {
+
+}
diff --git a/src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java b/src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java
new file mode 100755
index 0000000..f7053b1
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java
@@ -0,0 +1,83 @@
+package de.extio.lmlib.agent;
+
+import java.text.DecimalFormat;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import de.extio.lmlib.client.Completion;
+
+public final class AgentRequestStatistic {
+
+ private final AtomicInteger requests = new AtomicInteger();
+
+ private final AtomicLong inTokens = new AtomicLong();
+
+ private final AtomicLong outTokens = new AtomicLong();
+
+ private final Instant start = Instant.now();
+
+ private volatile Duration requestDuration = Duration.ofMillis(0L);
+
+ public AtomicInteger getRequests() {
+ return this.requests;
+ }
+
+ public AtomicLong getInTokens() {
+ return this.inTokens;
+ }
+
+ public AtomicLong getOutTokens() {
+ return this.outTokens;
+ }
+
+ public Duration getRequestDuration() {
+ return this.requestDuration;
+ }
+
+ public Duration getAbsoluteDuration() {
+ return Duration.between(this.start, Instant.now());
+ }
+
+ public String getTps() {
+ return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get()) / (double)this.getAbsoluteDuration().toMillis() * 1000.0);
+ }
+
+ public String getOutTps() {
+ return new DecimalFormat("#.##").format((double) this.outTokens.get() / (double)this.getAbsoluteDuration().toMillis() * 1000.0);
+ }
+
+ public void add(final Completion completion) {
+ this.getRequests().addAndGet(completion.requests());
+ this.getInTokens().addAndGet(completion.inTokens());
+ this.getOutTokens().addAndGet(completion.outTokens());
+ this.addRequestDuration(completion.duration());
+ }
+
+ public synchronized void addRequestDuration(final Duration duration) {
+ this.requestDuration = this.requestDuration.plus(duration);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("AgentRequestStatistic [requests=");
+ builder.append(this.requests);
+ builder.append(", inTokens=");
+ builder.append(this.inTokens);
+ builder.append(", outTokens=");
+ builder.append(this.outTokens);
+ builder.append(", requestDuration=");
+ builder.append(this.requestDuration);
+ builder.append(", getAbsoluteDuration()=");
+ builder.append(this.getAbsoluteDuration());
+ builder.append(", getTps()=");
+ builder.append(this.getTps());
+ builder.append(", getOutTps()=");
+ builder.append(this.getOutTps());
+ builder.append("]");
+ return builder.toString();
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/agent/AgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/AgentResponseHandler.java
new file mode 100755
index 0000000..f1d20c2
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/AgentResponseHandler.java
@@ -0,0 +1,9 @@
+package de.extio.lmlib.agent;
+
+import de.extio.lmlib.client.Completion;
+
+public interface AgentResponseHandler {
+
+ boolean handle(Split split, Completion completion);
+
+}
diff --git a/src/main/java/de/extio/lmlib/agent/AgentType.java b/src/main/java/de/extio/lmlib/agent/AgentType.java
new file mode 100755
index 0000000..23f6eaf
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/AgentType.java
@@ -0,0 +1,5 @@
+package de.extio.lmlib.agent;
+
+public enum AgentType {
+ COMPLETION, START_CONVERSATION, CONVERSATION;
+}
diff --git a/src/main/java/de/extio/lmlib/agent/JsonAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/JsonAgentResponseHandler.java
new file mode 100755
index 0000000..fa7e2cf
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/JsonAgentResponseHandler.java
@@ -0,0 +1,57 @@
+package de.extio.lmlib.agent;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import de.extio.lmlib.client.Completion;
+
+public class JsonAgentResponseHandler implements AgentResponseHandler {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(JsonAgentResponseHandler.class);
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+
+ @Override
+ public boolean handle(final Split split, final Completion completion) {
+ final int firstCurly = completion.response().indexOf("{");
+ final int lastCurly = completion.response().lastIndexOf("}");
+ final var response = completion.response().substring(firstCurly, lastCurly + 1);
+ JsonNode jsonNode;
+ try {
+ jsonNode = this.objectMapper.readTree(response);
+ }
+ catch (final Exception e) {
+ LOGGER.warn("Cannot parse json response: {}", completion.response());
+ return false;
+ }
+
+ boolean hasResponse = false;
+ final Iterator> fields = jsonNode.fields();
+ while (fields.hasNext()) {
+ final var entry = fields.next();
+ if (entry.getValue().isArray()) {
+ final var items = new HashSet<>();
+ entry.getValue().elements().forEachRemaining(item -> items.add(item.asText()));
+ split.context().getContext().put(entry.getKey(), List.copyOf(items));
+ hasResponse = true;
+ LOGGER.debug("Response: {} {}", entry.getKey(), items);
+ }
+ else if (entry.getValue().isTextual() || entry.getValue().isNumber() || entry.getValue().isBoolean()) {
+ split.context().getContext().put(entry.getKey(), List.of(entry.getValue().asText()));
+ hasResponse = true;
+ LOGGER.debug("Response: {} {}", entry.getKey(), entry.getValue().asText());
+ }
+ }
+
+ return hasResponse;
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/agent/Split.java b/src/main/java/de/extio/lmlib/agent/Split.java
new file mode 100644
index 0000000..3a35bc1
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/Split.java
@@ -0,0 +1,5 @@
+package de.extio.lmlib.agent;
+
+record Split(String text, AgentContext context) {
+
+}
diff --git a/src/main/java/de/extio/lmlib/agent/TextAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/TextAgentResponseHandler.java
new file mode 100755
index 0000000..152058d
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/agent/TextAgentResponseHandler.java
@@ -0,0 +1,27 @@
+package de.extio.lmlib.agent;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.extio.lmlib.client.Completion;
+
+public class TextAgentResponseHandler implements AgentResponseHandler {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TextAgentResponseHandler.class);
+
+ private final String key;
+
+ public TextAgentResponseHandler(final String key) {
+ this.key = key;
+ }
+
+ @Override
+ public boolean handle(final Split split, final Completion completion) {
+ split.context().getContext().put(this.key, List.of(completion.response()));
+ LOGGER.debug("Response: {}", completion.response());
+ return true;
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/Client.java b/src/main/java/de/extio/lmlib/client/Client.java
new file mode 100755
index 0000000..6c7edb2
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/Client.java
@@ -0,0 +1,13 @@
+package de.extio.lmlib.client;
+
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+
+public interface Client {
+
+ Completion completion(ModelCategory modelCategory, String system, String question, String fullText);
+
+ Completion conversation(ModelCategory modelCategory, Conversation conversation);
+
+ ModelProvider getModelProvider();
+}
diff --git a/src/main/java/de/extio/lmlib/client/ClientService.java b/src/main/java/de/extio/lmlib/client/ClientService.java
new file mode 100755
index 0000000..8e8a54f
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/ClientService.java
@@ -0,0 +1,28 @@
+package de.extio.lmlib.client;
+
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfileService;
+
+@Service
+public class ClientService {
+
+ @Autowired
+ private List clients;
+
+ @Autowired
+ private ModelProfileService modelProfileService;
+
+ public Client getClient(final ModelCategory category) {
+ final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile());
+ return this.clients.stream()
+ .filter(client -> client.getModelProvider() == modelProfile.modelProvider())
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("No client found for model profile " + category));
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/Completion.java b/src/main/java/de/extio/lmlib/client/Completion.java
new file mode 100755
index 0000000..e7c8f81
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/Completion.java
@@ -0,0 +1,7 @@
+package de.extio.lmlib.client;
+
+import java.time.Duration;
+
+public final record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, boolean cached) {
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/Conversation.java b/src/main/java/de/extio/lmlib/client/Conversation.java
new file mode 100755
index 0000000..aa11997
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/Conversation.java
@@ -0,0 +1,78 @@
+package de.extio.lmlib.client;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public final class Conversation {
+
+ public static Conversation create(final String system, final String user) {
+ final Conversation conversation = new Conversation();
+ if (system != null && !system.isEmpty()) {
+ conversation.addTurn(new Turn(TurnType.SYSTEM, system));
+ }
+ if (user != null && !user.isEmpty()) {
+ conversation.addTurn(new Turn(TurnType.USER, user));
+ }
+ return conversation;
+ }
+
+ public static Conversation create(final String user) {
+ final Conversation conversation = new Conversation();
+ if (user != null && !user.isEmpty()) {
+ conversation.addTurn(new Turn(TurnType.USER, user));
+ }
+ return conversation;
+ }
+
+ public static Conversation copy(final Conversation conversation) {
+ final Conversation copy = new Conversation();
+ if (conversation != null) {
+ synchronized (conversation.conversation) {
+ copy.conversation.addAll(conversation.conversation);
+ }
+ }
+ return copy;
+ }
+
+ private final List conversation = Collections.synchronizedList(new ArrayList<>());
+
+ private Conversation() {
+ }
+
+ public void addTurn(final Turn turn) {
+ this.conversation.add(turn);
+ }
+
+ public void replaceTurn(final Turn turn) {
+ synchronized (this.conversation) {
+ if (!this.conversation.isEmpty()) {
+ this.conversation.set(this.conversation.size() - 1, turn);
+ }
+ else {
+ this.addTurn(turn);
+ }
+ }
+ }
+
+ public List getConversation() {
+ return Collections.unmodifiableList(this.conversation);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("Conversation [conversation=");
+ builder.append(this.conversation);
+ builder.append("]");
+ return builder.toString();
+ }
+
+ public enum TurnType {
+ SYSTEM, USER, ASSISTANT
+ }
+
+ public record Turn(TurnType type, String text) {
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java b/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java
new file mode 100755
index 0000000..47796cc
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java
@@ -0,0 +1,40 @@
+package de.extio.lmlib.client;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import io.netty.channel.ChannelOption;
+import io.netty.handler.timeout.ReadTimeoutHandler;
+import io.netty.handler.timeout.WriteTimeoutHandler;
+import reactor.netty.http.client.HttpClient;
+
+@Configuration
+public class WebClientConfiguration {
+
+ private static final int TIMEOUT = 900000;
+
+ private static final int CONNECT_TIMEOUT = 10000;
+
+ @Bean
+ @Qualifier("lmLibWebClientBuilder")
+ WebClient.Builder lmLibWebClientBuilder() {
+ final HttpClient httpClient = HttpClient.create()
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
+ .responseTimeout(Duration.ofMillis(TIMEOUT))
+ .doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
+
+ return WebClient.builder()
+ .clientConnector(new ReactorClientHttpConnector(httpClient))
+ .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+ .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
new file mode 100755
index 0000000..b71a27e
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
@@ -0,0 +1,258 @@
+package de.extio.lmlib.client.azureai;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.azure.ai.inference.ChatCompletionsClient;
+import com.azure.ai.inference.ChatCompletionsClientBuilder;
+import com.azure.ai.inference.models.ChatCompletions;
+import com.azure.ai.inference.models.ChatCompletionsOptions;
+import com.azure.ai.inference.models.ChatRequestAssistantMessage;
+import com.azure.ai.inference.models.ChatRequestMessage;
+import com.azure.ai.inference.models.ChatRequestSystemMessage;
+import com.azure.ai.inference.models.ChatRequestUserMessage;
+import com.azure.ai.inference.models.CompletionsFinishReason;
+import com.azure.core.credential.AzureKeyCredential;
+import com.azure.core.http.policy.ExponentialBackoffOptions;
+import com.azure.core.http.policy.RetryOptions;
+
+import de.extio.lmlib.client.Client;
+import de.extio.lmlib.client.Completion;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfile;
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+import de.extio.lmlib.client.profile.ModelProfileService;
+import de.extio.lmlib.client.token.Tokenizer;
+
+public final class AzureAiClient implements Client {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClient.class);
+
+ private final ConcurrentMap chatCompletionsClients = new ConcurrentHashMap<>();
+
+ private final ModelProfileService modelProfileService;
+
+ private final Tokenizer tokenizer;
+
+ AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
+ this.modelProfileService = modelProfileService;
+ this.tokenizer = tokenizer;
+ }
+
+ @Override
+ public ModelProvider getModelProvider() {
+ return ModelProvider.AZURE_AI;
+ }
+
+ @Override
+ public Completion completion(final ModelCategory modelCategory, final String system, final String question, final String fullText) {
+ String text = "";
+ if (!question.isEmpty() && !fullText.isEmpty()) {
+ text = question + "\n" + fullText;
+ }
+ else if (!question.isEmpty()) {
+ text = question;
+ }
+ else if (!fullText.isEmpty()) {
+ text = fullText;
+ }
+ else {
+ throw new IllegalArgumentException("Either question or fullText must be set");
+ }
+
+ return this.conversation(modelCategory, Conversation.create(system, text));
+ }
+
+ @Override
+ public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
+ if (modelCategory == null) {
+ modelCategory = ModelCategory.MEDIUM;
+ }
+ final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
+ if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
+ throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
+ }
+
+ final List> chats = this.createChats(conversation, modelProfile);
+
+ final List answers = new ArrayList<>();
+ final CompletionStatistics statistics = new CompletionStatistics();
+ for (final List chat : chats) {
+ final StringBuilder answer = new StringBuilder();
+ this.requestCompletionsContinuations(chat, answer, statistics, modelCategory, modelProfile);
+ answers.add(answer);
+ }
+
+ // TODO: Reactivate when multiple (split) prompts have been implemented again
+ // if (answers.size() > 1) {
+ // this.summarize(answers, modelProfile, promptStrategy, statistics);
+ // }
+
+ return new Completion(
+ answers.stream().collect(Collectors.joining()),
+ statistics.requests,
+ statistics.duration,
+ statistics.inTokens,
+ statistics.outTokens,
+ false);
+ }
+
+ private List> createChats(final Conversation conversation, final ModelProfile modelProfile) {
+ final List chat = new ArrayList<>();
+
+ final Deque turns = new ArrayDeque<>(conversation.getConversation());
+ Turn systemTurn = null;
+ Turn userTurn = null;
+ Turn assistantTurn = null;
+ boolean first = true;
+
+ final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 50; // 50 is a margin for EOT and other special tokens
+ final AtomicInteger tokens = new AtomicInteger();
+
+ while (!turns.isEmpty()) {
+ final Turn turn = turns.pop();
+ switch (turn.type()) {
+ case SYSTEM:
+ systemTurn = turn;
+ break;
+ case USER:
+ userTurn = turn;
+ break;
+ case ASSISTANT:
+ assistantTurn = turn;
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown turn type: " + turn.type());
+ }
+
+ if (first && userTurn != null) {
+ if (systemTurn != null) {
+ chat.add(new ChatRequestSystemMessage(systemTurn.text()));
+ }
+ if (this.addChatRequestMessage(userTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
+ break;
+ }
+ userTurn = null;
+ first = false;
+ }
+ else if (!first && assistantTurn != null && userTurn != null) {
+ if (this.addChatRequestMessage(assistantTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestAssistantMessage::new)) {
+ break;
+ }
+ if (this.addChatRequestMessage(userTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
+ break;
+ }
+ userTurn = null;
+ assistantTurn = null;
+ }
+ }
+
+ return List.of(chat);
+ }
+
+ private boolean addChatRequestMessage(final String text, final int maxLength, final AtomicInteger tokens, final List chat, final ModelProfile modelProfile, final Function messageFactory) {
+ final int textTokens = this.tokenizer.count(text, modelProfile);
+
+ String trimmedText;
+ boolean trimmed = false;
+ if (tokens.addAndGet(textTokens) > maxLength) {
+ // TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
+ LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokens.get(), maxLength);
+ trimmedText = text.substring(0, text.length() - (tokens.get() - maxLength) / 3);
+ trimmed = true;
+ }
+ else {
+ trimmedText = text;
+ }
+
+ chat.add(messageFactory.apply(trimmedText));
+
+ return trimmed;
+ }
+
+ private void requestCompletionsContinuations(final List chat, final StringBuilder answer, final CompletionStatistics statistics, final ModelCategory modelCategory, final ModelProfile modelProfile) {
+ for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
+ final LocalDateTime start = LocalDateTime.now();
+
+ final var response = this.requestCompletions(chat, modelCategory, modelProfile);
+
+ final var choice = response.getChoice();
+
+ statistics.add(Duration.between(start, LocalDateTime.now()), response.getUsage().getPromptTokens(), response.getUsage().getCompletionTokens());
+
+ answer.append(choice.getMessage().getContent());
+
+ final var complete = !(choice.getFinishReason() == CompletionsFinishReason.TOKEN_LIMIT_REACHED);
+ if (complete) {
+ LOGGER.debug("Prompt response complete");
+ break;
+ }
+
+ answer.append('\n');
+ chat.add(new ChatRequestAssistantMessage(choice.getMessage().getContent()));
+ chat.add(new ChatRequestUserMessage("continue"));
+ }
+ }
+
+ private ChatCompletions requestCompletions(final List chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
+ LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
+
+ final var client = this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
+ final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
+ .setMaxRetries(99)
+ .setBaseDelay(Duration.ofSeconds(1))
+ .setMaxDelay(Duration.ofSeconds(10));
+ final RetryOptions retryOptions = new RetryOptions(exponentialOptions);
+
+ final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
+ .credential(new AzureKeyCredential(modelProfile.apiKey()))
+ .endpoint(modelProfile.url())
+ .retryOptions(retryOptions)
+ .buildClient();
+ return chatCompletionsClient;
+ });
+
+ final var options = new ChatCompletionsOptions(chat);
+ options.setModel(modelProfile.modelName());
+ options.setMaxTokens(modelProfile.maxTokens());
+ options.setTemperature(modelProfile.temperature());
+ options.setTopP(modelProfile.topP());
+
+ return client.complete(options);
+ }
+
+ static class CompletionStatistics {
+
+ int requests;
+
+ Duration duration = Duration.ofMillis(0l);
+
+ long inTokens;
+
+ long outTokens;
+
+ void add(final Duration duration, final int inTokens, final int outTokens) {
+ LOGGER.info("Request duration: {}; in: {}; out: {}; tps: {}", duration, inTokens, outTokens, String.format("%.2f", (inTokens + outTokens) / (double) duration.toMillis() * 1000.0));
+
+ this.requests++;
+ this.duration = this.duration.plus(duration);
+ this.inTokens += inTokens;
+ this.outTokens += outTokens;
+ }
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java
new file mode 100644
index 0000000..a8fdb0a
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java
@@ -0,0 +1,19 @@
+package de.extio.lmlib.client.azureai;
+
+import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.context.annotation.Bean;
+
+import de.extio.lmlib.client.profile.ModelProfileService;
+import de.extio.lmlib.client.token.Tokenizer;
+
+@AutoConfiguration
+@ConditionalOnClass(name = "com.azure.ai.inference.ChatCompletionsClient")
+public class AzureAiClientAutoConfiguration {
+
+ @Bean
+ AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
+ return new AzureAiClient(modelProfileService, tokenizer);
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/profile/ModelCategory.java b/src/main/java/de/extio/lmlib/client/profile/ModelCategory.java
new file mode 100755
index 0000000..df84bca
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/profile/ModelCategory.java
@@ -0,0 +1,27 @@
+package de.extio.lmlib.client.profile;
+
+public enum ModelCategory {
+
+ SMALL("profile.model.s", "S"),
+ MEDIUM("profile.model.m", "M"),
+ LARGE("profile.model.l", "L"),
+ HOT("profile.model.hot", "HOT"),
+ COLD("profile.model.cold", "COLD");
+
+ private final String modelProfile;
+
+ private final String shortName;
+
+ ModelCategory(final String modelProfile, final String shortName) {
+ this.modelProfile = modelProfile;
+ this.shortName = shortName;
+ }
+
+ public String getModelProfile() {
+ return this.modelProfile;
+ }
+
+ public String getShortName() {
+ return this.shortName;
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/profile/ModelProfile.java b/src/main/java/de/extio/lmlib/client/profile/ModelProfile.java
new file mode 100755
index 0000000..ffee661
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/profile/ModelProfile.java
@@ -0,0 +1,19 @@
+package de.extio.lmlib.client.profile;
+
+public record ModelProfile(
+ String prompt,
+ String tokenEncoding,
+ int maxTokens,
+ int maxContextLength,
+ double temperature,
+ double topP,
+ int maxContinuations,
+ ModelProvider modelProvider,
+ String modelName,
+ String url,
+ String apiKey) {
+
+ public static enum ModelProvider {
+ AZURE_AI, OAI_TEXT_COMPLETION;
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/profile/ModelProfileService.java b/src/main/java/de/extio/lmlib/client/profile/ModelProfileService.java
new file mode 100755
index 0000000..5153f22
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/profile/ModelProfileService.java
@@ -0,0 +1,113 @@
+package de.extio.lmlib.client.profile;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.bind.Bindable;
+import org.springframework.boot.context.properties.bind.Binder;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.core.env.Environment;
+import org.springframework.core.io.support.ResourcePropertySource;
+import org.springframework.stereotype.Service;
+
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+
+@Service
+public class ModelProfileService {
+
+ private final Environment environment;
+
+ private static final String MODEL_PROFILE_PREFIX = "profile";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class);
+
+ public ModelProfileService(final Environment environment) {
+ this.environment = environment;
+ }
+
+ @Cacheable("modelProfiles")
+ public ModelProfile getModelProfile(final String modelProfile) {
+ final var modelName = this.environment.getProperty(modelProfile);
+ if (modelName == null) {
+ throw new IllegalStateException("No model found for profile: " + modelProfile);
+ }
+ try {
+ final var resource = new ResourcePropertySource("classpath:" + modelName + ".properties");
+ final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts"))
+ .map(Object::toString)
+ .orElseThrow(() -> new IllegalStateException("No model name found for model: " + modelName));
+ final var tokenEncoding = Optional.ofNullable(resource.getProperty("tokenEncoding"))
+ .map(Object::toString)
+ .orElse("none");
+ final var maxTokens = Optional.ofNullable(resource.getProperty("maxTokens"))
+ .map(Object::toString)
+ .map(Integer::parseInt)
+ .orElseThrow(() -> new IllegalStateException("No model maxTokens found for model: " + modelName));
+ final var maxContextLength = Optional.ofNullable(resource.getProperty("maxContextLength"))
+ .map(Object::toString)
+ .map(Integer::parseInt)
+ .orElseThrow(() -> new IllegalStateException("No model maxContextLength found for model: " + modelName));
+ final var temperature = Optional.ofNullable(resource.getProperty("temperature"))
+ .map(Object::toString)
+ .map(Double::parseDouble)
+ .orElseThrow(() -> new IllegalStateException("No model temperature found for model: " + modelName));
+ final var topP = Optional.ofNullable(resource.getProperty("topP"))
+ .map(Object::toString)
+ .map(Double::parseDouble)
+ .orElseThrow(() -> new IllegalStateException("No model topP found for model: " + modelName));
+ final var maxContinuations = Optional.ofNullable(resource.getProperty("maxContinuations"))
+ .map(Object::toString)
+ .map(Integer::parseInt)
+ .orElseThrow(() -> new IllegalStateException("No model maxContinuations found for model: " + modelName));
+ final var modelProvider = Optional.ofNullable(resource.getProperty("modelProvider"))
+ .map(Object::toString)
+ .map(ModelProvider::valueOf)
+ .orElseThrow(() -> new IllegalStateException("No model modelProvider found for model: " + modelName));
+ final var modelNameCfg = Optional.ofNullable(resource.getProperty("modelName"))
+ .map(Object::toString)
+ .orElseThrow(() -> new IllegalStateException("No model modelName found for model: " + modelName));
+ final var url = Optional.ofNullable(resource.getProperty("url"))
+ .map(Object::toString)
+ .orElseThrow(() -> new IllegalStateException("No model url found for model: " + modelName));
+ final var apiKey = Optional.ofNullable(resource.getProperty("apiKey"))
+ .map(Object::toString)
+ .orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
+
+ return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, maxContinuations, modelProvider, modelNameCfg, url, apiKey);
+ }
+ catch (final IOException e) {
+ LOGGER.error("Error while reading model profile", e);
+ return null;
+ }
+ }
+
+ public List getModelProfileUrls(final ModelProvider modelProvider) {
+ final var modelProfiles = Binder.get(this.environment)
+ .bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
+ .orElse(Collections.emptyMap());
+
+ return modelProfiles.values().stream()
+ .filter(model -> model != null && !model.isEmpty())
+ .map(model -> {
+ try {
+ final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
+ if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
+ return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
+ }
+ return null;
+ }
+ catch (final IOException e) {
+ LOGGER.error("Error while reading model profile for: " + model, e);
+ return null;
+ }
+ })
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/ChatMLPromptStrategy.java b/src/main/java/de/extio/lmlib/client/prompt/ChatMLPromptStrategy.java
new file mode 100755
index 0000000..8a3b23b
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/ChatMLPromptStrategy.java
@@ -0,0 +1,49 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class ChatMLPromptStrategy implements PromptStrategy {
+
+ @Override
+ public StringBuilder start(final String system, final String question, final String text) {
+ final StringBuilder prompt = new StringBuilder();
+ prompt.append("<|im_start|>");
+ if (!system.isEmpty()) {
+ prompt.append("system\n");
+ prompt.append(system);
+ prompt.append("<|im_end|>\n");
+ }
+ prompt.append("<|im_start|>user\n");
+ prompt.append(question);
+ if (!question.isEmpty() && !text.isEmpty()) {
+ prompt.append("\n");
+ }
+ prompt.append(text);
+ prompt.append("<|im_end|>\n<|im_start|>assistant\n");
+ return prompt;
+ }
+
+ @Override
+ public void continue_(final StringBuilder prompt, final String assistant) {
+ this.next(prompt, assistant, "Continue");
+ }
+
+ @Override
+ public void next(final StringBuilder prompt, final String assistant, final String user) {
+ prompt.append(assistant);
+ prompt.append("<|im_end|>\n<|im_start|>user\n");
+ prompt.append(user);
+ prompt.append("<|im_end|>\n<|im_start|>assistant\n");
+ }
+
+ @Override
+ public String removeEOT(final String prompt) {
+ return prompt.strip().replace("<|im_end|>", "");
+ }
+
+ @Override
+ public String getPromptName() {
+ return "chatml";
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/Gemma2PromptStrategy.java b/src/main/java/de/extio/lmlib/client/prompt/Gemma2PromptStrategy.java
new file mode 100755
index 0000000..3150c0d
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/Gemma2PromptStrategy.java
@@ -0,0 +1,48 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class Gemma2PromptStrategy implements PromptStrategy {
+
+ @Override
+ public StringBuilder start(final String system, final String question, final String text) {
+ final StringBuilder prompt = new StringBuilder();
+ prompt.append("user\n");
+ if (!system.isEmpty()) {
+ prompt.append(system);
+ prompt.append("\n");
+ }
+ prompt.append(question);
+ if (!question.isEmpty() && !text.isEmpty()) {
+ prompt.append("\n");
+ }
+ prompt.append(text);
+ prompt.append("\nmodel\n");
+ return prompt;
+ }
+
+ @Override
+ public void continue_(final StringBuilder prompt, final String assistant) {
+ this.next(prompt, assistant, "Continue");
+ }
+
+ @Override
+ public void next(final StringBuilder prompt, final String assistant, final String user) {
+ prompt.append(assistant);
+ prompt.append("\nuser\n");
+ prompt.append(user);
+ prompt.append("\nmodel\n");
+ }
+
+ @Override
+ public String removeEOT(final String prompt) {
+ return prompt.strip().replace("", "");
+ }
+
+ @Override
+ public String getPromptName() {
+ return "gemma2";
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/Llama3PromptStrategy.java b/src/main/java/de/extio/lmlib/client/prompt/Llama3PromptStrategy.java
new file mode 100755
index 0000000..c658cb0
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/Llama3PromptStrategy.java
@@ -0,0 +1,51 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+/**
+ * @see https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
+ */
+@Component
+public class Llama3PromptStrategy implements PromptStrategy {
+
+ @Override
+ public StringBuilder start(final String system, final String question, final String text) {
+ final StringBuilder prompt = new StringBuilder();
+ prompt.append("<|begin_of_text|>");
+ if (!system.isEmpty()) {
+ prompt.append("<|start_header_id|>system<|end_header_id|>\n\n");
+ prompt.append(system);
+ prompt.append("<|eot_id|>");
+ }
+ prompt.append("<|start_header_id|>user<|end_header_id|>\n\n");
+ prompt.append(question);
+ if (!question.isEmpty() && !text.isEmpty()) {
+ prompt.append("\n\n");
+ }
+ prompt.append(text);
+ prompt.append("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
+
+ return prompt;
+ }
+
+ @Override
+ public void continue_(final StringBuilder prompt, final String assistant) {
+ this.next(prompt, assistant, "Continue");
+ }
+
+ @Override
+ public void next(final StringBuilder prompt, final String assistant, final String user) {
+ prompt.append(assistant);
+ prompt.append("<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n" + user + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
+ }
+
+ @Override
+ public String removeEOT(final String prompt) {
+ return prompt.strip().replace("<|eot_id|>", "");
+ }
+
+ @Override
+ public String getPromptName() {
+ return "llama3";
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/MixtralPromptStrategy.java b/src/main/java/de/extio/lmlib/client/prompt/MixtralPromptStrategy.java
new file mode 100755
index 0000000..cff09f9
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/MixtralPromptStrategy.java
@@ -0,0 +1,48 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class MixtralPromptStrategy implements PromptStrategy {
+
+ @Override
+ public StringBuilder start(final String system, final String question, final String text) {
+ final StringBuilder prompt = new StringBuilder();
+ prompt.append("[INST] ");
+ if (!system.isEmpty()) {
+ prompt.append(system);
+ if (!question.isEmpty() || !text.isEmpty()) {
+ prompt.append("\n");
+ }
+ }
+ if (!question.isEmpty()) {
+ prompt.append(question);
+ if (!text.isEmpty()) {
+ prompt.append("\n");
+ }
+ }
+ prompt.append(question);
+ prompt.append(" [/INST]");
+ return prompt;
+ }
+
+ @Override
+ public void continue_(final StringBuilder prompt, final String assistant) {
+ this.next(prompt, assistant, "Continue");
+ }
+
+ @Override
+ public void next(final StringBuilder prompt, final String assistant, final String user) {
+ prompt.append("\n\"");
+ prompt.append(assistant);
+ prompt.append("\" [INST] ");
+ prompt.append(user);
+ prompt.append(" [/INST]");
+ }
+
+ @Override
+ public String getPromptName() {
+ return "mixtral";
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/Phi3PromptStrategy.java b/src/main/java/de/extio/lmlib/client/prompt/Phi3PromptStrategy.java
new file mode 100755
index 0000000..76dcc02
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/Phi3PromptStrategy.java
@@ -0,0 +1,48 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class Phi3PromptStrategy implements PromptStrategy {
+
+ @Override
+ public StringBuilder start(final String system, final String question, final String text) {
+ final StringBuilder prompt = new StringBuilder();
+ prompt.append("<|user|>\n");
+ prompt.append(system);
+ if (!system.isEmpty()) {
+ prompt.append("\n");
+ }
+ prompt.append(question);
+ if (!question.isEmpty() && !text.isEmpty()) {
+ prompt.append("\n");
+ }
+ prompt.append(text);
+ prompt.append(" <|end|>\n<|assistant|>");
+ return prompt;
+ }
+
+ @Override
+ public void continue_(final StringBuilder prompt, final String assistant) {
+ this.next(prompt, assistant, "Continue");
+ }
+
+ @Override
+ public void next(final StringBuilder prompt, final String assistant, final String user) {
+ prompt.append("\n");
+ prompt.append(assistant);
+ prompt.append("<|user|>\n");
+ prompt.append(user);
+ prompt.append("<|end|>\n<|assistant|>");
+ }
+
+ @Override
+ public String removeEOT(final String prompt) {
+ return prompt.strip().replace("<|end|>", "");
+ }
+
+ @Override
+ public String getPromptName() {
+ return "phi3";
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/PromptStrategy.java b/src/main/java/de/extio/lmlib/client/prompt/PromptStrategy.java
new file mode 100755
index 0000000..c410473
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/PromptStrategy.java
@@ -0,0 +1,17 @@
+package de.extio.lmlib.client.prompt;
+
+public interface PromptStrategy {
+
+ String getPromptName();
+
+ StringBuilder start(String system, String question, String text);
+
+ void continue_(StringBuilder prompt, String assistant);
+
+ void next(StringBuilder prompt, String assistant, String user);
+
+ default String removeEOT(final String prompt) {
+ return prompt.strip();
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/PromptStrategyFactory.java b/src/main/java/de/extio/lmlib/client/prompt/PromptStrategyFactory.java
new file mode 100755
index 0000000..0fb018b
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/PromptStrategyFactory.java
@@ -0,0 +1,22 @@
+package de.extio.lmlib.client.prompt;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class PromptStrategyFactory {
+
+ private final Map strategies = new HashMap<>();
+
+ public PromptStrategyFactory(final List strategies) {
+ strategies.forEach(strategy -> this.strategies.put(strategy.getPromptName(), strategy));
+ }
+
+ public PromptStrategy getStrategy(final String modelPrompt) {
+ return this.strategies.get(modelPrompt);
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/prompt/VicunaPromptStrategy.java b/src/main/java/de/extio/lmlib/client/prompt/VicunaPromptStrategy.java
new file mode 100755
index 0000000..6746a02
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/prompt/VicunaPromptStrategy.java
@@ -0,0 +1,46 @@
+package de.extio.lmlib.client.prompt;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class VicunaPromptStrategy implements PromptStrategy {
+
+ @Override
+ public StringBuilder start(final String system, final String question, final String text) {
+ final StringBuilder prompt = new StringBuilder();
+ prompt.append(system);
+ prompt.append(" USER: ");
+ prompt.append(question);
+ if (!question.isEmpty() && !text.isEmpty()) {
+ prompt.append("\n");
+ }
+ prompt.append(text);
+ prompt.append(" ASSISTANT:");
+ return prompt;
+ }
+
+ @Override
+ public void continue_(final StringBuilder prompt, final String assistant) {
+ this.next(prompt, assistant, "Continue");
+ }
+
+ @Override
+ public void next(final StringBuilder prompt, final String assistant, final String user) {
+ prompt.append(" ");
+ prompt.append(assistant);
+ prompt.append("USER: ");
+ prompt.append(user);
+ prompt.append(" ASSISTANT:");
+ }
+
+ @Override
+ public String removeEOT(final String prompt) {
+ return prompt.strip().replace("", "");
+ }
+
+ @Override
+ public String getPromptName() {
+ return "vicuna";
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/ModelNameLoader.java b/src/main/java/de/extio/lmlib/client/textcompletion/ModelNameLoader.java
new file mode 100755
index 0000000..9ea1bdc
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/ModelNameLoader.java
@@ -0,0 +1,65 @@
+package de.extio.lmlib.client.textcompletion;
+
+import java.time.Duration;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.http.HttpMethod;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmlib.client.textcompletion.dto.Model;
+import de.extio.lmlib.client.textcompletion.dto.ModelsResponse;
+
+import reactor.util.retry.Retry;
+
+class ModelNameLoader implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class);
+
+ private final WebClient webClient;
+
+ private final Consumer consumer;
+
+ public ModelNameLoader(@Qualifier("textCompletionWebClientBuilder") final WebClient webClient, final Consumer consumer) {
+ this.consumer = consumer;
+ this.webClient = webClient;
+ }
+
+ @Override
+ public void run() {
+ String modelName = null;
+ do {
+ try {
+ LOGGER.info("Querying model name");
+
+ modelName = this.webClient
+ .method(HttpMethod.GET)
+ .uri(uriBuilder -> uriBuilder.path("/v1/models").build())
+ .retrieve()
+ .bodyToMono(ModelsResponse.class)
+ .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+ .block()
+ .getData()
+ .stream()
+ .map(Model::getId)
+ .findFirst()
+ .orElseThrow();
+ }
+ catch (final Exception e) {
+ LOGGER.error(e.getMessage(), e);
+ try {
+ Thread.sleep(1000l);
+ }
+ catch (final InterruptedException e1) {
+ throw new RuntimeException("Interrupted", e1);
+ }
+ }
+ } while (modelName == null);
+
+ LOGGER.info("Model name: {}", modelName);
+ this.consumer.accept(modelName);
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/ModelNameSupplier.java b/src/main/java/de/extio/lmlib/client/textcompletion/ModelNameSupplier.java
new file mode 100755
index 0000000..b6e72a9
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/ModelNameSupplier.java
@@ -0,0 +1,61 @@
+package de.extio.lmlib.client.textcompletion;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+import de.extio.lmlib.client.profile.ModelProfileService;
+
+@Component
+class ModelNameSupplier implements InitializingBean {
+
+ @Autowired
+ @Qualifier("lmLibWebClientBuilder")
+ private WebClient.Builder webClientBuilder;
+
+ private final Map> modelNames = new ConcurrentHashMap<>();
+
+ private final Map countDownLatches = new ConcurrentHashMap<>();
+
+ @Autowired
+ private ModelProfileService modelProfileService;
+
+ @Override
+ public void afterPropertiesSet() throws Exception {
+ final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION);
+ for (final var url : urls) {
+ this.countDownLatches.put(url, new CountDownLatch(1));
+ this.fetchModelName(url);
+ }
+ }
+
+ private void fetchModelName(final String url) {
+ final var webClient = this.webClientBuilder.baseUrl(url).build();
+ CompletableFuture.runAsync(() -> {
+ final var modelNameLoader = new ModelNameLoader(webClient, modelName -> {
+ this.modelNames.put(url, CompletableFuture.completedFuture(modelName));
+ this.countDownLatches.get(url).countDown();
+ });
+ modelNameLoader.run();
+ });
+ }
+
+ public String getModelName(final String url) {
+ try {
+ this.countDownLatches.get(url).await();
+ }
+ catch (final InterruptedException e) {
+ throw new RuntimeException("Interrupted", e);
+ }
+ return this.modelNames.get(url).join();
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java
new file mode 100755
index 0000000..d431650
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java
@@ -0,0 +1,304 @@
+package de.extio.lmlib.client.textcompletion;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmlib.client.Client;
+import de.extio.lmlib.client.Completion;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfile;
+import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
+import de.extio.lmlib.client.profile.ModelProfileService;
+import de.extio.lmlib.client.prompt.PromptStrategy;
+import de.extio.lmlib.client.prompt.PromptStrategyFactory;
+import de.extio.lmlib.client.textcompletion.dto.Choice;
+import de.extio.lmlib.client.textcompletion.dto.CompletionRequest;
+import de.extio.lmlib.client.textcompletion.dto.CompletionResponse;
+import de.extio.lmlib.client.token.Tokenizer;
+
+import reactor.util.retry.Retry;
+
+@Component
+public class TextCompletionClient implements Client, DisposableBean {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
+
+ @Autowired
+ @Qualifier("lmLibWebClientBuilder")
+ private WebClient.Builder webClientBuilder;
+
+ @Autowired
+ private PromptStrategyFactory promptStrategyFactory;
+
+ @Autowired
+ private ModelNameSupplier modelNameSupplier;
+
+ @Autowired
+ private Tokenizer tokenizer;
+
+ @Autowired
+ private ModelProfileService modelProfileService;
+
+ @Value("${client.collectStatistics:false}")
+ private boolean collectStatistics;
+
+ private final CompletionStatistics totalStatistics = new CompletionStatistics();
+
+ @Override
+ public ModelProvider getModelProvider() {
+ return ModelProvider.OAI_TEXT_COMPLETION;
+ }
+
+ @Override
+ public Completion completion(final ModelCategory modelCategory, final String system, final String question, final String fullText) {
+ String text = "";
+ if (!question.isEmpty() && !fullText.isEmpty()) {
+ text = question + "\n" + fullText;
+ }
+ else if (!question.isEmpty()) {
+ text = question;
+ }
+ else if (!fullText.isEmpty()) {
+ text = fullText;
+ }
+ else {
+ throw new IllegalArgumentException("Either question or fullText must be set");
+ }
+
+ return this.conversation(modelCategory, Conversation.create(system, text));
+ }
+
+ @Override
+ public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
+ if (modelCategory == null) {
+ modelCategory = ModelCategory.MEDIUM;
+ }
+ final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
+ if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION) {
+ throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
+ }
+
+ final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
+ if (promptStrategy == null) {
+ throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
+ }
+
+ final List prompts = this.createPrompts(conversation, modelCategory, modelProfile, promptStrategy);
+
+ final List answers = new ArrayList<>();
+ final CompletionStatistics statistics = new CompletionStatistics();
+ for (final StringBuilder prompt : prompts) {
+ final StringBuilder answer = new StringBuilder();
+ this.requestCompletionsContinuations(prompt, answer, statistics, modelProfile, promptStrategy);
+ answers.add(answer);
+ }
+
+ if (answers.size() > 1) {
+ this.summarize(answers, modelProfile, promptStrategy, statistics);
+ }
+
+ return new Completion(
+ answers.stream().collect(Collectors.joining()),
+ statistics.requests,
+ statistics.duration,
+ statistics.inTokens,
+ statistics.outTokens,
+ false);
+ }
+
+ private List createPrompts(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
+ StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
+
+ List tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
+ final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
+ if (tokenized.size() > maxLength) {
+ // TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
+ LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
+ tokenized = tokenized.subList(0, maxLength);
+ prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
+ }
+
+ LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
+
+ return List.of(prompt);
+ }
+
+ private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
+ final StringBuilder prompt = new StringBuilder();
+
+ final Deque turns = new ArrayDeque<>(conversation.getConversation());
+ Turn systemTurn = null;
+ Turn userTurn = null;
+ Turn assistantTurn = null;
+ boolean first = true;
+
+ while (!turns.isEmpty()) {
+ final Turn turn = turns.pop();
+ switch (turn.type()) {
+ case SYSTEM:
+ systemTurn = turn;
+ break;
+ case USER:
+ userTurn = turn;
+ break;
+ case ASSISTANT:
+ assistantTurn = turn;
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown turn type: " + turn.type());
+ }
+
+ if (first && userTurn != null) {
+ prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", "", userTurn.text()));
+ userTurn = null;
+ first = false;
+ }
+ else if (!first && assistantTurn != null && userTurn != null) {
+ promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
+ userTurn = null;
+ assistantTurn = null;
+ }
+ }
+
+ return prompt;
+ }
+
+ private void requestCompletionsContinuations(final StringBuilder prompt, final StringBuilder answer, final CompletionStatistics statistics,
+ final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
+ for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
+ final LocalDateTime start = LocalDateTime.now();
+
+ final String promptStr = prompt.toString();
+
+ final var response = this.requestCompletions(promptStr, modelProfile);
+
+ String content = null;
+ var complete = false;
+ if (response.getContent() != null) {
+ content = promptStrategy.removeEOT(response.getContent());
+ complete = response.isStoppedEos();
+ }
+ else {
+ final var choice = response.getChoices().getFirst();
+ content = promptStrategy.removeEOT(choice.getText());
+ complete = !Choice.FINISH_REASON_LENGTH.equals(choice.getFinishReason());
+ }
+
+ if (this.collectStatistics) {
+ final var dur = Duration.between(start, LocalDateTime.now());
+ final var in = this.tokenizer.count(promptStr, modelProfile);
+ final var out = this.tokenizer.count(content, modelProfile);
+ LOGGER.debug("Request duration: " + dur + "; in tokens: " + in + "; out tokens: " + out);
+ statistics.add(dur, in, out);
+ this.totalStatistics.add(dur, in, out);
+ }
+
+ answer.append(content);
+
+ if (complete) {
+ LOGGER.debug("Prompt response complete");
+ break;
+ }
+
+ answer.append('\n');
+ promptStrategy.continue_(prompt, content);
+ }
+ }
+
+ private CompletionResponse requestCompletions(final String prompt, final ModelProfile modelProfile) {
+ LOGGER.debug("Requesting completion at {}", modelProfile.url());
+
+ final var request = new CompletionRequest();
+ request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
+ request.setPrompt(prompt);
+ request.setMaxTokens(modelProfile.maxTokens());
+ request.setTemperature(modelProfile.temperature());
+ request.setTopP(modelProfile.topP());
+ request.setStream(false);
+
+ final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
+ return webClient
+ .method(HttpMethod.POST)
+ .uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
+ .bodyValue(request)
+ .retrieve()
+ .bodyToMono(CompletionResponse.class)
+ .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+ .block();
+ }
+
+ private void summarize(final List answers, final ModelProfile modelProfile, final PromptStrategy promptStrategy, final CompletionStatistics statistics) {
+ // TODO: Reactivate when multiple (split) prompts have been implemented again
+ // LOGGER.debug("Summarizing answers");
+ //
+ // final StringBuilder summary = new StringBuilder();
+ //
+ // final StringBuilder prompt = promptStrategy.start(instruction, question, "");
+ // answers.forEach(answer -> promptStrategy.next(prompt, answer.toString(), "Continue"));
+ // promptStrategy.next(prompt, "", "Generate now a full summary");
+ //
+ // this.requestCompletionsContinuations(prompt, summary, statistics, modelProfile, promptStrategy);
+ //
+ // answers.clear();
+ // answers.add(summary);
+ }
+
+ @Override
+ public void destroy() throws Exception {
+ if (this.collectStatistics) {
+ LOGGER.info("Client Statistics: {}", this.totalStatistics);
+ }
+ }
+
+ static class CompletionStatistics {
+
+ volatile int requests;
+
+ volatile Duration duration = Duration.ofMillis(0l);
+
+ volatile long inTokens;
+
+ volatile long outTokens;
+
+ synchronized void add(final Duration duration, final int inTokens, final int outTokens) {
+ this.requests++;
+ this.duration = this.duration.plus(duration);
+ this.inTokens += inTokens;
+ this.outTokens += outTokens;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("CompletionStatistics [requests=");
+ builder.append(this.requests);
+ builder.append(", duration=");
+ builder.append(this.duration);
+ builder.append(", inTokens=");
+ builder.append(this.inTokens);
+ builder.append(", outTokens=");
+ builder.append(this.outTokens);
+ builder.append("]");
+ return builder.toString();
+ }
+
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/dto/Choice.java b/src/main/java/de/extio/lmlib/client/textcompletion/dto/Choice.java
new file mode 100755
index 0000000..41cc2eb
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/dto/Choice.java
@@ -0,0 +1,59 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class Choice {
+
+ public static final String FINISH_REASON_STOP = "stop";
+
+ public static final String FINISH_REASON_LENGTH = "length";
+
+ public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
+
+ private String text;
+
+ private Double score;
+
+ private String finishReason;
+
+ @JsonProperty("text")
+ public String getText() {
+ return this.text;
+ }
+
+ public void setText(final String text) {
+ this.text = text;
+ }
+
+ @JsonProperty("score")
+ public Double getScore() {
+ return this.score;
+ }
+
+ public void setScore(final Double score) {
+ this.score = score;
+ }
+
+ @JsonProperty("finish_reason")
+ public String getFinishReason() {
+ return this.finishReason;
+ }
+
+ public void setFinishReason(final String finishReason) {
+ this.finishReason = finishReason;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("Choice [text=");
+ builder.append(this.text);
+ builder.append(", score=");
+ builder.append(this.score);
+ builder.append(", finishReason=");
+ builder.append(this.finishReason);
+ builder.append("]");
+ return builder.toString();
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionRequest.java b/src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionRequest.java
new file mode 100755
index 0000000..6f7d969
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionRequest.java
@@ -0,0 +1,72 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class CompletionRequest {
+
+ private String model;
+
+ private String prompt;
+
+ private Integer maxTokens;
+
+ private Double temperature;
+
+ private Double topP;
+
+ private Boolean stream;
+
+ @JsonProperty("model")
+ public String getModel() {
+ return this.model;
+ }
+
+ public void setModel(final String model) {
+ this.model = model;
+ }
+
+ @JsonProperty("prompt")
+ public String getPrompt() {
+ return this.prompt;
+ }
+
+ public void setPrompt(final String prompt) {
+ this.prompt = prompt;
+ }
+
+ @JsonProperty("max_tokens")
+ public Integer getMaxTokens() {
+ return this.maxTokens;
+ }
+
+ public void setMaxTokens(final Integer maxTokens) {
+ this.maxTokens = maxTokens;
+ }
+
+ @JsonProperty("temperature")
+ public Double getTemperature() {
+ return this.temperature;
+ }
+
+ public void setTemperature(final Double temperature) {
+ this.temperature = temperature;
+ }
+
+ @JsonProperty("top_p")
+ public Double getTopP() {
+ return this.topP;
+ }
+
+ public void setTopP(final Double topP) {
+ this.topP = topP;
+ }
+
+ @JsonProperty("stream")
+ public Boolean isStream() {
+ return this.stream;
+ }
+
+ public void setStream(final Boolean stream) {
+ this.stream = stream;
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionResponse.java b/src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionResponse.java
new file mode 100755
index 0000000..d9ed1f5
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/dto/CompletionResponse.java
@@ -0,0 +1,93 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class CompletionResponse {
+
+ private String id;
+
+ private String object;
+
+ private Integer created;
+
+ private List choices;
+
+ private String content;
+
+ private boolean isStoppedEos;
+
+ @JsonProperty("id")
+ public String getId() {
+ return this.id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ @JsonProperty("object")
+ public String getObject() {
+ return this.object;
+ }
+
+ public void setObject(final String object) {
+ this.object = object;
+ }
+
+ @JsonProperty("created")
+ public Integer getCreated() {
+ return this.created;
+ }
+
+ public void setCreated(final Integer created) {
+ this.created = created;
+ }
+
+ @JsonProperty("choices")
+ public List getChoices() {
+ return this.choices;
+ }
+
+ public void setChoices(final List choices) {
+ this.choices = choices;
+ }
+
+ @JsonProperty("content")
+ public String getContent() {
+ return this.content;
+ }
+
+ public void setContent(final String content) {
+ this.content = content;
+ }
+
+ @JsonProperty("stopped_eos")
+ public boolean isStoppedEos() {
+ return this.isStoppedEos;
+ }
+
+ public void setStoppedEos(final boolean isStoppedEos) {
+ this.isStoppedEos = isStoppedEos;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("CompletionResponse [id=");
+ builder.append(this.id);
+ builder.append(", object=");
+ builder.append(this.object);
+ builder.append(", created=");
+ builder.append(this.created);
+ builder.append(", choices=");
+ builder.append(this.choices);
+ builder.append(", content=");
+ builder.append(this.content);
+ builder.append(", isStoppedEos=");
+ builder.append(this.isStoppedEos);
+ builder.append("]");
+ return builder.toString();
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/dto/Model.java b/src/main/java/de/extio/lmlib/client/textcompletion/dto/Model.java
new file mode 100755
index 0000000..286331e
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/dto/Model.java
@@ -0,0 +1,74 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class Model {
+
+ private String id;
+
+ private String object;
+
+ private int created;
+
+ private String ownedBy;
+
+ private List permission;
+
+ private String root;
+
+ @JsonProperty("id")
+ public String getId() {
+ return this.id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ @JsonProperty("object")
+ public String getObject() {
+ return this.object;
+ }
+
+ public void setObject(final String object) {
+ this.object = object;
+ }
+
+ @JsonProperty("created")
+ public int getCreated() {
+ return this.created;
+ }
+
+ public void setCreated(final int created) {
+ this.created = created;
+ }
+
+ @JsonProperty("owned_by")
+ public String getOwned_by() {
+ return this.ownedBy;
+ }
+
+ public void setOwned_by(final String owned_by) {
+ this.ownedBy = owned_by;
+ }
+
+ @JsonProperty("permission")
+ public List getPermission() {
+ return this.permission;
+ }
+
+ public void setPermission(final List permission) {
+ this.permission = permission;
+ }
+
+ @JsonProperty("root")
+ public String getRoot() {
+ return this.root;
+ }
+
+ public void setRoot(final String root) {
+ this.root = root;
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/textcompletion/dto/ModelsResponse.java b/src/main/java/de/extio/lmlib/client/textcompletion/dto/ModelsResponse.java
new file mode 100755
index 0000000..9d30e54
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/textcompletion/dto/ModelsResponse.java
@@ -0,0 +1,42 @@
+package de.extio.lmlib.client.textcompletion.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public final class ModelsResponse {
+
+ private String object;
+
+ private List data;
+
+ @JsonProperty("object")
+ public String getObject() {
+ return this.object;
+ }
+
+ public void setObject(final String object) {
+ this.object = object;
+ }
+
+ @JsonProperty("data")
+ public List getData() {
+ return this.data;
+ }
+
+ public void setData(final List data) {
+ this.data = data;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("ModelsResponse [object=");
+ builder.append(this.object);
+ builder.append(", data=");
+ builder.append(this.data);
+ builder.append("]");
+ return builder.toString();
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/token/JTokkitTokenizer.java b/src/main/java/de/extio/lmlib/client/token/JTokkitTokenizer.java
new file mode 100755
index 0000000..1554400
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/token/JTokkitTokenizer.java
@@ -0,0 +1,53 @@
+package de.extio.lmlib.client.token;
+
+import java.util.List;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.knuddels.jtokkit.Encodings;
+import com.knuddels.jtokkit.api.Encoding;
+import com.knuddels.jtokkit.api.EncodingRegistry;
+import com.knuddels.jtokkit.api.EncodingType;
+import com.knuddels.jtokkit.api.IntArrayList;
+
+import de.extio.lmlib.client.profile.ModelProfile;
+
+final class JTokkitTokenizer implements Tokenizer {
+
+ private static final EncodingRegistry registry;
+
+ static {
+ registry = Encodings.newDefaultEncodingRegistry();
+ }
+
+ private final LoadingCache encodings = Caffeine
+ .newBuilder()
+ .build(k -> {
+ return registry.getEncoding(EncodingType.fromName(k).orElseThrow(() -> new IllegalArgumentException("Token encoding " + k)));
+ });
+
+ @Override
+ public List tokenize(final String txt, final ModelProfile modelProfile) {
+ return this.encodings
+ .get(modelProfile.tokenEncoding())
+ .encodeOrdinary(txt)
+ .boxed()
+ .stream()
+ .map(Long::valueOf)
+ .toList();
+ }
+
+ @Override
+ public int count(final String txt, final ModelProfile modelProfile) {
+ return this.encodings.get(modelProfile.tokenEncoding()).countTokensOrdinary(txt);
+ }
+
+ @Override
+ public String detokenize(final List tokens, final ModelProfile modelProfile) {
+ final var ial = new IntArrayList(tokens.size());
+ tokens.forEach(t -> ial.add(t.intValue()));
+ return this.encodings
+ .get(modelProfile.tokenEncoding())
+ .decode(ial);
+ }
+}
diff --git a/src/main/java/de/extio/lmlib/client/token/LlamaServerTokenizer.java b/src/main/java/de/extio/lmlib/client/token/LlamaServerTokenizer.java
new file mode 100755
index 0000000..db196a1
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/token/LlamaServerTokenizer.java
@@ -0,0 +1,75 @@
+package de.extio.lmlib.client.token;
+
+import java.time.Duration;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpMethod;
+import org.springframework.web.reactive.function.client.WebClient;
+import org.springframework.web.reactive.function.client.WebClient.Builder;
+
+import de.extio.lmlib.client.profile.ModelProfile;
+
+import reactor.util.retry.Retry;
+
+final class LlamaServerTokenizer implements Tokenizer {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class);
+
+ private final WebClient.Builder webClientBuilder;
+
+ public LlamaServerTokenizer(final Builder webClientBuilder) {
+ this.webClientBuilder = webClientBuilder;
+ }
+
+ @Override
+ public List tokenize(final String txt, final ModelProfile modelProfile) {
+ LOGGER.debug("Requesting tokenizing");
+ final var request = new TokenizeRequest(txt);
+ final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
+ final var response = webClient
+ .method(HttpMethod.POST)
+ .uri(uriBuilder -> uriBuilder.path("/tokenize").build())
+ .bodyValue(request)
+ .retrieve()
+ .bodyToMono(TokenizeResponse.class)
+ .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+ .block();
+ return response.tokens();
+ }
+
+ @Override
+ public int count(final String txt, final ModelProfile modelProfile) {
+ return this.tokenize(txt, modelProfile).size();
+ }
+
+ @Override
+ public String detokenize(final List tokens, final ModelProfile modelProfile) {
+ LOGGER.debug("Requesting detokenizing");
+ final var request = new DetokenizeRequest(tokens);
+ final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
+ final var response = webClient
+ .method(HttpMethod.POST)
+ .uri(uriBuilder -> uriBuilder.path("/detokenize").build())
+ .bodyValue(request)
+ .retrieve()
+ .bodyToMono(DetokenizeResponse.class)
+ .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed", rs.failure())))
+ .block();
+ return response.content();
+ }
+
+ private record TokenizeRequest(String content) {
+ }
+
+ private record TokenizeResponse(List tokens) {
+ }
+
+ private record DetokenizeRequest(List tokens) {
+ }
+
+ private record DetokenizeResponse(String content) {
+ }
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/token/Tokenizer.java b/src/main/java/de/extio/lmlib/client/token/Tokenizer.java
new file mode 100644
index 0000000..3fd2ff9
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/token/Tokenizer.java
@@ -0,0 +1,15 @@
+package de.extio.lmlib.client.token;
+
+import java.util.List;
+
+import de.extio.lmlib.client.profile.ModelProfile;
+
+public interface Tokenizer {
+
+ List tokenize(String txt, ModelProfile modelProfile);
+
+ int count(String txt, ModelProfile modelProfile);
+
+ String detokenize(List tokens, ModelProfile modelProfile);
+
+}
diff --git a/src/main/java/de/extio/lmlib/client/token/TokenizerAutoConfiguration.java b/src/main/java/de/extio/lmlib/client/token/TokenizerAutoConfiguration.java
new file mode 100644
index 0000000..d7f4fa1
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/token/TokenizerAutoConfiguration.java
@@ -0,0 +1,35 @@
+package de.extio.lmlib.client.token;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.reactive.function.client.WebClient;
+
+@AutoConfiguration
+public class TokenizerAutoConfiguration {
+
+ @Bean
+ @ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer")
+ @ConditionalOnMissingBean
+ Tokenizer llamaServerTokenizer(@Qualifier("lmLibWebClientBuilder") final WebClient.Builder webClientBuilder) {
+ return new LlamaServerTokenizer(webClientBuilder);
+ }
+
+ @Configuration(proxyBeanMethods = false)
+ @ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings")
+ @ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "jTokkit")
+ @ConditionalOnMissingBean
+ public static class SomeServiceConfiguration {
+
+ @Bean
+ Tokenizer jTokkitTokenizer() {
+ return new JTokkitTokenizer();
+ }
+
+ }
+
+}
diff --git a/src/test/java/de/extio/lmlib/agent/AgentTest.java b/src/test/java/de/extio/lmlib/agent/AgentTest.java
new file mode 100644
index 0000000..8bd0ca2
--- /dev/null
+++ b/src/test/java/de/extio/lmlib/agent/AgentTest.java
@@ -0,0 +1,228 @@
+package de.extio.lmlib.agent;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.PropertySource;
+
+import de.extio.lmlib.client.ClientService;
+import de.extio.lmlib.client.profile.ModelCategory;
+
+import io.netty.util.internal.ThreadLocalRandom;
+
+/**
+ * These tests have been verified with Llama 3.1 8B and Gemma2 27B models
+ */
+@SpringBootTest(webEnvironment = WebEnvironment.NONE)
+@SpringBootConfiguration
+@ComponentScan(basePackages = "de.extio.lmlib")
+@PropertySource("classpath:/application-test.properties")
+public class AgentTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AgentTest.class);
+
+ @Autowired
+ private AgentExecutorService agentExecutor;
+
+ @Autowired
+ private ClientService clientService;
+
+ @Test
+ void agenticFlow() throws IOException {
+ final var agents = Map.of(
+ "CodeSummarizer",
+ new Agent("CodeSummarizer",
+ AgentType.START_CONVERSATION,
+ ModelCategory.MEDIUM,
+ "Generate a summary of the following Java source code.",
+ "The source code is:\n{{code}}",
+ new TextAgentResponseHandler("summary"),
+ null,
+ null,
+ null,
+ context -> new AgentNext("FeatureIdentifier", null, null, null)),
+
+ "FeatureIdentifier",
+ new Agent("FeatureIdentifier",
+ AgentType.CONVERSATION,
+ ModelCategory.MEDIUM,
+ "",
+ "Generate a list of 5 distinct features. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"features\": [\"Feature 1\", \"Feature 2\", ... ] }",
+ new JsonAgentResponseHandler(),
+ null,
+ context -> context.getContext().get("features").forEach(feature -> LOGGER.info(feature.toString())),
+ null,
+ context -> new AgentNext("FeatureAnalyzer", null, null, null)),
+
+ "FeatureAnalyzer",
+ new Agent("FeatureAnalyzer",
+ AgentType.COMPLETION,
+ ModelCategory.MEDIUM,
+ "Analyze a feature of the following Java source code and describe the functionality and purpose of this feature. Only describe the implementation, don't suggest any possible improvements.",
+ """
+ The source code is:
+ {{code}}
+ The feature is: [[features]]""",
+ new TextAgentResponseHandler("featureDescription"),
+ null,
+ null,
+ contexts -> Agent.mergeContexts(List.of("features", "featureDescription"), contexts),
+ context -> new AgentNext("AnswerGenerator", null, null, null)),
+
+ "AnswerGenerator",
+ new Agent("AnswerGenerator",
+ AgentType.COMPLETION,
+ ModelCategory.MEDIUM,
+ "Generate a clear and concise blog post for an announcement from the following code analysis.",
+ """
+ Write a concise description of the purpose in the blog post first and then concise information for the following topics: {{topics}}
+ The code analysis:
+ {{summary}}
+ {{featureDescription}}""",
+ new TextAgentResponseHandler("blogPost"),
+ context -> {
+ context.getContext().put("topics", List.of(String.join(", ", context.getContext().get("features").stream().map(String::valueOf).toArray(size -> new String[size]))));
+ },
+ null,
+ null,
+ context -> new AgentNext(null, null, null, null)));
+
+ final var context = new AgentContext(agents);
+ context.getContext().put("code", List.of(Files.readString(Path.of("src/main/java/de/extio/lmlib/agent/Agent.java"))));
+
+ final var resultContexts = this.agentExecutor.walkGraph(agents.get("CodeSummarizer"), context);
+
+ final var resultContext = resultContexts.getFirst();
+ LOGGER.info(resultContext.getContext().get("blogPost").toString() + "\n\n" + resultContext.getGraph().toString() + "\n" + resultContext.getRequestStatistic().toString());
+
+ assertEquals(1, resultContexts.size());
+ assertEquals(5, resultContext.getContext().get("features").size());
+ assertEquals(1, resultContext.getContext().get("blogPost").size());
+ assertEquals(8, resultContext.getRequestStatistic().getRequests().get());
+ for (final var feature : resultContext.getContext().get("features")) {
+ assertTrue(this.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString()));
+ }
+ }
+
+ @Test
+ void doubleBranch() {
+ final var agents = Map.of(
+ "FirstLevel",
+ new Agent("FirstLevel",
+ AgentType.COMPLETION,
+ ModelCategory.MEDIUM,
+ "",
+ "Generate 5 random numbers from 1 to 10. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"numbers\": [\"Number 1\", \"Number 2\", ... ] }",
+ new JsonAgentResponseHandler(),
+ null,
+ null,
+ null,
+ context -> new AgentNext("SecondLevel", null, null, null)),
+
+ "SecondLevel",
+ new Agent("SecondLevel",
+ AgentType.COMPLETION,
+ ModelCategory.MEDIUM,
+ "",
+ "Generate 5 random numbers from 1 to 10. The previous number was [[numbers]]. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"random\": [\"Number 1\", \"Number 2\", ... ] }",
+ new JsonAgentResponseHandler(),
+ null,
+ null,
+ null,
+ context -> new AgentNext("Calculator", null, null, null)),
+
+ "Calculator",
+ new Agent("Calculator",
+ AgentType.COMPLETION,
+ ModelCategory.MEDIUM,
+ "",
+ "Add these 2 numbers: {{numbers}} + [[random]] ; Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"result\": \"result\" }",
+ new JsonAgentResponseHandler(),
+ null,
+ null,
+ null,
+ context -> new AgentNext("Grader", null, null, null)),
+
+ "Grader",
+ new Agent("Grader",
+ AgentType.COMPLETION,
+ ModelCategory.MEDIUM,
+ "You are a helpful assistant.",
+ "If you add {{numbers}} marbles to {{random}} marbles, do you get {{result}} marbles as the result? Answer with 'yes' or 'no'",
+ new TextAgentResponseHandler("grade"),
+ null,
+ null,
+ null,
+ context -> {
+ final boolean pass = context.getContext().get("grade").getFirst().toString().strip().toLowerCase().equals("yes");
+ if (!pass) {
+ try {
+ Integer.parseInt(context.getContext().get("numbers").getFirst().toString().strip());
+ Integer.parseInt(context.getContext().get("random").getFirst().toString().strip());
+ }
+ catch (final NumberFormatException exc) {
+ context.getContext().put("numbers", List.of(String.valueOf(ThreadLocalRandom.current().nextInt(10))));
+ context.getContext().put("random", List.of(String.valueOf(ThreadLocalRandom.current().nextInt(10))));
+ }
+ }
+ return new AgentNext(pass ? null : "Calculator", pass, null, null);
+ }));
+
+ final var context = new AgentContext(agents);
+
+ final var resultContexts = this.agentExecutor.walkGraph(agents.get("FirstLevel"), context);
+
+ assertEquals(25, resultContexts.size());
+ assertTrue(resultContexts.getFirst().getRequestStatistic().getRequests().get() >= 56);
+ for (final var resultContext : resultContexts) {
+ final int number0 = Integer.parseInt(resultContext.getContext().get("numbers").getFirst().toString().strip());
+ final int number1 = Integer.parseInt(resultContext.getContext().get("random").getFirst().toString().strip());
+ final int expected = number0 + number1;
+ assertEquals(expected, Integer.parseInt(resultContext.getContext().get("result").getFirst().toString().strip()));
+ LOGGER.info(number0 + " + " + number1 + " = " + expected + "; " + resultContext.getGraph().toString() + " " + resultContext.getRequestStatistic().toString());
+ }
+ }
+
+ private boolean assessScoreBinary(final String question, final String text) {
+ LOGGER.info(question);
+
+ final var client = this.clientService.getClient(ModelCategory.MEDIUM);
+
+ var score = 0;
+ while (Math.abs(score) < 3) {
+ final var completion = client.completion(
+ ModelCategory.MEDIUM,
+ """
+ You are a grader assessing the truthfulness of a given text to a user question.
+ Please provide a binary response 'true' or 'false' for the following text.
+ 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
+ Only provide the response as a single word and no preamble and no explanation.""",
+ "Here is the text:\n" + text,
+ "\nHere is the user question: " + question);
+ LOGGER.info(completion.response());
+ score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
+ if (Math.abs(score) == 2) {
+ break;
+ }
+ }
+
+ final var result = score > 0;
+ LOGGER.info("->" + String.valueOf(result));
+ return result;
+ }
+
+}
diff --git a/src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java b/src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java
new file mode 100755
index 0000000..e24d2ad
--- /dev/null
+++ b/src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java
@@ -0,0 +1,71 @@
+package de.extio.lmlib.client.azureai;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.PropertySource;
+
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.Conversation.TurnType;
+import de.extio.lmlib.client.profile.ModelCategory;
+
+@Disabled
+@SpringBootTest(webEnvironment = WebEnvironment.NONE)
+@SpringBootConfiguration
+@ComponentScan(basePackages = "de.extio.lmlib")
+@PropertySource("classpath:/application-test.properties")
+public class AzureAiClientTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClientTest.class);
+
+ @Autowired
+ private AzureAiClient client;
+
+ @Test
+ void completion() throws Exception {
+ final var completion = this.client.completion(
+ ModelCategory.MEDIUM,
+ "You are a clever story teller",
+ "Continue this story:",
+ "Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
+ LOGGER.info(completion.response());
+ }
+
+ @Test
+ void completionLarge() throws Exception {
+ final var completion = this.client.completion(
+ ModelCategory.LARGE,
+ "You are a clever story teller",
+ "Continue this story:",
+ "Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
+ LOGGER.info(completion.response());
+ }
+
+ @Test
+ void chat() throws Exception {
+ final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Berlin.");
+ conversation.addTurn(new Turn(TurnType.ASSISTANT,
+ "T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius."));
+ conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?"));
+
+ final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation);
+ LOGGER.info(completion.response());
+ }
+
+ @Test
+ void longPrompt() throws Exception {
+ final var completion = this.client.completion(
+ null,
+ "You are a helpful assistant",
+ "",
+ "Count the numbers: 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9");
+ LOGGER.info(completion.response());
+ }
+}
diff --git a/src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java b/src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java
new file mode 100755
index 0000000..42e93ae
--- /dev/null
+++ b/src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java
@@ -0,0 +1,198 @@
+package de.extio.lmlib.client.textcompletion;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.LocalDateTime;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.PropertySource;
+
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.Conversation.TurnType;
+import de.extio.lmlib.client.profile.ModelCategory;
+
+@SpringBootTest(webEnvironment = WebEnvironment.NONE)
+@SpringBootConfiguration
+@ComponentScan(basePackages = "de.extio.lmlib")
+@PropertySource("classpath:/application-test.properties")
+public class TextCompletionClientTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClientTest.class);
+
+ @Autowired
+ private TextCompletionClient client;
+
+ @Test
+ void completion() throws Exception {
+ final var completion = this.client.completion(
+ null,
+ "You are a helpful assistant",
+ "",
+ "Say the color is green");
+
+ LOGGER.info(completion.response());
+
+ assertTrue(this.assessScoreBinary("Does the text mention the color green?", completion.response()));
+ }
+
+ @Test
+ void chat() throws Exception {
+ final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
+ conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
+ conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
+ final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation);
+
+ LOGGER.info(completion.response());
+
+ assertTrue(this.assessScoreBinary("Does the text mention a color?", completion.response()));
+ }
+
+ @Disabled
+ @Test
+ void hugePrompt() throws Exception {
+ final StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 10000; i++) {
+ sb.append(i);
+ sb.append(", ");
+ }
+ sb.delete(sb.length() - 2, sb.length());
+
+ final var completion = this.client.completion(
+ null,
+ "You are counting machine",
+ "How many numbers do you count?",
+ sb.toString());
+ LOGGER.info(completion.response());
+ }
+
+ @Disabled
+ @Test
+ void parallelRequests() throws Exception {
+ final LocalDateTime start = LocalDateTime.now();
+
+ final Tasks tasks = this.createTasks();
+
+ final ExecutorService executor = Executors.newFixedThreadPool(4);
+ try {
+ CompletableFuture.allOf(
+ CompletableFuture.runAsync(tasks.task1, executor),
+ CompletableFuture.runAsync(tasks.task2, executor),
+ CompletableFuture.runAsync(tasks.task3, executor),
+ CompletableFuture.runAsync(tasks.task4, executor))
+ .join();
+ }
+ finally {
+ executor.shutdown();
+ }
+
+ LOGGER.info("Duration: " + java.time.Duration.between(start, LocalDateTime.now()));
+ }
+
+ @Disabled
+ @Test
+ void serialRequests() throws Exception {
+ final LocalDateTime start = LocalDateTime.now();
+
+ final Tasks tasks = this.createTasks();
+
+ tasks.task1.run();
+ tasks.task2.run();
+ tasks.task3.run();
+ tasks.task4.run();
+
+ LOGGER.info("Duration: " + java.time.Duration.between(start, LocalDateTime.now()));
+ }
+
+ private Tasks createTasks() {
+ return new Tasks(() -> {
+ try {
+ final var completion = this.client.completion(
+ null,
+ "You are a helpful assistant",
+ "",
+ "How do I calculate the annual profit margin?");
+ LOGGER.info(completion.response());
+ }
+ catch (final Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ }, () -> {
+ try {
+ final var completion = this.client.completion(
+ null,
+ "You are a helpful assistant",
+ "",
+ "How do I calculate the operating profit ratio?");
+ LOGGER.info(completion.response());
+ }
+ catch (final Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ }, () -> {
+ try {
+ final var completion = this.client.completion(
+ null,
+ "You are a helpful assistant",
+ "",
+ "How do I calculate the return of investment?");
+ LOGGER.info(completion.response());
+ }
+ catch (final Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ }, () -> {
+ try {
+ final var completion = this.client.completion(
+ null,
+ "You are a helpful assistant",
+ "",
+ "How do I calculate the return on net worth?");
+ LOGGER.info(completion.response());
+ }
+ catch (final Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ });
+ }
+
+ static record Tasks(Runnable task1, Runnable task2, Runnable task3, Runnable task4) {
+ }
+
+ private boolean assessScoreBinary(final String question, final String text) {
+ LOGGER.info(question);
+
+ var score = 0;
+ while (Math.abs(score) < 3) {
+ final var completion = this.client.completion(
+ ModelCategory.MEDIUM,
+ """
+ You are a grader assessing the truthfulness of a given text to a user question.
+ Please provide a binary response 'true' or 'false' for the following text.
+ 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
+ Only provide the response as a single word and no preamble and no explanation.""",
+ "Here is the text:\n" + text,
+ "\nHere is the user question: " + question);
+ LOGGER.info(completion.response());
+ score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
+ if (Math.abs(score) == 2) {
+ break;
+ }
+ }
+
+ final var result = score > 0;
+ LOGGER.info("->" + String.valueOf(result));
+ return result;
+ }
+}
diff --git a/src/test/java/de/extio/lmlib/client/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/client/token/TokenizerTest.java
new file mode 100644
index 0000000..58ba523
--- /dev/null
+++ b/src/test/java/de/extio/lmlib/client/token/TokenizerTest.java
@@ -0,0 +1,57 @@
+package de.extio.lmlib.client.token;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.web.reactive.function.client.WebClient;
+
+import de.extio.lmlib.client.profile.ModelCategory;
+import de.extio.lmlib.client.profile.ModelProfileService;
+
+@SpringBootTest(webEnvironment = WebEnvironment.NONE)
+@SpringBootConfiguration
+@ComponentScan(basePackages = "de.extio.lmlib")
+@PropertySource("classpath:/application-test.properties")
+public class TokenizerTest {
+
+ private static final String TEXT = "Hello this is a text! How are you doing? The weather is great today! Super sunny shiny!!!";
+
+ @Autowired
+ private ModelProfileService modelProfileService;
+
+ @Autowired
+ @Qualifier("lmLibWebClientBuilder")
+ private WebClient.Builder webClientBuilder;
+
+ @Test
+ void jTokkit() throws Exception {
+ final var tokenizer = new JTokkitTokenizer();
+ final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
+
+ assertEquals(21, tokenizer.count(TEXT, profile));
+ final var tokens = tokenizer.tokenize(TEXT, profile);
+ assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString());
+ assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
+ }
+
+ /**
+ * Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model.
+ */
+ @Test
+ void llamaServer() throws Exception {
+ final var tokenizer = new LlamaServerTokenizer(this.webClientBuilder);
+ final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
+
+ assertEquals(21, tokenizer.count(TEXT, profile));
+ final var tokens = tokenizer.tokenize(TEXT, profile);
+ assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString());
+ assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
+ }
+}
diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties
new file mode 100755
index 0000000..0def87f
--- /dev/null
+++ b/src/test/resources/application-test.properties
@@ -0,0 +1,11 @@
+profile.model.s=llama3-local
+profile.model.m=llama3-local
+profile.model.l=llama3-local
+profile.model.hot=llama3-local
+profile.model.cold=llama3-local
+
+agent.threads=4
+
+client.collectStatistics=true
+
+tokenizer.strategy=llamaServer
diff --git a/src/test/resources/llama3-local.properties b/src/test/resources/llama3-local.properties
new file mode 100755
index 0000000..f56566c
--- /dev/null
+++ b/src/test/resources/llama3-local.properties
@@ -0,0 +1,11 @@
+prompts=llama3
+tokenEncoding=cl100k_base
+maxTokens=2000
+maxContextLength=8000
+temperature=0.2
+topP=1.0
+maxContinuations=1
+modelProvider=OAI_TEXT_COMPLETION
+modelName=
+url=http://localhost:5001
+apiKey=
diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml
new file mode 100644
index 0000000..ed0f24b
--- /dev/null
+++ b/src/test/resources/logback.xml
@@ -0,0 +1,5 @@
+
+
+
+
+