From 4ff231227eed74a99e838682383d084dc5860cc4 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 9 May 2026 17:47:28 +0200 Subject: [PATCH] feat: Support OLLAMA provider - Added OllamaClient to handle conversations with the Ollama API. - Introduced OllamaDialect for managing thinking modes. - Created data classes: OllamaFunctionCall, OllamaListedModel, OllamaMessage, OllamaShowRequest, OllamaShowResponse, OllamaTagsResponse, OllamaToolCall, OllamaToolDefinition, and OllamaToolFunction for structured API communication. - Developed unit tests for OllamaClient to validate model listing, chat completion, streaming responses, and tool calling capabilities. - Configured test properties for local Ollama instance. --- README.md | 71 ++- .../java/de/extio/lmlib/agent/BaseAgent.java | 13 +- .../java/de/extio/lmlib/client/Client.java | 4 +- .../lmlib/client/ClientAutoConfiguration.java | 7 + .../de/extio/lmlib/client/ClientService.java | 13 +- .../de/extio/lmlib/client/Conversation.java | 3 +- .../lmlib/client/cached/CachedClient.java | 9 +- .../completion/chat/ChatCompletionClient.java | 11 +- .../completion/text/TextCompletionClient.java | 8 +- .../client/ollama/OllamaChatRequest.java | 75 +++ .../client/ollama/OllamaChatResponse.java | 174 ++++++ .../lmlib/client/ollama/OllamaClient.java | 577 ++++++++++++++++++ .../lmlib/client/ollama/OllamaDialect.java | 35 ++ .../client/ollama/OllamaFunctionCall.java | 39 ++ .../client/ollama/OllamaListedModel.java | 28 + .../lmlib/client/ollama/OllamaMessage.java | 74 +++ .../client/ollama/OllamaShowRequest.java | 28 + .../client/ollama/OllamaShowResponse.java | 19 + .../client/ollama/OllamaTagsResponse.java | 19 + .../lmlib/client/ollama/OllamaToolCall.java | 39 ++ .../client/ollama/OllamaToolDefinition.java | 28 + .../client/ollama/OllamaToolFunction.java | 41 ++ .../de/extio/lmlib/profile/ModelProfile.java | 2 +- .../lmlib/agent/AgentErrorHandlingTest.java | 4 +- .../java/de/extio/lmlib/agent/AgentTest.java | 6 +- .../lmlib/client/cached/CachedClientTest.java | 4 +- .../client/oai/CompletionClientTest.java | 12 +- .../lmlib/client/ollama/OllamaClientTest.java | 152 +++++ .../de/extio/lmlib/grader/GraderTest.java | 4 +- .../de/extio/lmlib/token/TokenizerTest.java | 7 +- .../resources/application-test.properties | 4 + .../local-ollama-chatcompletion.properties | 16 + src/test/resources/logback.xml | 2 +- 33 files changed, 1468 insertions(+), 60 deletions(-) create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaChatRequest.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaChatResponse.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaDialect.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaFunctionCall.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaListedModel.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaMessage.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaShowRequest.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaShowResponse.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaTagsResponse.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaToolCall.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaToolDefinition.java create mode 100644 src/main/java/de/extio/lmlib/client/ollama/OllamaToolFunction.java create mode 100644 src/test/java/de/extio/lmlib/client/ollama/OllamaClientTest.java create mode 100644 src/test/resources/local-ollama-chatcompletion.properties diff --git a/README.md b/README.md index 482230c..ba07e19 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,13 @@ lm-lib is published under the Apache License 2.0. lm-lib is designed for applications that need more than single prompt-response calls. Its center of gravity is the agent framework: structured multi-step flows such as classification, planning, fan-out analysis, synthesis, grading, feedback loops, and other graph-based execution patterns. -It also supports direct completion use cases through a client abstraction layer. The built-in clients currently target OpenAI-compatible endpoints, including local servers that expose the same API shape, but the library is intentionally structured so additional client implementations can be added over time. Applications can also provide their own client implementations by implementing the relevant interfaces. +It also supports direct completion use cases through a client abstraction layer. The built-in clients currently cover OpenAI-compatible chat and text completion endpoints as well as Ollama's native chat API, and the abstraction remains open for additional provider implementations over time. Applications can also provide their own client implementations by implementing the relevant interfaces. -The current built-in client set supports both chat completions and legacy text completions, with model-specific prompt strategies for text completion models. +The current built-in client set supports OpenAI-compatible chat completions, legacy text completions, and native Ollama chat completions, with model-specific prompt strategies for text completion models. ## Key Capabilities -- OpenAI-compatible chat and text completions +- OpenAI-compatible chat and text completions plus native Ollama chat completions - Chat-completion tool calling with tool definitions, returned tool calls, and tool-result follow-up rounds - Streaming and non-streaming APIs - Separate reasoning capture for models that expose reasoning output @@ -240,7 +240,7 @@ Each profile file is a `.properties` resource on the classpath. | `maxContextLength` | Integer | Yes | Maximum total context length. | `16000`, `32768`, `128000` | | `temperature` | Double | Yes | Sampling temperature. Must be greater than zero. | `0.2`, `0.4`, `0.7` | | `topP` | Double | Yes | Nucleus sampling parameter. Must be greater than zero. | `1.0`, `0.9` | -| `modelProvider` | Enum | Yes | Completion client type. | `OAI_CHAT_COMPLETION`, `OAI_TEXT_COMPLETION` | +| `modelProvider` | Enum | Yes | Completion client type. | `OAI_CHAT_COMPLETION`, `OAI_TEXT_COMPLETION`, `OLLAMA` | | `modelName` | String | No | Model name or deployment. Can be empty for endpoints where the model should be auto-detected. | `gpt-4`, `Llama-3.3-70B-Instruct`, `` | | `url` | String | Yes | Base API URL. | `https://api.openai.com/v1`, `http://localhost:5001` | | `apiKey` | String | No | Bearer token or Spring placeholder. Can be empty for local servers. | `sk-...`, `${apikey}`, `` | @@ -298,6 +298,27 @@ cost1MInTokens=0 cost1MOutTokens=0 ``` +Local Ollama chat completion: + +```properties +category=M +prompts= +tokenizer=jtokkit +tokenEncoding=cl100k_base +maxTokens=2500 +maxContextLength=32768 +temperature=0.4 +topP=1.0 +modelProvider=OLLAMA +modelName= +url=http://localhost:11434 +apiKey= +cost1MInTokens=0 +cost1MOutTokens=0 +``` + +Direct Ollama cloud access uses the same provider with `url=https://ollama.com/api` and a Bearer token in `apiKey`. Local Ollama access usually does not require authentication. + ## Client Implementations lm-lib exposes a client abstraction intended to support multiple backend implementations. @@ -307,17 +328,25 @@ The built-in provider implementations currently support these types: ```properties modelProvider=OAI_CHAT_COMPLETION modelProvider=OAI_TEXT_COMPLETION +modelProvider=OLLAMA ``` -Chat completions are the preferred default. They are simpler to use because the library only needs to send the conversation. Text completions are still useful when you need explicit prompt formatting or must work with instruct models that depend on raw prompt structure. +### OpenAI-Compatible Providers -Tool calling is currently supported only for chat completion models. `TextCompletionClient` does not support tools, and `supportsToolCalling()` can be used on `Client`, `ClientService`, and `BaseAgent` to detect availability before enabling tool-aware flows. +`OAI_CHAT_COMPLETION` and `OAI_TEXT_COMPLETION` target the OpenAI-style API surface. -These built-in providers are OpenAI-compatible today, but that is an implementation detail of the current client set, not a hard architectural limit of the library. Additional provider integrations can be added in the library itself, and applications can plug in their own client implementations where needed. +Use `OAI_CHAT_COMPLETION` for cloud APIs such as OpenAI itself and for local or self-hosted servers that expose a compatible `/v1/chat/completions` interface. This is the preferred default because lm-lib can send the conversation directly, supports streaming, and exposes tool calls through the standard chat-completion flow. + +Use `OAI_TEXT_COMPLETION` when you need explicit prompt templating or must target instruct-style models that still depend on raw prompt construction rather than structured chat messages. This is mainly useful for older compatible backends or local servers configured around prompt templates. + +OpenAI-compatible deployments commonly fall into two operational patterns: + +- Cloud endpoints: hosted APIs with a fixed base URL such as `https://api.openai.com/v1`, normally with a Bearer API key. +- Local or self-hosted endpoints: llama-server, vLLM-compatible gateways, or other OpenAI-style frontends exposed on an internal URL such as `http://localhost:5001`. If you need to adapt request shaping for a specific OpenAI-compatible provider, you can optionally register an `OpenAiProviderDialect` bean. This allows applications to suppress or switch request fields that a provider does not support while keeping `ModelProfile` provider-agnostic and runtime-driven. -The default dialect behavior is: +The default OpenAI dialect behavior is: - `sendUsage(...) = false` - `sendReasoning(...) = true` @@ -360,17 +389,23 @@ OpenAiProviderDialect legacyOpenAiProviderDialect() { } ``` -If no `OpenAiProviderDialect` bean is present, lm-lib follows the official-oriented defaults above, with `usage` left disabled unless you opt in. +### Ollama Provider -For text completions, select a prompt strategy with the `prompts` property. lm-lib ships built-in strategies for common model families including Llama, GPT-OSS, Gemma, Mistral, Phi, Qwen, Vicuna, ChatML, Alpaca, and a no-formatting strategy. +`OLLAMA` targets Ollama's native API surface rather than the OpenAI-compatible transport. -Example: +The Ollama provider uses `/api/chat` for completions, `/api/tags` for model discovery, and `/api/show` for capability discovery. It supports local Ollama installations at `http://localhost:11434` as well as direct cloud access at `https://ollama.com/api` with Bearer authentication. -```properties -prompts=llama3 -``` +If you need to shape Ollama thinking behavior, you can optionally register an `OllamaDialect` bean. Its `think(...)` hook controls whether lm-lib requests separate thinking output and whether it uses the generic boolean mode or Ollama's string levels such as `low`, `medium`, or `high`. -Use `jTokkit` or `llamaServer` for tokenization depending on your deployment model. +lm-lib caches `/api/show` model details per resolved model. Thinking and tool definitions are sent only when the model advertises the corresponding capability, and `supportsToolCalling(modelProfile)` probes the same cached capability data. + +### Tool Calling Support + +Tool calling is exposed through the same lm-lib API across providers that support it. The capability is model-aware rather than purely provider-wide, so callers should check `supportsToolCalling(modelProfile)` or `supportsToolCalling(modelCategory)` on `Client` or `ClientService` before enabling tool-aware flows. + +In practice, text-completion clients do not support tools, while chat-oriented clients may support them depending on the resolved backend and model. For Ollama, lm-lib probes cached `/api/show` capability metadata. For OpenAI-compatible chat providers, support is determined by the selected client type. + +The library keeps the caller-facing tool-calling flow uniform even when providers differ internally. You still pass `ToolDefinition` and `ToolCallData`, inspect `CompletionFinishReason.TOOL_CALLS`, and append tool results through the same conversation API. If a backend does not expose every request-side control, lm-lib keeps the public API stable and applies those controls on a best-effort basis. ## Usage @@ -391,7 +426,7 @@ final var completion = client.conversation(ModelCategory.MEDIUM, conversation, n ### Tool Calling -Tool calling lets chat completion models return one or more structured tool invocations instead of a final natural-language answer. The library exposes this in both the direct client API and the agent runtime. +Tool calling lets chat completion models return one or more structured tool invocations instead of a final natural-language answer. The library exposes this in both the direct client API and the agent runtime across OpenAI-compatible chat providers and the native Ollama client. The main tool-calling types are: @@ -446,7 +481,7 @@ if (completion.finishReason() == CompletionFinishReason.TOOL_CALLS) { - `ToolCallData.auto(...)`: model may decide whether to call a tool - `ToolCallData.required(...)`: model must call one or more of the provided tools - `ToolCallData.force(...)`: model must call a specific named tool -- `toolCallData.withParallelToolCalls(true)`: asks OpenAI-compatible backends for multiple independent tool calls in one turn when the server and model support `parallel_tool_calls` +- `toolCallData.withParallelToolCalls(true)`: asks the backend for multiple independent tool calls in one turn when the provider exposes such a control; providers that do not expose a parallel flag treat this as a best-effort no-op while still returning multiple tool calls when the model emits them If a model returns multiple tool calls, append one assistant turn containing the returned `toolCalls()`, then append one tool turn per returned `tool_call_id` before requesting the follow-up completion. @@ -465,7 +500,7 @@ Both streaming and non-streaming execution paths are supported. If a chat completion decides to call a tool, `finishReason()` is `CompletionFinishReason.TOOL_CALLS` and `toolCalls()` contains the requested function calls. Streaming chat completions also aggregate tool calls into the final `Completion`, including multi-tool responses when `parallel_tool_calls` is enabled and supported by the backend. -This lets you stream visible output while still capturing final structured accounting data. +Provider-specific streaming formats are normalized into the same `Chunk` and `Completion` objects. This lets you stream visible output while still capturing final structured accounting data, regardless of whether the backend is OpenAI-compatible or Ollama-native. ## Completions Cache diff --git a/src/main/java/de/extio/lmlib/agent/BaseAgent.java b/src/main/java/de/extio/lmlib/agent/BaseAgent.java index c1baee7..5f3667f 100644 --- a/src/main/java/de/extio/lmlib/agent/BaseAgent.java +++ b/src/main/java/de/extio/lmlib/agent/BaseAgent.java @@ -31,7 +31,6 @@ import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.Conversation.TurnType; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; -import de.extio.lmlib.profile.ModelProfile.ModelProvider; public interface BaseAgent { @@ -83,15 +82,7 @@ public interface BaseAgent { return ToolCallData.auto(toolDefinitions); } - default boolean supportsToolCalling(final AgentContext context) { - final var modelProfile = this.modelProfile(context); - return modelProfile != null && modelProfile.modelProvider() == ModelProvider.OAI_CHAT_COMPLETION; - } - default boolean supportsToolCalling(final AgentContext context, final ClientService clientService) { - if (clientService == null) { - return this.supportsToolCalling(context); - } final var modelProfile = this.modelProfile(context); if (modelProfile != null) { return clientService.supportsToolCalling(modelProfile); @@ -161,7 +152,9 @@ public interface BaseAgent { final boolean skipCache = split.context().isSkipCache() || split.context().isAlwaysSkipCache(); split.context().setSkipCache(false); final var conversation = this.setupConversation(split); - var toolCallData = completionClient.supportsToolCalling() ? this.toolCallData(split.context()) : null; + final var requestedModelProfile = this.modelProfile(split.context()); + final var supportsToolCalling = requestedModelProfile != null ? completionClient.supportsToolCalling(requestedModelProfile) : completionClient.supportsToolCalling(this.modelCategory(split.context())); + var toolCallData = supportsToolCalling ? this.toolCallData(split.context()) : null; this.rememberToolCallData(split.context(), toolCallData); boolean parseable = false; diff --git a/src/main/java/de/extio/lmlib/client/Client.java b/src/main/java/de/extio/lmlib/client/Client.java index cfc55e0..f8099a3 100755 --- a/src/main/java/de/extio/lmlib/client/Client.java +++ b/src/main/java/de/extio/lmlib/client/Client.java @@ -21,5 +21,7 @@ public interface Client { ModelProvider getModelProvider(); - boolean supportsToolCalling(); + boolean supportsToolCalling(ModelCategory modelCategory); + + boolean supportsToolCalling(ModelProfile modelProfile); } diff --git a/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java b/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java index 04f6185..04d9cf1 100644 --- a/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java +++ b/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java @@ -6,6 +6,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; +import de.extio.lmlib.client.ollama.OllamaClient; import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient; import de.extio.lmlib.client.oai.completion.text.TextCompletionClient; import de.extio.lmlib.profile.ProfileAutoConfiguration; @@ -33,5 +34,11 @@ public class ClientAutoConfiguration { ChatCompletionClient chatCompletionClient() { return new ChatCompletionClient(); } + + @Bean + @ConditionalOnMissingBean + OllamaClient ollamaClient() { + return new OllamaClient(); + } } diff --git a/src/main/java/de/extio/lmlib/client/ClientService.java b/src/main/java/de/extio/lmlib/client/ClientService.java index 0be2a17..c697647 100755 --- a/src/main/java/de/extio/lmlib/client/ClientService.java +++ b/src/main/java/de/extio/lmlib/client/ClientService.java @@ -38,7 +38,7 @@ public class ClientService { } public boolean supportsToolCalling(final ModelCategory category) { - return this.getClient(category).supportsToolCalling(); + return this.getClient(category).supportsToolCalling(category); } @Cacheable("clients") @@ -56,7 +56,7 @@ public class ClientService { } public boolean supportsToolCalling(final ModelProfile modelProfile) { - return this.getClient(modelProfile).supportsToolCalling(); + return this.getClient(modelProfile).supportsToolCalling(modelProfile); } private static class InterceptingClient implements Client { @@ -122,8 +122,13 @@ public class ClientService { } @Override - public boolean supportsToolCalling() { - return this.client.supportsToolCalling(); + public boolean supportsToolCalling(final ModelCategory modelCategory) { + return this.client.supportsToolCalling(modelCategory); + } + + @Override + public boolean supportsToolCalling(final ModelProfile modelProfile) { + return this.client.supportsToolCalling(modelProfile); } } diff --git a/src/main/java/de/extio/lmlib/client/Conversation.java b/src/main/java/de/extio/lmlib/client/Conversation.java index d195b93..9deca58 100755 --- a/src/main/java/de/extio/lmlib/client/Conversation.java +++ b/src/main/java/de/extio/lmlib/client/Conversation.java @@ -61,7 +61,8 @@ public final class Conversation { if (completion == null) { throw new IllegalArgumentException("completion must not be null"); } - this.appendToolCallRound(completion.response(), completion.toolCalls(), toolResults); + final var assistantText = completion.toolCalls().isEmpty() ? completion.response() : ""; + this.appendToolCallRound(assistantText, completion.toolCalls(), toolResults); } public void appendToolCallRound(final String assistantText, final List toolCalls, final List toolResults) { diff --git a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java index be653a5..eca2654 100644 --- a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java +++ b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java @@ -78,8 +78,13 @@ public class CachedClient implements Client { } @Override - public boolean supportsToolCalling() { - return this.client.supportsToolCalling(); + public boolean supportsToolCalling(final ModelCategory modelCategory) { + return this.client.supportsToolCalling(modelCategory); + } + + @Override + public boolean supportsToolCalling(final ModelProfile modelProfile) { + return this.client.supportsToolCalling(modelProfile); } private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final boolean skipCache, final Supplier supplier, final String... keys) { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index 3d4aa64..8f5416e 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -24,6 +24,7 @@ import de.extio.lmlib.client.ToolCallData; import de.extio.lmlib.client.ToolDefinition; import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; +import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; @@ -36,8 +37,14 @@ public class ChatCompletionClient extends AbstractCompletionClient { } @Override - public boolean supportsToolCalling() { - return true; + public boolean supportsToolCalling(final ModelCategory modelCategory) { + final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory); + return this.supportsToolCalling(modelProfile); + } + + @Override + public boolean supportsToolCalling(final ModelProfile modelProfile) { + return modelProfile != null && modelProfile.modelProvider() == ModelProvider.OAI_CHAT_COMPLETION; } @Override diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 64adbef..ad6c1cf 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -23,6 +23,7 @@ import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.ToolCallData; import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; +import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.prompt.PromptStrategy; @@ -41,7 +42,12 @@ public class TextCompletionClient extends AbstractCompletionClient { } @Override - public boolean supportsToolCalling() { + public boolean supportsToolCalling(final ModelCategory modelCategory) { + return false; + } + + @Override + public boolean supportsToolCalling(final ModelProfile modelProfile) { return false; } diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaChatRequest.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaChatRequest.java new file mode 100644 index 0000000..5b7af56 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaChatRequest.java @@ -0,0 +1,75 @@ +package de.extio.lmlib.client.ollama; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaChatRequest { + + private String model; + + private List messages; + + private List tools; + + private Map options; + + private Boolean stream; + + private Object think; + + @JsonProperty("model") + public String getModel() { + return this.model; + } + + public void setModel(final String model) { + this.model = model; + } + + @JsonProperty("messages") + public List getMessages() { + return this.messages; + } + + public void setMessages(final List messages) { + this.messages = messages; + } + + @JsonProperty("tools") + public List getTools() { + return this.tools; + } + + public void setTools(final List tools) { + this.tools = tools; + } + + @JsonProperty("options") + public Map getOptions() { + return this.options; + } + + public void setOptions(final Map options) { + this.options = options; + } + + @JsonProperty("stream") + public Boolean getStream() { + return this.stream; + } + + public void setStream(final Boolean stream) { + this.stream = stream; + } + + @JsonProperty("think") + public Object getThink() { + return this.think; + } + + public void setThink(final Object think) { + this.think = think; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaChatResponse.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaChatResponse.java new file mode 100644 index 0000000..beb6b0b --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaChatResponse.java @@ -0,0 +1,174 @@ +package de.extio.lmlib.client.ollama; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaChatResponse { + + private String model; + + private String createdAt; + + private OllamaMessage message; + + private Boolean done; + + private String doneReason; + + private Long totalDuration; + + private Long loadDuration; + + private Integer promptEvalCount; + + private Long promptEvalDuration; + + private Integer evalCount; + + private Long evalDuration; + + private String error; + + @JsonProperty("model") + public String getModel() { + return this.model; + } + + public void setModel(final String model) { + this.model = model; + } + + @JsonProperty("created_at") + public String getCreatedAt() { + return this.createdAt; + } + + public void setCreatedAt(final String createdAt) { + this.createdAt = createdAt; + } + + @JsonProperty("message") + public OllamaMessage getMessage() { + return this.message; + } + + public void setMessage(final OllamaMessage message) { + this.message = message; + } + + @JsonProperty("done") + public Boolean getDone() { + return this.done; + } + + public void setDone(final Boolean done) { + this.done = done; + } + + @JsonProperty("done_reason") + public String getDoneReason() { + return this.doneReason; + } + + public void setDoneReason(final String doneReason) { + this.doneReason = doneReason; + } + + @JsonProperty("total_duration") + public Long getTotalDuration() { + return this.totalDuration; + } + + public void setTotalDuration(final Long totalDuration) { + this.totalDuration = totalDuration; + } + + @JsonProperty("load_duration") + public Long getLoadDuration() { + return this.loadDuration; + } + + public void setLoadDuration(final Long loadDuration) { + this.loadDuration = loadDuration; + } + + @JsonProperty("prompt_eval_count") + public Integer getPromptEvalCount() { + return this.promptEvalCount; + } + + public void setPromptEvalCount(final Integer promptEvalCount) { + this.promptEvalCount = promptEvalCount; + } + + @JsonProperty("prompt_eval_duration") + public Long getPromptEvalDuration() { + return this.promptEvalDuration; + } + + public void setPromptEvalDuration(final Long promptEvalDuration) { + this.promptEvalDuration = promptEvalDuration; + } + + @JsonProperty("eval_count") + public Integer getEvalCount() { + return this.evalCount; + } + + public void setEvalCount(final Integer evalCount) { + this.evalCount = evalCount; + } + + @JsonProperty("eval_duration") + public Long getEvalDuration() { + return this.evalDuration; + } + + public void setEvalDuration(final Long evalDuration) { + this.evalDuration = evalDuration; + } + + @JsonProperty("error") + public String getError() { + return this.error; + } + + public void setError(final String error) { + this.error = error; + } + + void copyFrom(final OllamaChatResponse other) { + if (other.model != null && !other.model.isBlank()) { + this.model = other.model; + } + if (other.createdAt != null && !other.createdAt.isBlank()) { + this.createdAt = other.createdAt; + } + if (other.done != null) { + this.done = other.done; + } + if (other.doneReason != null && !other.doneReason.isBlank()) { + this.doneReason = other.doneReason; + } + if (other.totalDuration != null) { + this.totalDuration = other.totalDuration; + } + if (other.loadDuration != null) { + this.loadDuration = other.loadDuration; + } + if (other.promptEvalCount != null) { + this.promptEvalCount = other.promptEvalCount; + } + if (other.promptEvalDuration != null) { + this.promptEvalDuration = other.promptEvalDuration; + } + if (other.evalCount != null) { + this.evalCount = other.evalCount; + } + if (other.evalDuration != null) { + this.evalDuration = other.evalDuration; + } + if (other.error != null && !other.error.isBlank()) { + this.error = other.error; + } + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java new file mode 100644 index 0000000..c719f97 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java @@ -0,0 +1,577 @@ +package de.extio.lmlib.client.ollama; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatusCode; +import org.springframework.web.client.RestClient; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestClientResponseException; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import de.extio.lmlib.client.Chunk; +import de.extio.lmlib.client.Client; +import de.extio.lmlib.client.Completion; +import de.extio.lmlib.client.CompletionFinishReason; +import de.extio.lmlib.client.CompletionStatistics; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.Conversation.Turn; +import de.extio.lmlib.client.ToolCall; +import de.extio.lmlib.client.ToolCallData; +import de.extio.lmlib.client.ToolDefinition; +import de.extio.lmlib.client.oai.completion.FinishReasons; +import de.extio.lmlib.profile.ModelCategory; +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; +import de.extio.lmlib.profile.ModelProfileService; +import de.extio.lmlib.token.TokenizerResolver; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +public class OllamaClient implements Client { + + private static final Logger LOGGER = LoggerFactory.getLogger(OllamaClient.class); + + private static final OllamaDialect DEFAULT_OLLAMA_DIALECT = new OllamaDialect() { + }; + + private final Map> resolvedModelNames = new ConcurrentHashMap<>(); + + private final Map resolvedModelDetails = new ConcurrentHashMap<>(); + + private final ObjectMapper objectMapper = JsonMapper.builder() + .changeDefaultPropertyInclusion(incl -> incl.withValueInclusion(JsonInclude.Include.NON_NULL)) + .changeDefaultPropertyInclusion(incl -> incl.withContentInclusion(JsonInclude.Include.NON_NULL)) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .build(); + + @Autowired + @Qualifier("lmLibRestClientBuilder") + private RestClient.Builder restClientBuilder; + + @Autowired + private TokenizerResolver tokenizerResolver; + + @Autowired + private ModelProfileService modelProfileService; + + @Autowired(required = false) + private OllamaDialect ollamaDialect; + + @Override + public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final ToolCallData toolCallData, final boolean skipCache) { + return this.streamConversation(modelCategory, conversation, null, toolCallData, skipCache); + } + + @Override + public Completion conversation(final ModelProfile modelProfile, final Conversation conversation, final ToolCallData toolCallData, final boolean skipCache) { + return this.streamConversation(modelProfile, conversation, null, toolCallData, skipCache); + } + + @Override + public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer chunkConsumer, final ToolCallData toolCallData, final boolean skipCache) { + final var modelCategory = modelCategory_ == null ? ModelCategory.MEDIUM : modelCategory_; + final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory); + return this.streamConversation(modelProfile, conversation, chunkConsumer, toolCallData, skipCache); + } + + @Override + public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer chunkConsumer, final ToolCallData toolCallData, final boolean skipCache) { + if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OLLAMA) { + throw new IllegalArgumentException("Invalid ModelProfile"); + } + + final var request = this.createRequest(conversation, modelProfile, chunkConsumer != null, toolCallData); + final String requestBody; + try { + requestBody = this.objectMapper.writeValueAsString(request); + } + catch (final JacksonException e) { + throw new IllegalStateException("Cannot serialize request body", e); + } + + LOGGER.debug("Requesting Ollama chat at {}", modelProfile.url()); + final var start = LocalDateTime.now(); + final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); + + var requestSpec = restClient + .method(HttpMethod.POST) + .uri("/api/chat") + .header("Content-Type", "application/json"); + if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { + requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); + } + + final var response = requestSpec + .body(requestBody) + .exchange((clientRequest, clientResponse) -> { + if (clientResponse.getStatusCode().isError()) { + this.logErrorResponse(clientResponse.getStatusCode(), clientResponse.getBody()); + throw new IllegalStateException("Error response from server: " + clientResponse.getStatusCode()); + } + if (chunkConsumer == null) { + final var ollamaResponse = this.objectMapper.readValue(clientResponse.getBody(), OllamaChatResponse.class); + this.throwIfEmbeddedError(ollamaResponse); + return ollamaResponse; + } + return this.readStream(clientResponse.getBody(), chunkConsumer); + }); + + this.throwIfEmbeddedError(response); + final var content = response.getMessage() != null ? response.getMessage().getContent() : null; + final var reasoning = response.getMessage() != null ? response.getMessage().getThinking() : null; + final var toolCalls = this.toToolCalls(response.getMessage() != null ? response.getMessage().getToolCalls() : List.of()); + final var finishReason = this.mapFinishReason(response.getDoneReason(), toolCalls); + final var statistics = this.createCompletionStatistics(modelProfile, start, response, conversation, content, reasoning); + return new Completion(content, reasoning, finishReason, statistics, toolCalls); + } + + @Override + public List getModelNames(final ModelProfile modelProfile, final boolean forceReload) { + if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OLLAMA) { + return List.of(); + } + if (modelProfile.modelName() != null && !modelProfile.modelName().isBlank()) { + return List.of(modelProfile.modelName()); + } + if (modelProfile.url() == null || modelProfile.url().isBlank()) { + return List.of(); + } + final var cacheKey = modelProfile.category() + "|" + modelProfile.url() + "|" + modelProfile.apiKey(); + if (forceReload) { + final var modelNames = this.loadModelNames(modelProfile); + this.resolvedModelNames.put(cacheKey, modelNames); + return modelNames; + } + return this.resolvedModelNames.computeIfAbsent(cacheKey, key -> this.loadModelNames(modelProfile)); + } + + @Override + public ModelProvider getModelProvider() { + return ModelProvider.OLLAMA; + } + + @Override + public boolean supportsToolCalling(final ModelCategory modelCategory) { + final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory); + return this.supportsToolCalling(modelProfile); + } + + @Override + public boolean supportsToolCalling(final ModelProfile modelProfile) { + if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OLLAMA) { + return false; + } + final var modelName = this.getPrimaryModelName(modelProfile); + if (modelName == null || modelName.isBlank()) { + return false; + } + return this.supportsToolCapability(this.getModelDetails(modelProfile, modelName)); + } + + private OllamaChatRequest createRequest(final Conversation conversation, final ModelProfile modelProfile, final boolean streaming, final ToolCallData toolCallData) { + final var modelName = this.getPrimaryModelName(modelProfile); + if (modelName == null || modelName.isBlank()) { + throw new IllegalStateException("No model name resolved for Ollama profile " + modelProfile.category()); + } + final var modelDetails = this.getModelDetails(modelProfile, modelName); + final var request = new OllamaChatRequest(); + request.setModel(modelName); + request.setMessages(this.createMessages(conversation)); + request.setStream(streaming); + final var think = this.createThinkRequestValue(modelProfile, modelDetails); + if (think != null) { + request.setThink(think); + } + request.setOptions(this.createOptions(modelProfile)); + if (toolCallData != null && toolCallData.hasTools()) { + if (this.supportsToolCapability(modelDetails)) { + request.setTools(this.createTools(toolCallData.tools())); + } + else { + LOGGER.debug("Ollama model {} does not advertise tool capability. Tools will be omitted from the request.", modelName); + } + } + return request; + } + + private Object createThinkRequestValue(final ModelProfile modelProfile, final OllamaShowResponse modelDetails) { + final var thinkMode = this.getOllamaDialect().think(modelProfile); + if (thinkMode == null || !thinkMode.isEnabled()) { + return null; + } + if (!this.supportsThinkingCapability(modelDetails)) { + LOGGER.debug("Ollama model {} does not advertise thinking capability. Think will be omitted from the request.", this.getPrimaryModelName(modelProfile)); + return null; + } + return thinkMode; + } + + private List createMessages(final Conversation conversation) { + final List messages = new ArrayList<>(); + final Map toolCallNamesById = new LinkedHashMap<>(); + for (final Turn turn : conversation.getConversation()) { + final var message = new OllamaMessage(); + message.setRole(this.mapRole(turn)); + message.setContent(turn.text() != null ? turn.text() : ""); + if (!turn.toolCalls().isEmpty()) { + message.setToolCalls(turn.toolCalls().stream().map(toolCall -> { + final var function = new OllamaFunctionCall(); + function.setName(toolCall.name()); + function.setArguments(this.parseJsonOrText(toolCall.arguments())); + final var ollamaToolCall = new OllamaToolCall(); + ollamaToolCall.setId(toolCall.id()); + ollamaToolCall.setType(toolCall.type()); + ollamaToolCall.setFunction(function); + if (toolCall.id() != null && !toolCall.id().isBlank() && toolCall.name() != null && !toolCall.name().isBlank()) { + toolCallNamesById.put(toolCall.id(), toolCall.name()); + } + return ollamaToolCall; + }).toList()); + } + message.setToolCallId(turn.toolCallId()); + if (turn.type() == Conversation.TurnType.TOOL && turn.toolCallId() != null && !turn.toolCallId().isBlank()) { + message.setToolName(toolCallNamesById.get(turn.toolCallId())); + } + messages.add(message); + } + return messages; + } + + private String mapRole(final Turn turn) { + return switch (turn.type()) { + case SYSTEM -> "system"; + case USER -> "user"; + case ASSISTANT -> "assistant"; + case TOOL -> "tool"; + }; + } + + private Map createOptions(final ModelProfile modelProfile) { + final var options = new LinkedHashMap(); + options.put("num_predict", modelProfile.maxTokens()); + options.put("temperature", modelProfile.temperature()); + options.put("top_p", modelProfile.topP()); + return options; + } + + private List createTools(final List toolDefinitions) { + return toolDefinitions.stream().map(toolDefinition -> { + final var function = new OllamaToolFunction(); + function.setName(toolDefinition.name()); + function.setDescription(toolDefinition.description()); + function.setParameters(toolDefinition.parameters()); + final var tool = new OllamaToolDefinition(); + tool.setType("function"); + tool.setFunction(function); + return tool; + }).toList(); + } + + private OllamaChatResponse readStream(final InputStream inputStream, final Consumer chunkConsumer) throws IOException { + final var response = new OllamaChatResponse(); + final var contentBuilder = new StringBuilder(); + final var reasoningBuilder = new StringBuilder(); + final List toolCalls = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.isEmpty()) { + continue; + } + final var streamResponse = this.objectMapper.readValue(line, OllamaChatResponse.class); + this.throwIfEmbeddedError(streamResponse); + response.copyFrom(streamResponse); + if (streamResponse.getMessage() != null) { + String content = null; + String reasoning = null; + if (streamResponse.getMessage().getContent() != null && !streamResponse.getMessage().getContent().isEmpty()) { + content = streamResponse.getMessage().getContent(); + contentBuilder.append(content); + } + if (streamResponse.getMessage().getThinking() != null && !streamResponse.getMessage().getThinking().isEmpty()) { + reasoning = streamResponse.getMessage().getThinking(); + reasoningBuilder.append(reasoning); + } + if (content != null || reasoning != null) { + chunkConsumer.accept(new Chunk(content, reasoning)); + } + if (streamResponse.getMessage().getToolCalls() != null && !streamResponse.getMessage().getToolCalls().isEmpty()) { + toolCalls.clear(); + toolCalls.addAll(this.toToolCalls(streamResponse.getMessage().getToolCalls())); + } + } + if (Boolean.TRUE.equals(streamResponse.getDone())) { + break; + } + } + } + final var message = new OllamaMessage(); + message.setRole("assistant"); + message.setContent(contentBuilder.toString()); + message.setThinking(reasoningBuilder.toString()); + message.setToolCalls(this.toOllamaToolCalls(toolCalls)); + response.setMessage(message); + return response; + } + + private List toToolCalls(final List ollamaToolCalls) { + if (ollamaToolCalls == null || ollamaToolCalls.isEmpty()) { + return List.of(); + } + final List toolCalls = new ArrayList<>(); + for (int i = 0; i < ollamaToolCalls.size(); i++) { + final var toolCall = ollamaToolCalls.get(i); + if (toolCall == null || toolCall.getFunction() == null || toolCall.getFunction().getName() == null || toolCall.getFunction().getName().isBlank()) { + continue; + } + toolCalls.add(new ToolCall( + toolCall.getId() != null && !toolCall.getId().isBlank() ? toolCall.getId() : "ollama-tool-call-" + (i + 1), + toolCall.getType() != null && !toolCall.getType().isBlank() ? toolCall.getType() : "function", + toolCall.getFunction().getName(), + this.serializeArguments(toolCall.getFunction().getArguments()))); + } + return List.copyOf(toolCalls); + } + + private List toOllamaToolCalls(final List toolCalls) { + return toolCalls.stream().map(toolCall -> { + final var function = new OllamaFunctionCall(); + function.setName(toolCall.name()); + function.setArguments(this.parseJsonOrText(toolCall.arguments())); + final var ollamaToolCall = new OllamaToolCall(); + ollamaToolCall.setId(toolCall.id()); + ollamaToolCall.setType(toolCall.type()); + ollamaToolCall.setFunction(function); + return ollamaToolCall; + }).toList(); + } + + private CompletionFinishReason mapFinishReason(final String finishReason, final List toolCalls) { + if (toolCalls != null && !toolCalls.isEmpty()) { + return CompletionFinishReason.TOOL_CALLS; + } + return switch (finishReason) { + case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED; + case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED; + case FinishReasons.FINISH_REASON_TOOL_CALLS -> CompletionFinishReason.TOOL_CALLS; + default -> CompletionFinishReason.DONE; + }; + } + + private CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final OllamaChatResponse response, final Conversation conversation, final String content, final String reasoning) { + int inTokens = 0; + int outTokens = 0; + int reasoningOutTokens = 0; + if (response.getPromptEvalCount() != null && response.getPromptEvalCount() > 0) { + inTokens = response.getPromptEvalCount(); + } + if (response.getEvalCount() != null && response.getEvalCount() > 0) { + outTokens = response.getEvalCount(); + } + if (reasoning != null && !reasoning.isBlank()) { + reasoningOutTokens = this.tokenizerResolver.count(reasoning, modelProfile); + outTokens = Math.max(0, outTokens - reasoningOutTokens); + } + if (inTokens <= 0) { + inTokens = this.tokenizerResolver.count(this.createPromptForCounting(conversation), modelProfile); + } + if (outTokens <= 0 && content != null && !content.isBlank()) { + outTokens = this.tokenizerResolver.count(content, modelProfile); + } + final var duration = response.getTotalDuration() != null && response.getTotalDuration() > 0 + ? Duration.ofNanos(response.getTotalDuration()) + : Duration.between(start, LocalDateTime.now()); + var cost = BigDecimal.ZERO; + cost = cost.add(BigDecimal.valueOf(inTokens).multiply(modelProfile.costPerInToken())); + cost = cost.add(BigDecimal.valueOf(outTokens).multiply(modelProfile.costPerOutToken())); + cost = cost.add(BigDecimal.valueOf(reasoningOutTokens).multiply(modelProfile.costPerReasoningOutToken())); + return new CompletionStatistics(1, duration, inTokens, 0, outTokens, reasoningOutTokens, cost, false); + } + + private String createPromptForCounting(final Conversation conversation) { + final var builder = new StringBuilder(); + for (final var turn : conversation.getConversation()) { + builder.append(turn.type().name()); + builder.append(':'); + builder.append(turn.text() != null ? turn.text() : ""); + builder.append('\n'); + for (final var toolCall : turn.toolCalls()) { + builder.append(toolCall.name()); + builder.append(':'); + builder.append(toolCall.arguments()); + builder.append('\n'); + } + } + return builder.toString(); + } + + private String getPrimaryModelName(final ModelProfile modelProfile) { + return this.getModelNames(modelProfile, false).stream().findFirst().orElse(""); + } + + private List loadModelNames(final ModelProfile modelProfile) { + try { + var requestSpec = this.restClientBuilder.baseUrl(modelProfile.url()).build() + .method(HttpMethod.GET) + .uri("/api/tags"); + if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { + requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); + } + final var response = requestSpec.retrieve().body(OllamaTagsResponse.class); + return response == null || response.getModels() == null ? List.of() + : response.getModels().stream() + .map(model -> model.getName() != null && !model.getName().isBlank() ? model.getName() : model.getModel()) + .filter(Objects::nonNull) + .filter(modelName -> !modelName.isBlank()) + .toList(); + } + catch (final RestClientResponseException e) { + LOGGER.warn("Failed to load Ollama model names: {} {}", e.getStatusCode(), e.getStatusText()); + return List.of(); + } + catch (final RestClientException e) { + LOGGER.warn("Failed to load Ollama model names: {}", e.getMessage()); + return List.of(); + } + catch (final RuntimeException e) { + LOGGER.warn("Failed to load Ollama model names", e); + return List.of(); + } + } + + private OllamaShowResponse getModelDetails(final ModelProfile modelProfile, final String modelName) { + final var cacheKey = this.cacheKey(modelProfile, modelName); + return this.resolvedModelDetails.computeIfAbsent(cacheKey, key -> this.loadModelDetails(modelProfile, modelName)); + } + + private OllamaShowResponse loadModelDetails(final ModelProfile modelProfile, final String modelName) { + try { + final var request = new OllamaShowRequest(); + request.setModel(modelName); + request.setVerbose(false); + var requestSpec = this.restClientBuilder.baseUrl(modelProfile.url()).build() + .method(HttpMethod.POST) + .uri("/api/show") + .header("Content-Type", "application/json"); + if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { + requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); + } + final var requestBody = this.objectMapper.writeValueAsString(request); + final var response = requestSpec.body(requestBody).retrieve().body(OllamaShowResponse.class); + return response == null ? new OllamaShowResponse() : response; + } + catch (final RestClientResponseException e) { + LOGGER.warn("Failed to load Ollama model details for {}: {} {}", modelName, e.getStatusCode(), e.getStatusText()); + return new OllamaShowResponse(); + } + catch (final RestClientException e) { + LOGGER.warn("Failed to load Ollama model details for {}: {}", modelName, e.getMessage()); + return new OllamaShowResponse(); + } + catch (final JacksonException e) { + LOGGER.warn("Failed to serialize Ollama show request for {}", modelName, e); + return new OllamaShowResponse(); + } + catch (final RuntimeException e) { + LOGGER.warn("Failed to load Ollama model details for {}", modelName, e); + return new OllamaShowResponse(); + } + } + + private boolean supportsThinkingCapability(final OllamaShowResponse modelDetails) { + return this.hasCapability(modelDetails, "thinking"); + } + + private boolean supportsToolCapability(final OllamaShowResponse modelDetails) { + return this.hasCapability(modelDetails, "tools") + || this.hasCapability(modelDetails, "tool") + || this.hasCapability(modelDetails, "tool-calling") + || this.hasCapability(modelDetails, "tool_calling"); + } + + private boolean hasCapability(final OllamaShowResponse modelDetails, final String expectedCapability) { + if (modelDetails == null || modelDetails.getCapabilities() == null || modelDetails.getCapabilities().isEmpty()) { + return false; + } + return modelDetails.getCapabilities().stream() + .filter(Objects::nonNull) + .map(capability -> capability.toLowerCase(Locale.ROOT).trim()) + .anyMatch(capability -> capability.equals(expectedCapability) || capability.contains(expectedCapability)); + } + + private String cacheKey(final ModelProfile modelProfile, final String modelName) { + return modelProfile.category() + "|" + modelProfile.url() + "|" + modelProfile.apiKey() + "|" + modelName; + } + + private OllamaDialect getOllamaDialect() { + return this.ollamaDialect == null ? DEFAULT_OLLAMA_DIALECT : this.ollamaDialect; + } + + private void throwIfEmbeddedError(final OllamaChatResponse response) { + if (response != null && response.getError() != null && !response.getError().isBlank()) { + LOGGER.warn("Error response from Ollama: {}", response.getError()); + throw new IllegalStateException(response.getError()); + } + } + + private void logErrorResponse(final HttpStatusCode statusCode, final InputStream responseBody) { + try { + LOGGER.warn("Error response from server: {} {}", statusCode, new String(responseBody.readAllBytes(), StandardCharsets.UTF_8)); + } + catch (final IOException e) { + LOGGER.warn("Error response from server: {} ", statusCode, e.getMessage()); + } + } + + private Object parseJsonOrText(final String value) { + if (value == null || value.isBlank()) { + return value; + } + try { + return this.objectMapper.readValue(value, Object.class); + } + catch (final Exception e) { + return value; + } + } + + private String serializeArguments(final Object value) { + if (value == null) { + return null; + } + if (value instanceof final String string) { + return string; + } + try { + return this.objectMapper.writeValueAsString(value); + } + catch (final JacksonException e) { + throw new IllegalStateException("Cannot serialize tool call arguments", e); + } + } + +} diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaDialect.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaDialect.java new file mode 100644 index 0000000..0a6eb75 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaDialect.java @@ -0,0 +1,35 @@ +package de.extio.lmlib.client.ollama; + +import com.fasterxml.jackson.annotation.JsonValue; + +import de.extio.lmlib.profile.ModelProfile; + +public interface OllamaDialect { + + default ThinkMode think(final ModelProfile modelProfile) { + return ThinkMode.ENABLED; + } + + enum ThinkMode { + DISABLED(Boolean.FALSE), + ENABLED(Boolean.TRUE), + LOW("low"), + MEDIUM("medium"), + HIGH("high"); + + private final Object wireValue; + + ThinkMode(final Object wireValue) { + this.wireValue = wireValue; + } + + public boolean isEnabled() { + return (this.wireValue instanceof final Boolean val && val) || (this.wireValue instanceof final String str && !str.isBlank()); + } + + @JsonValue + public Object getWireValue() { + return this.wireValue; + } + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaFunctionCall.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaFunctionCall.java new file mode 100644 index 0000000..a954a77 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaFunctionCall.java @@ -0,0 +1,39 @@ +package de.extio.lmlib.client.ollama; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaFunctionCall { + + private String name; + + private String description; + + private Object arguments; + + @JsonProperty("name") + public String getName() { + return this.name; + } + + public void setName(final String name) { + this.name = name; + } + + @JsonProperty("description") + public String getDescription() { + return this.description; + } + + public void setDescription(final String description) { + this.description = description; + } + + @JsonProperty("arguments") + public Object getArguments() { + return this.arguments; + } + + public void setArguments(final Object arguments) { + this.arguments = arguments; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaListedModel.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaListedModel.java new file mode 100644 index 0000000..608610a --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaListedModel.java @@ -0,0 +1,28 @@ +package de.extio.lmlib.client.ollama; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaListedModel { + + private String name; + + private String model; + + @JsonProperty("name") + public String getName() { + return this.name; + } + + public void setName(final String name) { + this.name = name; + } + + @JsonProperty("model") + public String getModel() { + return this.model; + } + + public void setModel(final String model) { + this.model = model; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaMessage.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaMessage.java new file mode 100644 index 0000000..d2771f2 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaMessage.java @@ -0,0 +1,74 @@ +package de.extio.lmlib.client.ollama; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaMessage { + + private String role; + + private String content; + + private String thinking; + + private List toolCalls; + + private String toolCallId; + + private String toolName; + + @JsonProperty("role") + public String getRole() { + return this.role; + } + + public void setRole(final String role) { + this.role = role; + } + + @JsonProperty("content") + public String getContent() { + return this.content; + } + + public void setContent(final String content) { + this.content = content; + } + + @JsonProperty("thinking") + public String getThinking() { + return this.thinking; + } + + public void setThinking(final String thinking) { + this.thinking = thinking; + } + + @JsonProperty("tool_calls") + public List getToolCalls() { + return this.toolCalls; + } + + public void setToolCalls(final List toolCalls) { + this.toolCalls = toolCalls; + } + + @JsonProperty("tool_call_id") + public String getToolCallId() { + return this.toolCallId; + } + + public void setToolCallId(final String toolCallId) { + this.toolCallId = toolCallId; + } + + @JsonProperty("tool_name") + public String getToolName() { + return this.toolName; + } + + public void setToolName(final String toolName) { + this.toolName = toolName; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaShowRequest.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaShowRequest.java new file mode 100644 index 0000000..490ee1e --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaShowRequest.java @@ -0,0 +1,28 @@ +package de.extio.lmlib.client.ollama; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaShowRequest { + + private String model; + + private Boolean verbose; + + @JsonProperty("model") + public String getModel() { + return this.model; + } + + public void setModel(final String model) { + this.model = model; + } + + @JsonProperty("verbose") + public Boolean getVerbose() { + return this.verbose; + } + + public void setVerbose(final Boolean verbose) { + this.verbose = verbose; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaShowResponse.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaShowResponse.java new file mode 100644 index 0000000..cd02cd3 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaShowResponse.java @@ -0,0 +1,19 @@ +package de.extio.lmlib.client.ollama; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaShowResponse { + + private List capabilities; + + @JsonProperty("capabilities") + public List getCapabilities() { + return this.capabilities; + } + + public void setCapabilities(final List capabilities) { + this.capabilities = capabilities; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaTagsResponse.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaTagsResponse.java new file mode 100644 index 0000000..c65172f --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaTagsResponse.java @@ -0,0 +1,19 @@ +package de.extio.lmlib.client.ollama; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaTagsResponse { + + private List models; + + @JsonProperty("models") + public List getModels() { + return this.models; + } + + public void setModels(final List models) { + this.models = models; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaToolCall.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaToolCall.java new file mode 100644 index 0000000..5a20eba --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaToolCall.java @@ -0,0 +1,39 @@ +package de.extio.lmlib.client.ollama; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaToolCall { + + private String id; + + private String type; + + private OllamaFunctionCall function; + + @JsonProperty("id") + public String getId() { + return this.id; + } + + public void setId(final String id) { + this.id = id; + } + + @JsonProperty("type") + public String getType() { + return this.type; + } + + public void setType(final String type) { + this.type = type; + } + + @JsonProperty("function") + public OllamaFunctionCall getFunction() { + return this.function; + } + + public void setFunction(final OllamaFunctionCall function) { + this.function = function; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaToolDefinition.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaToolDefinition.java new file mode 100644 index 0000000..b3259cb --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaToolDefinition.java @@ -0,0 +1,28 @@ +package de.extio.lmlib.client.ollama; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaToolDefinition { + + private String type; + + private OllamaToolFunction function; + + @JsonProperty("type") + public String getType() { + return this.type; + } + + public void setType(final String type) { + this.type = type; + } + + @JsonProperty("function") + public OllamaToolFunction getFunction() { + return this.function; + } + + public void setFunction(final OllamaToolFunction function) { + this.function = function; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaToolFunction.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaToolFunction.java new file mode 100644 index 0000000..3ae2c61 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaToolFunction.java @@ -0,0 +1,41 @@ +package de.extio.lmlib.client.ollama; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class OllamaToolFunction { + + private String name; + + private String description; + + private Map parameters; + + @JsonProperty("name") + public String getName() { + return this.name; + } + + public void setName(final String name) { + this.name = name; + } + + @JsonProperty("description") + public String getDescription() { + return this.description; + } + + public void setDescription(final String description) { + this.description = description; + } + + @JsonProperty("parameters") + public Map getParameters() { + return this.parameters; + } + + public void setParameters(final Map parameters) { + this.parameters = parameters; + } +} \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfile.java b/src/main/java/de/extio/lmlib/profile/ModelProfile.java index 27b37e5..eb54d59 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfile.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfile.java @@ -45,6 +45,6 @@ public record ModelProfile( } public static enum ModelProvider { - OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION; + OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION, OLLAMA; } } diff --git a/src/test/java/de/extio/lmlib/agent/AgentErrorHandlingTest.java b/src/test/java/de/extio/lmlib/agent/AgentErrorHandlingTest.java index 8d77556..4bd68b6 100644 --- a/src/test/java/de/extio/lmlib/agent/AgentErrorHandlingTest.java +++ b/src/test/java/de/extio/lmlib/agent/AgentErrorHandlingTest.java @@ -34,6 +34,7 @@ import de.extio.lmlib.client.ToolCallData; import de.extio.lmlib.client.ToolDefinition; import de.extio.lmlib.client.ToolParameters; import de.extio.lmlib.profile.ModelCategory; +import de.extio.lmlib.profile.ModelProfile; class AgentErrorHandlingTest { @@ -98,7 +99,8 @@ class AgentErrorHandlingTest { @Test void toolCallDataIsRetainedForFollowUpConversationRequests() { final var client = mock(Client.class); - when(client.supportsToolCalling()).thenReturn(true); + when(client.supportsToolCalling(any(ModelProfile.class))).thenReturn(true); + when(client.supportsToolCalling(any(ModelCategory.class))).thenReturn(true); final var requestCounter = new AtomicInteger(); final var firstRequestToolCallData = new AtomicReference(); final var secondRequestToolCallData = new AtomicReference(); diff --git a/src/test/java/de/extio/lmlib/agent/AgentTest.java b/src/test/java/de/extio/lmlib/agent/AgentTest.java index 4422c85..ffb265d 100644 --- a/src/test/java/de/extio/lmlib/agent/AgentTest.java +++ b/src/test/java/de/extio/lmlib/agent/AgentTest.java @@ -21,7 +21,7 @@ import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.context.annotation.PropertySource; +import org.springframework.test.context.TestPropertySource; import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler; import de.extio.lmlib.agent.responsehandler.TextAgentResponseHandler; @@ -43,7 +43,7 @@ import de.extio.lmlib.profile.ModelProfileService; @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @EnableAutoConfiguration -@PropertySource("classpath:/application-test.properties") +@TestPropertySource("classpath:/application-test.properties") public class AgentTest { private static final Logger LOGGER = LoggerFactory.getLogger(AgentTest.class); @@ -367,7 +367,7 @@ public class AgentTest { } @Override - public boolean supportsToolCalling(final AgentContext context) { + public boolean supportsToolCalling(final AgentContext context, final ClientService clientService) { return true; } diff --git a/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java index 794c38e..fbe9260 100755 --- a/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java +++ b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java @@ -16,7 +16,7 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; +import org.springframework.test.context.TestPropertySource; import de.extio.lmlib.client.ClientService; import de.extio.lmlib.client.Conversation; @@ -29,7 +29,7 @@ import de.extio.lmlib.profile.ModelCategory; @SpringBootConfiguration @EnableAutoConfiguration @Import({ TestCachedClientRepository.Config.class }) -@PropertySource("classpath:/application-test.properties") +@TestPropertySource("classpath:/application-test.properties") public class CachedClientTest { private static final Logger LOGGER = LoggerFactory.getLogger(CachedClientTest.class); diff --git a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java index 9ba6c58..e26c546 100755 --- a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java +++ b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java @@ -20,7 +20,7 @@ import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.context.annotation.PropertySource; +import org.springframework.test.context.TestPropertySource; import de.extio.lmlib.client.ClientService; import de.extio.lmlib.client.CompletionFinishReason; @@ -41,7 +41,7 @@ import de.extio.lmlib.profile.ModelProfileService; @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @EnableAutoConfiguration -@PropertySource("classpath:/application-test.properties") +@TestPropertySource("classpath:/application-test.properties") public class CompletionClientTest { private static final Logger LOGGER = LoggerFactory.getLogger(CompletionClientTest.class); @@ -174,8 +174,8 @@ public class CompletionClientTest { toolCallData, false); - assertTrue(this.chatCompletionclient.supportsToolCalling()); - assertFalse(this.textCompletionClient.supportsToolCalling()); + assertTrue(this.chatCompletionclient.supportsToolCalling(modelProfile)); + assertFalse(this.textCompletionClient.supportsToolCalling(this.modelProfileService.getModelProfile("profile.model.textcompletion"))); assertTrue(this.clientService.supportsToolCalling(modelProfile)); assertTrue(completion.finishReason() == CompletionFinishReason.TOOL_CALLS); assertEquals(2, completion.toolCalls().size()); @@ -226,8 +226,8 @@ public class CompletionClientTest { ToolCallData.required(List.of(getWeather, getCpu)).withParallelToolCalls(true), false); - assertTrue(this.chatCompletionclient.supportsToolCalling()); - assertFalse(this.textCompletionClient.supportsToolCalling()); + assertTrue(this.chatCompletionclient.supportsToolCalling(modelProfile)); + assertFalse(this.textCompletionClient.supportsToolCalling(this.modelProfileService.getModelProfile("profile.model.textcompletion"))); assertTrue(this.clientService.supportsToolCalling(modelProfile)); assertTrue(completion.finishReason() == CompletionFinishReason.TOOL_CALLS); assertEquals(2, completion.toolCalls().size()); diff --git a/src/test/java/de/extio/lmlib/client/ollama/OllamaClientTest.java b/src/test/java/de/extio/lmlib/client/ollama/OllamaClientTest.java new file mode 100644 index 0000000..882559b --- /dev/null +++ b/src/test/java/de/extio/lmlib/client/ollama/OllamaClientTest.java @@ -0,0 +1,152 @@ +package de.extio.lmlib.client.ollama; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.test.context.TestPropertySource; + +import de.extio.lmlib.client.ClientService; +import de.extio.lmlib.client.CompletionFinishReason; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.ToolCall; +import de.extio.lmlib.client.ToolCallData; +import de.extio.lmlib.client.ToolDefinition; +import de.extio.lmlib.client.ToolParameters; +import de.extio.lmlib.grader.Grader2; +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfileService; + +@Disabled("This test requires a running Ollama instance") +@SpringBootTest(webEnvironment = WebEnvironment.NONE) +@SpringBootConfiguration +@EnableAutoConfiguration +@TestPropertySource("classpath:/application-test.properties") +public class OllamaClientTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(OllamaClientTest.class); + + @Autowired + private OllamaClient ollamaClient; + + @Autowired + private ModelProfileService modelProfileService; + + @Autowired + private ClientService clientService; + + @Test + void listModels() throws Exception { + final var modelProfile = this.modelProfile(); + final var modelNames = this.ollamaClient.getModelNames(modelProfile, true); + LOGGER.info("Models: {}", modelNames); + assertFalse(modelNames.isEmpty()); + assertFalse(modelNames.getFirst().isBlank()); + assertEquals(ModelProfile.ModelProvider.OLLAMA, this.clientService.getClient(modelProfile).getModelProvider()); + } + + @Test + void chatCompletion() throws Exception { + final var modelProfile = this.modelProfile(); + final var completion = this.ollamaClient.conversation( + modelProfile, + Conversation.create("You are a helpful assistant", "Say the color is green"), + null, + false); + + LOGGER.info(completion.reasoning()); + LOGGER.info(completion.response()); + + assertTrue(Grader2.assessScoreBinary("Does the text mention the color green?", completion.response(), modelProfile, this.clientService)); + } + + @Test + void streamChatCompletion() throws Exception { + final var modelProfile = this.modelProfile(); + final var conversation = Conversation.create("You are a helpful assistant", "I would like to play a game of chess."); + conversation.addTurn(new Conversation.Turn(Conversation.TurnType.ASSISTANT, "Which opening do you choose?")); + conversation.addTurn(new Conversation.Turn(Conversation.TurnType.USER, "You choose, but explain the opening to me.")); + final var responseBuilder = new StringBuilder(); + final var reasoningBuilder = new StringBuilder(); + final var completion = this.ollamaClient.streamConversation(modelProfile, conversation, chunk -> { + if (chunk.reasoningContent() != null) { + System.out.print(chunk.reasoningContent()); + reasoningBuilder.append(chunk.reasoningContent()); + } + if (chunk.content() != null) { + System.out.print(chunk.content()); + responseBuilder.append(chunk.content()); + } + }, null, false); + + assertEquals(completion.response(), responseBuilder.toString()); + if (completion.reasoning() != null) { + assertEquals(completion.reasoning(), reasoningBuilder.toString()); + } + assertTrue(Grader2.assessScoreBinary("Does the text explain a chess opening?", completion.response(), modelProfile, this.clientService)); + } + + @Test + void toolCallingChatCompletion() throws Exception { + final var modelProfile = this.modelProfile(); + final var weatherTool = new ToolDefinition( + "get_weather", + "Gets the current weather for a given location.", + ToolParameters.create(Map.of("location", "City and country, for example Berlin, Germany")), + true); + final var conversation = Conversation.create( + "You are a helpful assistant. Use the provided tools when the user asks for location information. After receiving tool results, answer the question from the tool output.", + "What is the weather in Berlin, Germany? Use the tool."); + final var completion = this.ollamaClient.conversation( + modelProfile, + conversation, + ToolCallData.required(List.of(weatherTool)).withParallelToolCalls(true), + false); + + LOGGER.info(completion.reasoning()); + LOGGER.info(completion.response()); + LOGGER.info("Tool calls: {}", completion.toolCalls()); + + assertEquals(CompletionFinishReason.TOOL_CALLS, completion.finishReason()); + assertFalse(completion.toolCalls().isEmpty()); + final var weatherToolCall = this.findToolCall(completion.toolCalls(), "get_weather"); + assertTrue(weatherToolCall.arguments().contains("Berlin")); + + conversation.appendToolCallRound(completion, List.of( + new Conversation.ToolResult(ToolParameters.create() + .add("forecast", "Berlin is rainy and 21C") + .json(), weatherToolCall.id()))); + LOGGER.info("Conversation after tool call: {}", conversation.getConversation()); + + final var finalCompletion = this.ollamaClient.conversation(modelProfile, conversation, ToolCallData.auto(List.of(weatherTool)), false); + LOGGER.info(finalCompletion.reasoning()); + LOGGER.info(finalCompletion.response()); + LOGGER.info("Tool calls: {}", finalCompletion.toolCalls()); + assertEquals(CompletionFinishReason.DONE, finalCompletion.finishReason()); + assertTrue(Grader2.assessScoreBinary("Does the text say that Berlin is rainy and 21C?", finalCompletion.response(), modelProfile, this.clientService)); + } + + private ModelProfile modelProfile() { + return this.modelProfileService.getModelProfile("profile.model.ollama"); + } + + private ToolCall findToolCall(final List toolCalls, final String toolName) { + return toolCalls.stream() + .filter(toolCall -> toolName.equals(toolCall.name())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Missing tool call: " + toolName)); + } + +} \ No newline at end of file diff --git a/src/test/java/de/extio/lmlib/grader/GraderTest.java b/src/test/java/de/extio/lmlib/grader/GraderTest.java index c79cf50..7e954a3 100644 --- a/src/test/java/de/extio/lmlib/grader/GraderTest.java +++ b/src/test/java/de/extio/lmlib/grader/GraderTest.java @@ -28,7 +28,7 @@ import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.context.annotation.PropertySource; +import org.springframework.test.context.TestPropertySource; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -45,7 +45,7 @@ import de.extio.lmlib.profile.ModelCategory; @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @EnableAutoConfiguration -@PropertySource("classpath:/application-test.properties") +@TestPropertySource("classpath:/application-test.properties") @TestInstance(Lifecycle.PER_CLASS) public class GraderTest { diff --git a/src/test/java/de/extio/lmlib/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/token/TokenizerTest.java index 8775dbb..c1c8631 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerTest.java @@ -1,9 +1,6 @@ package de.extio.lmlib.token; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.math.BigDecimal; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -13,7 +10,7 @@ import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.context.annotation.PropertySource; +import org.springframework.test.context.TestPropertySource; import org.springframework.web.client.RestClient; import de.extio.lmlib.profile.ModelCategory; @@ -22,7 +19,7 @@ import de.extio.lmlib.profile.ModelProfileService; @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @EnableAutoConfiguration -@PropertySource("classpath:/application-test.properties") +@TestPropertySource("classpath:/application-test.properties") public class TokenizerTest { private static final String TEXT = "Hello this is a text! How are you doing? The weather is great today! Super sunny shiny!!!"; diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties index 676d257..65307de 100755 --- a/src/test/resources/application-test.properties +++ b/src/test/resources/application-test.properties @@ -6,8 +6,12 @@ profile.model.cold=local-chatcompletion profile.model.textcompletion=local-completion profile.model.chatcompletion=local-chatcompletion +profile.model.ollama=local-ollama-chatcompletion #profile.model.azure=llama3.3-70b-azure agent.threads=2 client.collectStatistics=true + +logging.level.root=INFO +logging.level.de.extio.lmlib=DEBUG diff --git a/src/test/resources/local-ollama-chatcompletion.properties b/src/test/resources/local-ollama-chatcompletion.properties new file mode 100644 index 0000000..de5490c --- /dev/null +++ b/src/test/resources/local-ollama-chatcompletion.properties @@ -0,0 +1,16 @@ +category=OLLAMA +prompts= +tokenizer=fallback +tokenEncoding=cl100k_base +maxTokens=2500 +maxContextLength=32768 +temperature=0.3 +topP=0.95 +modelProvider=OLLAMA +modelName= +url=http://localhost:11434 +apiKey= +cost1MInTokens=0 +cost1MCachedInTokens=0 +cost1MOutTokens=0 +cost1MReasoningOutTokens=0 \ No newline at end of file diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml index ed0f24b..a817ba1 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback.xml @@ -1,5 +1,5 @@ - +