From 33f76defa5fe7ccfaa9ae4f7c84022a7dbbe729e Mon Sep 17 00:00:00 2001 From: Stephan Birkl Date: Thu, 7 May 2026 18:22:12 +0200 Subject: [PATCH] feat: Enhance OpenAiProviderDialect for improved request handling and update related classes - Refactored OpenAiProviderDialect to include methods for managing chat token limits and reasoning effort. - Updated ChatCompletionClient and AbstractCompletionClient to utilize the new dialect methods for request shaping. - Modified ChatCompletionRequest to replace reasoning properties with a single reasoningEffort field. - Adjusted tests to reflect changes in request handling and dialect behavior. - Updated README to clarify the new request shaping capabilities and defaults. --- README.md | 37 ++++---- .../client/oai/OpenAiProviderDialect.java | 89 +++++++++++++++++-- .../completion/AbstractCompletionClient.java | 10 +-- .../completion/chat/ChatCompletionClient.java | 48 +++++----- .../chat/ChatCompletionRequest.java | 47 ++-------- .../chat/ChatCompletionClientRequestTest.java | 64 +++++++------ .../text/TextCompletionClientRequestTest.java | 10 +-- 7 files changed, 177 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index d505d79..1ebcd98 100644 --- a/README.md +++ b/README.md @@ -202,8 +202,6 @@ final var modelProfile = new ModelProfile( new BigDecimal("0.125").divide(new BigDecimal(1000000)), new BigDecimal("10.0").divide(new BigDecimal(1000000)), BigDecimal.ZERO, - null, - null, "DYNAMIC" ); @@ -317,25 +315,31 @@ Tool calling is currently supported only for chat completion models. `TextComple These built-in providers are OpenAI-compatible today, but that is an implementation detail of the current client set, not a hard architectural limit of the library. Additional provider integrations can be added in the library itself, and applications can plug in their own client implementations where needed. -If you need to adapt request shaping for a specific OpenAI-compatible provider, you can optionally register an `OpenAiProviderDialect` bean. This allows applications to suppress request fields that a provider does not support while keeping `ModelProfile` provider-agnostic and runtime-driven. +If you need to adapt request shaping for a specific OpenAI-compatible provider, you can optionally register an `OpenAiProviderDialect` bean. This allows applications to suppress or switch request fields that a provider does not support while keeping `ModelProfile` provider-agnostic and runtime-driven. The default dialect behavior is: -- `sendUsage(...) = true` +- `sendUsage(...) = false` - `sendReasoning(...) = true` -- `sendMaxCompletionTokens(...) = true` -- `reasoningEffort(...) = "medium"` -- `reasoningSummaryDetails(...) = "concise"` +- `chatTokenLimitParameterMode(...) = ChatTokenLimitParameterMode.MAX_COMPLETION_TOKENS` +- `reasoningEffort(...) = ReasoningEffort.MEDIUM` + +Notes: + +- `usage` is a llama-server style extension and is therefore disabled unless a dialect explicitly enables it. +- `reasoning_effort` is sent as a top-level request field when reasoning is enabled. +- `max_completion_tokens` is the default because it matches the current OpenAI recommendation. +- Some older OpenAI-compatible servers still require `max_tokens`, which can be selected through the dialect. Example: ```java @Bean -OpenAiProviderDialect azureOpenAiProviderDialect() { +OpenAiProviderDialect legacyOpenAiProviderDialect() { return new OpenAiProviderDialect() { @Override public boolean sendUsage(final ModelProfile modelProfile) { - return false; + return true; } @Override @@ -344,24 +348,19 @@ OpenAiProviderDialect azureOpenAiProviderDialect() { } @Override - public boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { - return false; + public ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) { + return ChatTokenLimitParameterMode.MAX_TOKENS; } @Override - public String reasoningEffort(final ModelProfile modelProfile) { - return "medium"; - } - - @Override - public String reasoningSummaryDetails(final ModelProfile modelProfile) { - return "concise"; + public ReasoningEffort reasoningEffort(final ModelProfile modelProfile) { + return ReasoningEffort.MEDIUM; } }; } ``` -If no `OpenAiProviderDialect` bean is present, all OpenAI-compatible request features remain enabled by default. +If no `OpenAiProviderDialect` bean is present, lm-lib follows the official-oriented defaults above, with `usage` left disabled unless you opt in. For text completions, select a prompt strategy with the `prompts` property. lm-lib ships built-in strategies for common model families including Llama, GPT-OSS, Gemma, Mistral, Phi, Qwen, Vicuna, ChatML, Alpaca, and a no-formatting strategy. diff --git a/src/main/java/de/extio/lmlib/client/oai/OpenAiProviderDialect.java b/src/main/java/de/extio/lmlib/client/oai/OpenAiProviderDialect.java index 16e042e..18930cf 100644 --- a/src/main/java/de/extio/lmlib/client/oai/OpenAiProviderDialect.java +++ b/src/main/java/de/extio/lmlib/client/oai/OpenAiProviderDialect.java @@ -1,26 +1,101 @@ package de.extio.lmlib.client.oai; +import com.fasterxml.jackson.annotation.JsonValue; + import de.extio.lmlib.profile.ModelProfile; +/** + * Optional dialect hook for OpenAI-compatible clients. + *

+ * Applications can register this as a Spring bean to adapt request shaping for + * providers that are not fully aligned with the current OpenAI Chat + * Completions API surface. + *

+ * If no bean is present, lm-lib uses the defaults defined on this interface. + */ public interface OpenAiProviderDialect { + /** + * Whether lm-lib should send the top-level {@code usage} request flag. + *

+ * This is disabled by default because it is not part of the official OpenAI + * Chat Completions request schema and is mainly useful for llama-server style + * extensions. + * + * @param modelProfile the resolved model profile + * @return {@code true} if the request should include the top-level + * {@code usage} flag + */ default boolean sendUsage(final ModelProfile modelProfile) { - return true; + return false; } + /** + * Whether lm-lib should send a top-level {@code reasoning_effort} request + * field. + * + * @param modelProfile the resolved model profile + * @return {@code true} if {@code reasoning_effort} should be sent + */ default boolean sendReasoning(final ModelProfile modelProfile) { return true; } - default boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { - return true; + /** + * Selects which token limit request parameter should be sent for chat + * completions. + *

+ * {@code max_completion_tokens} is the current OpenAI recommendation, while + * older llama-server style implementations may still require + * {@code max_tokens}. + * + * @param modelProfile the resolved model profile + * @return the token limit parameter mode to use + */ + default ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) { + return ChatTokenLimitParameterMode.MAX_COMPLETION_TOKENS; } - default String reasoningEffort(final ModelProfile modelProfile) { - return "medium"; + /** + * Returns the reasoning effort to send for chat completions when reasoning is + * enabled. + * + * @param modelProfile the resolved model profile + * @return the reasoning effort value to send + */ + default ReasoningEffort reasoningEffort(final ModelProfile modelProfile) { + return ReasoningEffort.MEDIUM; } - default String reasoningSummaryDetails(final ModelProfile modelProfile) { - return "concise"; + /** + * Controls which chat token limit field lm-lib should send. + */ + enum ChatTokenLimitParameterMode { + MAX_COMPLETION_TOKENS, + MAX_TOKENS, + BOTH + } + + /** + * Official OpenAI reasoning effort values for chat completions. + */ + enum ReasoningEffort { + NONE("none"), + MINIMAL("minimal"), + LOW("low"), + MEDIUM("medium"), + HIGH("high"), + XHIGH("xhigh"); + + private final String wireValue; + + ReasoningEffort(final String wireValue) { + this.wireValue = wireValue; + } + + @JsonValue + public String getWireValue() { + return this.wireValue; + } } } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index 51935bd..d2f7765 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -156,18 +156,14 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean return this.getOpenAiProviderDialect().sendReasoning(modelProfile); } - protected boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { - return this.getOpenAiProviderDialect().sendMaxCompletionTokens(modelProfile); + protected OpenAiProviderDialect.ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) { + return this.getOpenAiProviderDialect().chatTokenLimitParameterMode(modelProfile); } - protected String reasoningEffort(final ModelProfile modelProfile) { + protected OpenAiProviderDialect.ReasoningEffort reasoningEffort(final ModelProfile modelProfile) { return this.getOpenAiProviderDialect().reasoningEffort(modelProfile); } - protected String reasoningSummaryDetails(final ModelProfile modelProfile) { - return this.getOpenAiProviderDialect().reasoningSummaryDetails(modelProfile); - } - protected String getPrimaryModelName(final ModelProfile modelProfile) { return this.getModelNames(modelProfile, false).stream().findFirst().orElse(""); } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index 8bcbda7..8295814 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -169,10 +169,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { final var request = new ChatCompletionRequest(); request.setModel(this.getPrimaryModelName(modelProfile)); request.setMessages(chat); - request.setMaxTokens(modelProfile.maxTokens()); - if (this.sendMaxCompletionTokens(modelProfile)) { - request.setMaxCompletionTokens(modelProfile.maxTokens()); - } + this.applyChatTokenLimitParameterMode(request, modelProfile); request.setTemperature(modelProfile.temperature()); request.setTopP(modelProfile.topP()); request.setStream(streaming); @@ -183,26 +180,8 @@ public class ChatCompletionClient extends AbstractCompletionClient { request.setUsage(true); } final var reasoningEffort = this.reasoningEffort(modelProfile); - if (this.sendReasoning(modelProfile) && reasoningEffort != null && !reasoningEffort.isBlank()) { - request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); - try { - request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(reasoningEffort)); - } - catch (final IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid reasoning effort for the current OAI provider dialect: " + reasoningEffort, e); - } - } - final var reasoningSummaryDetails = this.reasoningSummaryDetails(modelProfile); - if (this.sendReasoning(modelProfile) && reasoningSummaryDetails != null && !reasoningSummaryDetails.isBlank()) { - if (request.getReasoning() == null) { - request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); - } - try { - request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(reasoningSummaryDetails).name()); - } - catch (final IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid reasoning summary details for the current OAI provider dialect: " + reasoningSummaryDetails, e); - } + if (this.sendReasoning(modelProfile) && reasoningEffort != null) { + request.setReasoningEffort(reasoningEffort); } if (toolCallData != null && toolCallData.hasTools()) { request.setTools(this.createTools(toolCallData.tools())); @@ -212,6 +191,27 @@ public class ChatCompletionClient extends AbstractCompletionClient { return request; } + private void applyChatTokenLimitParameterMode(final ChatCompletionRequest request, final ModelProfile modelProfile) { + final var mode = this.chatTokenLimitParameterMode(modelProfile); + if (mode == null) { + throw new IllegalArgumentException("Chat token limit parameter mode must not be null"); + } + switch (mode) { + case MAX_COMPLETION_TOKENS: + request.setMaxCompletionTokens(modelProfile.maxTokens()); + break; + case MAX_TOKENS: + request.setMaxTokens(modelProfile.maxTokens()); + break; + case BOTH: + request.setMaxCompletionTokens(modelProfile.maxTokens()); + request.setMaxTokens(modelProfile.maxTokens()); + break; + default: + throw new IllegalArgumentException("Unsupported chat token limit parameter mode: " + mode); + } + } + private List createTools(final List toolDefinitions) { return toolDefinitions.stream().map(toolDefinition -> { final var function = new ChatCompletionRequest.ChatCompletionFunction(); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java index aa0437e..6187097 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java @@ -5,6 +5,7 @@ import java.util.Map; import com.fasterxml.jackson.annotation.*; +import de.extio.lmlib.client.oai.OpenAiProviderDialect.ReasoningEffort; import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest; final class ChatCompletionRequest extends AbstractCompletionRequest { @@ -13,7 +14,7 @@ final class ChatCompletionRequest extends AbstractCompletionRequest { private Integer maxCompletionTokens; - private ChatCompletionsRequestReasoning reasoning; + private ReasoningEffort reasoningEffort; private List tools; @@ -39,13 +40,13 @@ final class ChatCompletionRequest extends AbstractCompletionRequest { this.maxCompletionTokens = maxCompletionTokens; } - @JsonProperty("reasoning") - public ChatCompletionsRequestReasoning getReasoning() { - return this.reasoning; + @JsonProperty("reasoning_effort") + public ReasoningEffort getReasoningEffort() { + return this.reasoningEffort; } - public void setReasoning(final ChatCompletionsRequestReasoning reasoning) { - this.reasoning = reasoning; + public void setReasoningEffort(final ReasoningEffort reasoningEffort) { + this.reasoningEffort = reasoningEffort; } @JsonProperty("tools") @@ -75,40 +76,6 @@ final class ChatCompletionRequest extends AbstractCompletionRequest { this.parallelToolCalls = parallelToolCalls; } - final static class ChatCompletionsRequestReasoning { - - private ChatCompletionsRequestReasoningEffort effort; - - private String summary; - - @JsonProperty("effort") - public ChatCompletionsRequestReasoningEffort getEffort() { - return this.effort; - } - - public void setEffort(final ChatCompletionsRequestReasoningEffort effort) { - this.effort = effort; - } - - @JsonProperty("summary") - public String getSummary() { - return this.summary; - } - - public void setSummary(final String summary) { - this.summary = summary; - } - - } - - static enum ChatCompletionsRequestReasoningEffort { - minimal, low, medium, high - } - - static enum ChatCompletionsRequestReasoningSummaryDetails { - auto, concise, detailed - } - static final class ChatCompletionTool { private String type; diff --git a/src/test/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClientRequestTest.java b/src/test/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClientRequestTest.java index 0f19b19..f9edfd5 100644 --- a/src/test/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClientRequestTest.java +++ b/src/test/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClientRequestTest.java @@ -2,7 +2,6 @@ package de.extio.lmlib.client.oai.completion.chat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -18,7 +17,7 @@ import de.extio.lmlib.profile.ModelProfile.ModelProvider; class ChatCompletionClientRequestTest { @Test - void defaultsToAllRequestFieldsWhenNoDialectIsConfigured() { + void defaultsToOfficialRequestFieldsWhenNoDialectIsConfigured() { final var client = new TestChatCompletionClient(); final var request = client.createRequest( List.of(new ChatMessage("user", "hello", null)), @@ -26,30 +25,24 @@ class ChatCompletionClientRequestTest { false, null); - assertTrue(request.getUsage()); - assertNotNull(request.getReasoning()); - assertEquals(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.medium, request.getReasoning().getEffort()); - assertEquals("concise", request.getReasoning().getSummary()); + assertNull(request.getUsage()); + assertEquals(OpenAiProviderDialect.ReasoningEffort.MEDIUM, request.getReasoningEffort()); assertEquals(2048, request.getMaxCompletionTokens()); + assertNull(request.getMaxTokens()); } @Test - void omitsDialectDisabledRequestFields() { + void omitsDialectDisabledReasoningAndUsesLegacyMaxTokens() { final var client = new TestChatCompletionClient(); client.setDialect(new OpenAiProviderDialect() { - @Override - public boolean sendUsage(final ModelProfile modelProfile) { - return false; - } - @Override public boolean sendReasoning(final ModelProfile modelProfile) { return false; } @Override - public boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { - return false; + public ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) { + return ChatTokenLimitParameterMode.MAX_TOKENS; } }); @@ -60,24 +53,19 @@ class ChatCompletionClientRequestTest { null); assertNull(request.getUsage()); - assertNull(request.getReasoning()); + assertNull(request.getReasoningEffort()); assertNull(request.getMaxCompletionTokens()); assertEquals(2048, request.getMaxTokens()); assertFalse(request.isStream()); } @Test - void usesDialectProvidedReasoningDefaults() { + void usesDialectProvidedReasoningEffort() { final var client = new TestChatCompletionClient(); client.setDialect(new OpenAiProviderDialect() { @Override - public String reasoningEffort(final ModelProfile modelProfile) { - return "high"; - } - - @Override - public String reasoningSummaryDetails(final ModelProfile modelProfile) { - return "detailed"; + public ReasoningEffort reasoningEffort(final ModelProfile modelProfile) { + return ReasoningEffort.HIGH; } }); @@ -87,9 +75,33 @@ class ChatCompletionClientRequestTest { false, null); - assertNotNull(request.getReasoning()); - assertEquals(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.high, request.getReasoning().getEffort()); - assertEquals("detailed", request.getReasoning().getSummary()); + assertEquals(OpenAiProviderDialect.ReasoningEffort.HIGH, request.getReasoningEffort()); + } + + @Test + void canEnableUsageAndSendBothTokenLimitFields() { + final var client = new TestChatCompletionClient(); + client.setDialect(new OpenAiProviderDialect() { + @Override + public boolean sendUsage(final ModelProfile modelProfile) { + return true; + } + + @Override + public ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) { + return ChatTokenLimitParameterMode.BOTH; + } + }); + + final var request = client.createRequest( + List.of(new ChatMessage("user", "hello", null)), + this.createProfile(), + false, + null); + + assertTrue(request.getUsage()); + assertEquals(2048, request.getMaxCompletionTokens()); + assertEquals(2048, request.getMaxTokens()); } private ModelProfile createProfile() { diff --git a/src/test/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClientRequestTest.java b/src/test/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClientRequestTest.java index cdf8299..3430078 100644 --- a/src/test/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClientRequestTest.java +++ b/src/test/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClientRequestTest.java @@ -14,26 +14,26 @@ import de.extio.lmlib.profile.ModelProfile.ModelProvider; class TextCompletionClientRequestTest { @Test - void defaultsToUsageWhenNoDialectIsConfigured() { + void omitsUsageWhenNoDialectIsConfigured() { final var client = new TestTextCompletionClient(); final var request = client.createRequest("prompt", this.createProfile(), false); - assertTrue(request.getUsage()); + assertNull(request.getUsage()); } @Test - void omitsUsageWhenDialectDisablesIt() { + void includesUsageWhenDialectEnablesIt() { final var client = new TestTextCompletionClient(); client.setDialect(new OpenAiProviderDialect() { @Override public boolean sendUsage(final ModelProfile modelProfile) { - return false; + return true; } }); final var request = client.createRequest("prompt", this.createProfile(), false); - assertNull(request.getUsage()); + assertTrue(request.getUsage()); } private ModelProfile createProfile() {