diff --git a/README.md b/README.md index d98833c..d505d79 100644 --- a/README.md +++ b/README.md @@ -250,8 +250,6 @@ Each profile file is a `.properties` resource on the classpath. | `cost1MCachedInTokens` | Double | No | Cost per 1 million cached input tokens. Defaults to `cost1MInTokens`. | `0.35`, `0` | | `cost1MOutTokens` | Double | Yes | Cost per 1 million output tokens. | `0.71`, `15.00`, `0` | | `cost1MReasoningOutTokens` | Double | No | Cost per 1 million reasoning output tokens. Defaults to `0`. | `0`, `60.00` | -| `reasoningEffort` | String | No | Reasoning effort for reasoning-capable models. | `low`, `medium`, `high` | -| `reasoningSummaryDetails` | String | No | How reasoning content should be summarized. | `auto`, `concise`, `detailed` | Notes: @@ -260,7 +258,6 @@ Notes: - Cost properties are only used for statistics and accounting. - `cost1MCachedInTokens` defaults to the normal input token cost when not provided. - `cost1MReasoningOutTokens` defaults to `0` when not provided. -- Reasoning properties only apply to models that expose separate reasoning output. - `tokenEncoding` is primarily relevant for tokenizer implementations that support multiple encodings, such as `jTokkit`. ### Example Profile Files @@ -282,8 +279,6 @@ url=http://localhost:5001 apiKey= cost1MInTokens=0 cost1MOutTokens=0 -reasoningEffort=medium -reasoningSummaryDetails=auto ``` Local text completion: @@ -322,6 +317,52 @@ Tool calling is currently supported only for chat completion models. `TextComple These built-in providers are OpenAI-compatible today, but that is an implementation detail of the current client set, not a hard architectural limit of the library. Additional provider integrations can be added in the library itself, and applications can plug in their own client implementations where needed. +If you need to adapt request shaping for a specific OpenAI-compatible provider, you can optionally register an `OpenAiProviderDialect` bean. This allows applications to suppress request fields that a provider does not support while keeping `ModelProfile` provider-agnostic and runtime-driven. + +The default dialect behavior is: + +- `sendUsage(...) = true` +- `sendReasoning(...) = true` +- `sendMaxCompletionTokens(...) = true` +- `reasoningEffort(...) = "medium"` +- `reasoningSummaryDetails(...) = "concise"` + +Example: + +```java +@Bean +OpenAiProviderDialect azureOpenAiProviderDialect() { + return new OpenAiProviderDialect() { + @Override + public boolean sendUsage(final ModelProfile modelProfile) { + return false; + } + + @Override + public boolean sendReasoning(final ModelProfile modelProfile) { + return false; + } + + @Override + public boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { + return false; + } + + @Override + public String reasoningEffort(final ModelProfile modelProfile) { + return "medium"; + } + + @Override + public String reasoningSummaryDetails(final ModelProfile modelProfile) { + return "concise"; + } + }; +} +``` + +If no `OpenAiProviderDialect` bean is present, all OpenAI-compatible request features remain enabled by default. + For text completions, select a prompt strategy with the `prompts` property. lm-lib ships built-in strategies for common model families including Llama, GPT-OSS, Gemma, Mistral, Phi, Qwen, Vicuna, ChatML, Alpaca, and a no-formatting strategy. Example: diff --git a/src/main/java/de/extio/lmlib/client/oai/OpenAiProviderDialect.java b/src/main/java/de/extio/lmlib/client/oai/OpenAiProviderDialect.java new file mode 100644 index 0000000..16e042e --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/OpenAiProviderDialect.java @@ -0,0 +1,26 @@ +package de.extio.lmlib.client.oai; + +import de.extio.lmlib.profile.ModelProfile; + +public interface OpenAiProviderDialect { + + default boolean sendUsage(final ModelProfile modelProfile) { + return true; + } + + default boolean sendReasoning(final ModelProfile modelProfile) { + return true; + } + + default boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { + return true; + } + + default String reasoningEffort(final ModelProfile modelProfile) { + return "medium"; + } + + default String reasoningSummaryDetails(final ModelProfile modelProfile) { + return "concise"; + } +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index ca999cf..51935bd 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -35,6 +35,7 @@ import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.ToolCallData; import de.extio.lmlib.client.oai.Model; import de.extio.lmlib.client.oai.ModelsResponse; +import de.extio.lmlib.client.oai.OpenAiProviderDialect; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; @@ -44,6 +45,9 @@ import de.extio.lmlib.token.TokenizerResolver; public abstract class AbstractCompletionClient implements Client, DisposableBean { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class); + + private static final OpenAiProviderDialect DEFAULT_OPENAI_PROVIDER_DIALECT = new OpenAiProviderDialect() { + }; @Autowired @Qualifier("lmLibRestClientBuilder") @@ -54,6 +58,9 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean @Autowired protected ModelProfileService modelProfileService; + + @Autowired(required = false) + protected OpenAiProviderDialect openAiProviderDialect; @Value("${client.collectStatistics:false}") protected boolean collectStatistics; @@ -137,6 +144,30 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean protected abstract Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer chunkConsumer, final ToolCallData toolCallData); + protected OpenAiProviderDialect getOpenAiProviderDialect() { + return this.openAiProviderDialect == null ? DEFAULT_OPENAI_PROVIDER_DIALECT : this.openAiProviderDialect; + } + + protected boolean sendUsage(final ModelProfile modelProfile) { + return this.getOpenAiProviderDialect().sendUsage(modelProfile); + } + + protected boolean sendReasoning(final ModelProfile modelProfile) { + return this.getOpenAiProviderDialect().sendReasoning(modelProfile); + } + + protected boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { + return this.getOpenAiProviderDialect().sendMaxCompletionTokens(modelProfile); + } + + protected String reasoningEffort(final ModelProfile modelProfile) { + return this.getOpenAiProviderDialect().reasoningEffort(modelProfile); + } + + protected String reasoningSummaryDetails(final ModelProfile modelProfile) { + return this.getOpenAiProviderDialect().reasoningSummaryDetails(modelProfile); + } + protected String getPrimaryModelName(final ModelProfile modelProfile) { return this.getModelNames(modelProfile, false).stream().findFirst().orElse(""); } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index ff9eba5..8bcbda7 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -48,45 +48,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { @Override protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer chunkConsumer, final ToolCallData toolCallData) { final var chat = createChats(conversation, modelProfile); - - final var request = new ChatCompletionRequest(); - request.setModel(this.getPrimaryModelName(modelProfile)); - request.setMessages(chat); - request.setMaxTokens(modelProfile.maxTokens()); - request.setMaxCompletionTokens(modelProfile.maxTokens()); - request.setTemperature(modelProfile.temperature()); - request.setTopP(modelProfile.topP()); - request.setStream(chunkConsumer != null); - if (chunkConsumer != null) { - this.configureStreamOptions(true, request::setStreamOptions); - } else { - request.setUsage(true); - } - if (modelProfile.reasoningEffort() != null && !modelProfile.reasoningEffort().isBlank()) { - request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); - try { - request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(modelProfile.reasoningEffort())); - } - catch (final IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid reasoning effort in the model profile: " + modelProfile.reasoningEffort(), e); - } - } - if (modelProfile.reasoningSummaryDetails() != null && !modelProfile.reasoningSummaryDetails().isBlank()) { - if (request.getReasoning() == null) { - request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); - } - try { - request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(modelProfile.reasoningSummaryDetails()).name()); - } - catch (final IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid reasoning summary details in the model profile: " + modelProfile.reasoningSummaryDetails(), e); - } - } - if (toolCallData != null && toolCallData.hasTools()) { - request.setTools(this.createTools(toolCallData.tools())); - request.setToolChoice(this.createToolChoice(toolCallData)); - request.setParallelToolCalls(toolCallData.parallelToolCalls()); - } + final var request = this.createRequest(chat, modelProfile, chunkConsumer != null, toolCallData); String requestBody; try { @@ -203,6 +165,53 @@ public class ChatCompletionClient extends AbstractCompletionClient { return new Completion(content, reasoning, finishReason, statistics, toolCalls); } + ChatCompletionRequest createRequest(final List chat, final ModelProfile modelProfile, final boolean streaming, final ToolCallData toolCallData) { + final var request = new ChatCompletionRequest(); + request.setModel(this.getPrimaryModelName(modelProfile)); + request.setMessages(chat); + request.setMaxTokens(modelProfile.maxTokens()); + if (this.sendMaxCompletionTokens(modelProfile)) { + request.setMaxCompletionTokens(modelProfile.maxTokens()); + } + request.setTemperature(modelProfile.temperature()); + request.setTopP(modelProfile.topP()); + request.setStream(streaming); + if (streaming) { + this.configureStreamOptions(true, request::setStreamOptions); + } + else if (this.sendUsage(modelProfile)) { + request.setUsage(true); + } + final var reasoningEffort = this.reasoningEffort(modelProfile); + if (this.sendReasoning(modelProfile) && reasoningEffort != null && !reasoningEffort.isBlank()) { + request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); + try { + request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(reasoningEffort)); + } + catch (final IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid reasoning effort for the current OAI provider dialect: " + reasoningEffort, e); + } + } + final var reasoningSummaryDetails = this.reasoningSummaryDetails(modelProfile); + if (this.sendReasoning(modelProfile) && reasoningSummaryDetails != null && !reasoningSummaryDetails.isBlank()) { + if (request.getReasoning() == null) { + request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); + } + try { + request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(reasoningSummaryDetails).name()); + } + catch (final IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid reasoning summary details for the current OAI provider dialect: " + reasoningSummaryDetails, e); + } + } + if (toolCallData != null && toolCallData.hasTools()) { + request.setTools(this.createTools(toolCallData.tools())); + request.setToolChoice(this.createToolChoice(toolCallData)); + request.setParallelToolCalls(toolCallData.parallelToolCalls()); + } + return request; + } + private List createTools(final List toolDefinitions) { return toolDefinitions.stream().map(toolDefinition -> { final var function = new ChatCompletionRequest.ChatCompletionFunction(); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 44ed82a..4e18b02 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -53,19 +53,7 @@ public class TextCompletionClient extends AbstractCompletionClient { } final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy); - - final var request = new CompletionRequest(); - request.setModel(this.getPrimaryModelName(modelProfile)); - request.setPrompt(prompt); - request.setMaxTokens(modelProfile.maxTokens()); - request.setTemperature(modelProfile.temperature()); - request.setTopP(modelProfile.topP()); - request.setStream(chunkConsumer != null); - if (chunkConsumer != null) { - this.configureStreamOptions(true, request::setStreamOptions); - } else { - request.setUsage(true); - } + final var request = this.createRequest(prompt, modelProfile, chunkConsumer != null); String requestBody; try { @@ -196,6 +184,23 @@ public class TextCompletionClient extends AbstractCompletionClient { final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), response.getTimings(), prompt, content, reasoning); return new Completion(content, reasoning, finishReason, statistics); } + + CompletionRequest createRequest(final String prompt, final ModelProfile modelProfile, final boolean streaming) { + final var request = new CompletionRequest(); + request.setModel(this.getPrimaryModelName(modelProfile)); + request.setPrompt(prompt); + request.setMaxTokens(modelProfile.maxTokens()); + request.setTemperature(modelProfile.temperature()); + request.setTopP(modelProfile.topP()); + request.setStream(streaming); + if (streaming) { + this.configureStreamOptions(true, request::setStreamOptions); + } + else if (this.sendUsage(modelProfile)) { + request.setUsage(true); + } + return request; + } private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) { StringBuilder prompt = this.formatConversation(conversation, promptStrategy); diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfile.java b/src/main/java/de/extio/lmlib/profile/ModelProfile.java index ea31fc8..27b37e5 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfile.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfile.java @@ -18,8 +18,6 @@ public record ModelProfile( BigDecimal costPerCachedInToken, BigDecimal costPerOutToken, BigDecimal costPerReasoningOutToken, - String reasoningEffort, - String reasoningSummaryDetails, String category) { public ModelProfile { diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java index d44b2ad..d8027fa 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java @@ -90,12 +90,6 @@ public class ModelProfileService { .map(Object::toString) .map(value -> new BigDecimal(value).divide(new BigDecimal(1000000.0))) .orElse(BigDecimal.ZERO); - final var reasoningEffort = Optional.ofNullable(resource.getProperty("reasoningEffort")) - .map(Object::toString) - .orElse(null); - final var reasoningSummaryDetails = Optional.ofNullable(resource.getProperty("reasoningSummaryDetails")) - .map(Object::toString) - .orElse(null); final var category = Optional .ofNullable(modelCategory) .map(ModelCategory::getShortName) @@ -112,7 +106,7 @@ public class ModelProfileService { .orElse(null); } - return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category); + return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, category); } catch (final IOException e) { LOGGER.error("Error while reading model profile", e); diff --git a/src/test/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClientRequestTest.java b/src/test/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClientRequestTest.java new file mode 100644 index 0000000..0f19b19 --- /dev/null +++ b/src/test/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClientRequestTest.java @@ -0,0 +1,121 @@ +package de.extio.lmlib.client.oai.completion.chat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.math.BigDecimal; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import de.extio.lmlib.client.oai.OpenAiProviderDialect; +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; + +class ChatCompletionClientRequestTest { + + @Test + void defaultsToAllRequestFieldsWhenNoDialectIsConfigured() { + final var client = new TestChatCompletionClient(); + final var request = client.createRequest( + List.of(new ChatMessage("user", "hello", null)), + this.createProfile(), + false, + null); + + assertTrue(request.getUsage()); + assertNotNull(request.getReasoning()); + assertEquals(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.medium, request.getReasoning().getEffort()); + assertEquals("concise", request.getReasoning().getSummary()); + assertEquals(2048, request.getMaxCompletionTokens()); + } + + @Test + void omitsDialectDisabledRequestFields() { + final var client = new TestChatCompletionClient(); + client.setDialect(new OpenAiProviderDialect() { + @Override + public boolean sendUsage(final ModelProfile modelProfile) { + return false; + } + + @Override + public boolean sendReasoning(final ModelProfile modelProfile) { + return false; + } + + @Override + public boolean sendMaxCompletionTokens(final ModelProfile modelProfile) { + return false; + } + }); + + final var request = client.createRequest( + List.of(new ChatMessage("user", "hello", null)), + this.createProfile(), + false, + null); + + assertNull(request.getUsage()); + assertNull(request.getReasoning()); + assertNull(request.getMaxCompletionTokens()); + assertEquals(2048, request.getMaxTokens()); + assertFalse(request.isStream()); + } + + @Test + void usesDialectProvidedReasoningDefaults() { + final var client = new TestChatCompletionClient(); + client.setDialect(new OpenAiProviderDialect() { + @Override + public String reasoningEffort(final ModelProfile modelProfile) { + return "high"; + } + + @Override + public String reasoningSummaryDetails(final ModelProfile modelProfile) { + return "detailed"; + } + }); + + final var request = client.createRequest( + List.of(new ChatMessage("user", "hello", null)), + this.createProfile(), + false, + null); + + assertNotNull(request.getReasoning()); + assertEquals(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.high, request.getReasoning().getEffort()); + assertEquals("detailed", request.getReasoning().getSummary()); + } + + private ModelProfile createProfile() { + return new ModelProfile( + "", + "fallback", + "cl100k_base", + 2048, + 8192, + 0.4, + 1.0, + ModelProvider.OAI_CHAT_COMPLETION, + "test-model", + "http://localhost:5001", + "", + BigDecimal.ZERO, + BigDecimal.ZERO, + BigDecimal.ZERO, + BigDecimal.ZERO, + "TEST"); + } + + private static final class TestChatCompletionClient extends ChatCompletionClient { + + private void setDialect(final OpenAiProviderDialect openAiProviderDialect) { + this.openAiProviderDialect = openAiProviderDialect; + } + } +} diff --git a/src/test/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClientRequestTest.java b/src/test/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClientRequestTest.java new file mode 100644 index 0000000..cdf8299 --- /dev/null +++ b/src/test/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClientRequestTest.java @@ -0,0 +1,65 @@ +package de.extio.lmlib.client.oai.completion.text; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.math.BigDecimal; + +import org.junit.jupiter.api.Test; + +import de.extio.lmlib.client.oai.OpenAiProviderDialect; +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; + +class TextCompletionClientRequestTest { + + @Test + void defaultsToUsageWhenNoDialectIsConfigured() { + final var client = new TestTextCompletionClient(); + final var request = client.createRequest("prompt", this.createProfile(), false); + + assertTrue(request.getUsage()); + } + + @Test + void omitsUsageWhenDialectDisablesIt() { + final var client = new TestTextCompletionClient(); + client.setDialect(new OpenAiProviderDialect() { + @Override + public boolean sendUsage(final ModelProfile modelProfile) { + return false; + } + }); + + final var request = client.createRequest("prompt", this.createProfile(), false); + + assertNull(request.getUsage()); + } + + private ModelProfile createProfile() { + return new ModelProfile( + "gpt-oss", + "fallback", + "cl100k_base", + 2048, + 8192, + 0.4, + 1.0, + ModelProvider.OAI_TEXT_COMPLETION, + "test-model", + "http://localhost:5001", + "", + BigDecimal.ZERO, + BigDecimal.ZERO, + BigDecimal.ZERO, + BigDecimal.ZERO, + "TEST"); + } + + private static final class TestTextCompletionClient extends TextCompletionClient { + + private void setDialect(final OpenAiProviderDialect openAiProviderDialect) { + this.openAiProviderDialect = openAiProviderDialect; + } + } +} diff --git a/src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java b/src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java index a72c933..469923f 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java @@ -55,8 +55,6 @@ public class TokenizerResolverTest { BigDecimal.ZERO, BigDecimal.ZERO, BigDecimal.ZERO, - null, - null, "TEST"); } diff --git a/src/test/resources/local-chatcompletion.properties b/src/test/resources/local-chatcompletion.properties index 608ecfc..2ada5d7 100755 --- a/src/test/resources/local-chatcompletion.properties +++ b/src/test/resources/local-chatcompletion.properties @@ -13,6 +13,4 @@ apiKey= cost1MInTokens=0 cost1MCachedInTokens=0 cost1MOutTokens=0 -cost1MReasoningOutTokens=0 -reasoningEffort=medium -reasoningSummaryDetails=auto \ No newline at end of file +cost1MReasoningOutTokens=0 \ No newline at end of file