Merge branch 'main' of github.com:extio-de/lm-lib
This commit is contained in:
56
README.md
56
README.md
@@ -47,7 +47,7 @@ The published artifacts are hosted on Maven Central, so no additional repository
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>de.extio</groupId>
|
<groupId>de.extio</groupId>
|
||||||
<artifactId>lm-lib</artifactId>
|
<artifactId>lm-lib</artifactId>
|
||||||
<version>2.6.0</version>
|
<version>2.8.0</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -202,8 +202,6 @@ final var modelProfile = new ModelProfile(
|
|||||||
new BigDecimal("0.125").divide(new BigDecimal(1000000)),
|
new BigDecimal("0.125").divide(new BigDecimal(1000000)),
|
||||||
new BigDecimal("10.0").divide(new BigDecimal(1000000)),
|
new BigDecimal("10.0").divide(new BigDecimal(1000000)),
|
||||||
BigDecimal.ZERO,
|
BigDecimal.ZERO,
|
||||||
null,
|
|
||||||
null,
|
|
||||||
"DYNAMIC"
|
"DYNAMIC"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -250,8 +248,6 @@ Each profile file is a `.properties` resource on the classpath.
|
|||||||
| `cost1MCachedInTokens` | Double | No | Cost per 1 million cached input tokens. Defaults to `cost1MInTokens`. | `0.35`, `0` |
|
| `cost1MCachedInTokens` | Double | No | Cost per 1 million cached input tokens. Defaults to `cost1MInTokens`. | `0.35`, `0` |
|
||||||
| `cost1MOutTokens` | Double | Yes | Cost per 1 million output tokens. | `0.71`, `15.00`, `0` |
|
| `cost1MOutTokens` | Double | Yes | Cost per 1 million output tokens. | `0.71`, `15.00`, `0` |
|
||||||
| `cost1MReasoningOutTokens` | Double | No | Cost per 1 million reasoning output tokens. Defaults to `0`. | `0`, `60.00` |
|
| `cost1MReasoningOutTokens` | Double | No | Cost per 1 million reasoning output tokens. Defaults to `0`. | `0`, `60.00` |
|
||||||
| `reasoningEffort` | String | No | Reasoning effort for reasoning-capable models. | `low`, `medium`, `high` |
|
|
||||||
| `reasoningSummaryDetails` | String | No | How reasoning content should be summarized. | `auto`, `concise`, `detailed` |
|
|
||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
|
|
||||||
@@ -260,7 +256,6 @@ Notes:
|
|||||||
- Cost properties are only used for statistics and accounting.
|
- Cost properties are only used for statistics and accounting.
|
||||||
- `cost1MCachedInTokens` defaults to the normal input token cost when not provided.
|
- `cost1MCachedInTokens` defaults to the normal input token cost when not provided.
|
||||||
- `cost1MReasoningOutTokens` defaults to `0` when not provided.
|
- `cost1MReasoningOutTokens` defaults to `0` when not provided.
|
||||||
- Reasoning properties only apply to models that expose separate reasoning output.
|
|
||||||
- `tokenEncoding` is primarily relevant for tokenizer implementations that support multiple encodings, such as `jTokkit`.
|
- `tokenEncoding` is primarily relevant for tokenizer implementations that support multiple encodings, such as `jTokkit`.
|
||||||
|
|
||||||
### Example Profile Files
|
### Example Profile Files
|
||||||
@@ -282,8 +277,6 @@ url=http://localhost:5001
|
|||||||
apiKey=
|
apiKey=
|
||||||
cost1MInTokens=0
|
cost1MInTokens=0
|
||||||
cost1MOutTokens=0
|
cost1MOutTokens=0
|
||||||
reasoningEffort=medium
|
|
||||||
reasoningSummaryDetails=auto
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Local text completion:
|
Local text completion:
|
||||||
@@ -322,6 +315,53 @@ Tool calling is currently supported only for chat completion models. `TextComple
|
|||||||
|
|
||||||
These built-in providers are OpenAI-compatible today, but that is an implementation detail of the current client set, not a hard architectural limit of the library. Additional provider integrations can be added in the library itself, and applications can plug in their own client implementations where needed.
|
These built-in providers are OpenAI-compatible today, but that is an implementation detail of the current client set, not a hard architectural limit of the library. Additional provider integrations can be added in the library itself, and applications can plug in their own client implementations where needed.
|
||||||
|
|
||||||
|
If you need to adapt request shaping for a specific OpenAI-compatible provider, you can optionally register an `OpenAiProviderDialect` bean. This allows applications to suppress or switch request fields that a provider does not support while keeping `ModelProfile` provider-agnostic and runtime-driven.
|
||||||
|
|
||||||
|
The default dialect behavior is:
|
||||||
|
|
||||||
|
- `sendUsage(...) = false`
|
||||||
|
- `sendReasoning(...) = true`
|
||||||
|
- `chatTokenLimitParameterMode(...) = ChatTokenLimitParameterMode.MAX_COMPLETION_TOKENS`
|
||||||
|
- `reasoningEffort(...) = ReasoningEffort.MEDIUM`
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- `usage` is a llama-server style extension and is therefore disabled unless a dialect explicitly enables it.
|
||||||
|
- `reasoning_effort` is sent as a top-level request field when reasoning is enabled.
|
||||||
|
- `max_completion_tokens` is the default because it matches the current OpenAI recommendation.
|
||||||
|
- Some older OpenAI-compatible servers still require `max_tokens`, which can be selected through the dialect.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```java
|
||||||
|
@Bean
|
||||||
|
OpenAiProviderDialect legacyOpenAiProviderDialect() {
|
||||||
|
return new OpenAiProviderDialect() {
|
||||||
|
@Override
|
||||||
|
public boolean sendUsage(final ModelProfile modelProfile) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean sendReasoning(final ModelProfile modelProfile) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) {
|
||||||
|
return ChatTokenLimitParameterMode.MAX_TOKENS;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ReasoningEffort reasoningEffort(final ModelProfile modelProfile) {
|
||||||
|
return ReasoningEffort.MEDIUM;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
If no `OpenAiProviderDialect` bean is present, lm-lib follows the official-oriented defaults above, with `usage` left disabled unless you opt in.
|
||||||
|
|
||||||
For text completions, select a prompt strategy with the `prompts` property. lm-lib ships built-in strategies for common model families including Llama, GPT-OSS, Gemma, Mistral, Phi, Qwen, Vicuna, ChatML, Alpaca, and a no-formatting strategy.
|
For text completions, select a prompt strategy with the `prompts` property. lm-lib ships built-in strategies for common model families including Llama, GPT-OSS, Gemma, Mistral, Phi, Qwen, Vicuna, ChatML, Alpaca, and a no-formatting strategy.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
|
|||||||
2
pom.xml
2
pom.xml
@@ -6,7 +6,7 @@
|
|||||||
<name>lm-lib</name>
|
<name>lm-lib</name>
|
||||||
<description>Modular Java library for LLM-powered applications, model clients, and agentic orchestration.</description>
|
<description>Modular Java library for LLM-powered applications, model clients, and agentic orchestration.</description>
|
||||||
<url>https://github.com/extio-de/lm-lib</url>
|
<url>https://github.com/extio-de/lm-lib</url>
|
||||||
<version>2.6.0</version>
|
<version>2.8.0</version>
|
||||||
|
|
||||||
<licenses>
|
<licenses>
|
||||||
<license>
|
<license>
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
package de.extio.lmlib.client.oai;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonValue;
|
||||||
|
|
||||||
|
import de.extio.lmlib.profile.ModelProfile;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optional dialect hook for OpenAI-compatible clients.
|
||||||
|
* <p>
|
||||||
|
* Applications can register this as a Spring bean to adapt request shaping for
|
||||||
|
* providers that are not fully aligned with the current OpenAI Chat
|
||||||
|
* Completions API surface.
|
||||||
|
* <p>
|
||||||
|
* If no bean is present, lm-lib uses the defaults defined on this interface.
|
||||||
|
*/
|
||||||
|
public interface OpenAiProviderDialect {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether lm-lib should send the top-level {@code usage} request flag.
|
||||||
|
* <p>
|
||||||
|
* This is disabled by default because it is not part of the official OpenAI
|
||||||
|
* Chat Completions request schema and is mainly useful for llama-server style
|
||||||
|
* extensions.
|
||||||
|
*
|
||||||
|
* @param modelProfile the resolved model profile
|
||||||
|
* @return {@code true} if the request should include the top-level
|
||||||
|
* {@code usage} flag
|
||||||
|
*/
|
||||||
|
default boolean sendUsage(final ModelProfile modelProfile) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether lm-lib should send a top-level {@code reasoning_effort} request
|
||||||
|
* field.
|
||||||
|
*
|
||||||
|
* @param modelProfile the resolved model profile
|
||||||
|
* @return {@code true} if {@code reasoning_effort} should be sent
|
||||||
|
*/
|
||||||
|
default boolean sendReasoning(final ModelProfile modelProfile) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Selects which token limit request parameter should be sent for chat
|
||||||
|
* completions.
|
||||||
|
* <p>
|
||||||
|
* {@code max_completion_tokens} is the current OpenAI recommendation, while
|
||||||
|
* older llama-server style implementations may still require
|
||||||
|
* {@code max_tokens}.
|
||||||
|
*
|
||||||
|
* @param modelProfile the resolved model profile
|
||||||
|
* @return the token limit parameter mode to use
|
||||||
|
*/
|
||||||
|
default ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) {
|
||||||
|
return ChatTokenLimitParameterMode.MAX_COMPLETION_TOKENS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the reasoning effort to send for chat completions when reasoning is
|
||||||
|
* enabled.
|
||||||
|
*
|
||||||
|
* @param modelProfile the resolved model profile
|
||||||
|
* @return the reasoning effort value to send
|
||||||
|
*/
|
||||||
|
default ReasoningEffort reasoningEffort(final ModelProfile modelProfile) {
|
||||||
|
return ReasoningEffort.MEDIUM;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Controls which chat token limit field lm-lib should send.
|
||||||
|
*/
|
||||||
|
enum ChatTokenLimitParameterMode {
|
||||||
|
MAX_COMPLETION_TOKENS,
|
||||||
|
MAX_TOKENS,
|
||||||
|
BOTH
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Official OpenAI reasoning effort values for chat completions.
|
||||||
|
*/
|
||||||
|
enum ReasoningEffort {
|
||||||
|
NONE("none"),
|
||||||
|
MINIMAL("minimal"),
|
||||||
|
LOW("low"),
|
||||||
|
MEDIUM("medium"),
|
||||||
|
HIGH("high"),
|
||||||
|
XHIGH("xhigh");
|
||||||
|
|
||||||
|
private final String wireValue;
|
||||||
|
|
||||||
|
ReasoningEffort(final String wireValue) {
|
||||||
|
this.wireValue = wireValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonValue
|
||||||
|
public String getWireValue() {
|
||||||
|
return this.wireValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,6 +35,7 @@ import de.extio.lmlib.client.Conversation;
|
|||||||
import de.extio.lmlib.client.ToolCallData;
|
import de.extio.lmlib.client.ToolCallData;
|
||||||
import de.extio.lmlib.client.oai.Model;
|
import de.extio.lmlib.client.oai.Model;
|
||||||
import de.extio.lmlib.client.oai.ModelsResponse;
|
import de.extio.lmlib.client.oai.ModelsResponse;
|
||||||
|
import de.extio.lmlib.client.oai.OpenAiProviderDialect;
|
||||||
import de.extio.lmlib.profile.ModelCategory;
|
import de.extio.lmlib.profile.ModelCategory;
|
||||||
import de.extio.lmlib.profile.ModelProfile;
|
import de.extio.lmlib.profile.ModelProfile;
|
||||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||||
@@ -45,6 +46,9 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
|||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
|
||||||
|
|
||||||
|
private static final OpenAiProviderDialect DEFAULT_OPENAI_PROVIDER_DIALECT = new OpenAiProviderDialect() {
|
||||||
|
};
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
@Qualifier("lmLibRestClientBuilder")
|
@Qualifier("lmLibRestClientBuilder")
|
||||||
protected RestClient.Builder restClientBuilder;
|
protected RestClient.Builder restClientBuilder;
|
||||||
@@ -55,6 +59,9 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
|||||||
@Autowired
|
@Autowired
|
||||||
protected ModelProfileService modelProfileService;
|
protected ModelProfileService modelProfileService;
|
||||||
|
|
||||||
|
@Autowired(required = false)
|
||||||
|
protected OpenAiProviderDialect openAiProviderDialect;
|
||||||
|
|
||||||
@Value("${client.collectStatistics:false}")
|
@Value("${client.collectStatistics:false}")
|
||||||
protected boolean collectStatistics;
|
protected boolean collectStatistics;
|
||||||
|
|
||||||
@@ -137,6 +144,26 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
|||||||
|
|
||||||
protected abstract Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer, final ToolCallData toolCallData);
|
protected abstract Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer, final ToolCallData toolCallData);
|
||||||
|
|
||||||
|
protected OpenAiProviderDialect getOpenAiProviderDialect() {
|
||||||
|
return this.openAiProviderDialect == null ? DEFAULT_OPENAI_PROVIDER_DIALECT : this.openAiProviderDialect;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected boolean sendUsage(final ModelProfile modelProfile) {
|
||||||
|
return this.getOpenAiProviderDialect().sendUsage(modelProfile);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected boolean sendReasoning(final ModelProfile modelProfile) {
|
||||||
|
return this.getOpenAiProviderDialect().sendReasoning(modelProfile);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OpenAiProviderDialect.ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) {
|
||||||
|
return this.getOpenAiProviderDialect().chatTokenLimitParameterMode(modelProfile);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OpenAiProviderDialect.ReasoningEffort reasoningEffort(final ModelProfile modelProfile) {
|
||||||
|
return this.getOpenAiProviderDialect().reasoningEffort(modelProfile);
|
||||||
|
}
|
||||||
|
|
||||||
protected String getPrimaryModelName(final ModelProfile modelProfile) {
|
protected String getPrimaryModelName(final ModelProfile modelProfile) {
|
||||||
return this.getModelNames(modelProfile, false).stream().findFirst().orElse("");
|
return this.getModelNames(modelProfile, false).stream().findFirst().orElse("");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,45 +48,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
|||||||
@Override
|
@Override
|
||||||
protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer, final ToolCallData toolCallData) {
|
protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer, final ToolCallData toolCallData) {
|
||||||
final var chat = createChats(conversation, modelProfile);
|
final var chat = createChats(conversation, modelProfile);
|
||||||
|
final var request = this.createRequest(chat, modelProfile, chunkConsumer != null, toolCallData);
|
||||||
final var request = new ChatCompletionRequest();
|
|
||||||
request.setModel(this.getPrimaryModelName(modelProfile));
|
|
||||||
request.setMessages(chat);
|
|
||||||
request.setMaxTokens(modelProfile.maxTokens());
|
|
||||||
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
|
||||||
request.setTemperature(modelProfile.temperature());
|
|
||||||
request.setTopP(modelProfile.topP());
|
|
||||||
request.setStream(chunkConsumer != null);
|
|
||||||
if (chunkConsumer != null) {
|
|
||||||
this.configureStreamOptions(true, request::setStreamOptions);
|
|
||||||
} else {
|
|
||||||
request.setUsage(true);
|
|
||||||
}
|
|
||||||
if (modelProfile.reasoningEffort() != null && !modelProfile.reasoningEffort().isBlank()) {
|
|
||||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
|
||||||
try {
|
|
||||||
request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(modelProfile.reasoningEffort()));
|
|
||||||
}
|
|
||||||
catch (final IllegalArgumentException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid reasoning effort in the model profile: " + modelProfile.reasoningEffort(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (modelProfile.reasoningSummaryDetails() != null && !modelProfile.reasoningSummaryDetails().isBlank()) {
|
|
||||||
if (request.getReasoning() == null) {
|
|
||||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(modelProfile.reasoningSummaryDetails()).name());
|
|
||||||
}
|
|
||||||
catch (final IllegalArgumentException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid reasoning summary details in the model profile: " + modelProfile.reasoningSummaryDetails(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (toolCallData != null && toolCallData.hasTools()) {
|
|
||||||
request.setTools(this.createTools(toolCallData.tools()));
|
|
||||||
request.setToolChoice(this.createToolChoice(toolCallData));
|
|
||||||
request.setParallelToolCalls(toolCallData.parallelToolCalls());
|
|
||||||
}
|
|
||||||
|
|
||||||
String requestBody;
|
String requestBody;
|
||||||
try {
|
try {
|
||||||
@@ -203,6 +165,53 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
|||||||
return new Completion(content, reasoning, finishReason, statistics, toolCalls);
|
return new Completion(content, reasoning, finishReason, statistics, toolCalls);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ChatCompletionRequest createRequest(final List<ChatMessage> chat, final ModelProfile modelProfile, final boolean streaming, final ToolCallData toolCallData) {
|
||||||
|
final var request = new ChatCompletionRequest();
|
||||||
|
request.setModel(this.getPrimaryModelName(modelProfile));
|
||||||
|
request.setMessages(chat);
|
||||||
|
this.applyChatTokenLimitParameterMode(request, modelProfile);
|
||||||
|
request.setTemperature(modelProfile.temperature());
|
||||||
|
request.setTopP(modelProfile.topP());
|
||||||
|
request.setStream(streaming);
|
||||||
|
if (streaming) {
|
||||||
|
this.configureStreamOptions(true, request::setStreamOptions);
|
||||||
|
}
|
||||||
|
else if (this.sendUsage(modelProfile)) {
|
||||||
|
request.setUsage(true);
|
||||||
|
}
|
||||||
|
final var reasoningEffort = this.reasoningEffort(modelProfile);
|
||||||
|
if (this.sendReasoning(modelProfile) && reasoningEffort != null) {
|
||||||
|
request.setReasoningEffort(reasoningEffort);
|
||||||
|
}
|
||||||
|
if (toolCallData != null && toolCallData.hasTools()) {
|
||||||
|
request.setTools(this.createTools(toolCallData.tools()));
|
||||||
|
request.setToolChoice(this.createToolChoice(toolCallData));
|
||||||
|
request.setParallelToolCalls(toolCallData.parallelToolCalls());
|
||||||
|
}
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void applyChatTokenLimitParameterMode(final ChatCompletionRequest request, final ModelProfile modelProfile) {
|
||||||
|
final var mode = this.chatTokenLimitParameterMode(modelProfile);
|
||||||
|
if (mode == null) {
|
||||||
|
throw new IllegalArgumentException("Chat token limit parameter mode must not be null");
|
||||||
|
}
|
||||||
|
switch (mode) {
|
||||||
|
case MAX_COMPLETION_TOKENS:
|
||||||
|
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
||||||
|
break;
|
||||||
|
case MAX_TOKENS:
|
||||||
|
request.setMaxTokens(modelProfile.maxTokens());
|
||||||
|
break;
|
||||||
|
case BOTH:
|
||||||
|
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
||||||
|
request.setMaxTokens(modelProfile.maxTokens());
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("Unsupported chat token limit parameter mode: " + mode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private List<ChatCompletionRequest.ChatCompletionTool> createTools(final List<ToolDefinition> toolDefinitions) {
|
private List<ChatCompletionRequest.ChatCompletionTool> createTools(final List<ToolDefinition> toolDefinitions) {
|
||||||
return toolDefinitions.stream().map(toolDefinition -> {
|
return toolDefinitions.stream().map(toolDefinition -> {
|
||||||
final var function = new ChatCompletionRequest.ChatCompletionFunction();
|
final var function = new ChatCompletionRequest.ChatCompletionFunction();
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import java.util.Map;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.annotation.*;
|
import com.fasterxml.jackson.annotation.*;
|
||||||
|
|
||||||
|
import de.extio.lmlib.client.oai.OpenAiProviderDialect.ReasoningEffort;
|
||||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;
|
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;
|
||||||
|
|
||||||
final class ChatCompletionRequest extends AbstractCompletionRequest {
|
final class ChatCompletionRequest extends AbstractCompletionRequest {
|
||||||
@@ -13,7 +14,7 @@ final class ChatCompletionRequest extends AbstractCompletionRequest {
|
|||||||
|
|
||||||
private Integer maxCompletionTokens;
|
private Integer maxCompletionTokens;
|
||||||
|
|
||||||
private ChatCompletionsRequestReasoning reasoning;
|
private ReasoningEffort reasoningEffort;
|
||||||
|
|
||||||
private List<ChatCompletionTool> tools;
|
private List<ChatCompletionTool> tools;
|
||||||
|
|
||||||
@@ -39,13 +40,13 @@ final class ChatCompletionRequest extends AbstractCompletionRequest {
|
|||||||
this.maxCompletionTokens = maxCompletionTokens;
|
this.maxCompletionTokens = maxCompletionTokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty("reasoning")
|
@JsonProperty("reasoning_effort")
|
||||||
public ChatCompletionsRequestReasoning getReasoning() {
|
public ReasoningEffort getReasoningEffort() {
|
||||||
return this.reasoning;
|
return this.reasoningEffort;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setReasoning(final ChatCompletionsRequestReasoning reasoning) {
|
public void setReasoningEffort(final ReasoningEffort reasoningEffort) {
|
||||||
this.reasoning = reasoning;
|
this.reasoningEffort = reasoningEffort;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty("tools")
|
@JsonProperty("tools")
|
||||||
@@ -75,40 +76,6 @@ final class ChatCompletionRequest extends AbstractCompletionRequest {
|
|||||||
this.parallelToolCalls = parallelToolCalls;
|
this.parallelToolCalls = parallelToolCalls;
|
||||||
}
|
}
|
||||||
|
|
||||||
final static class ChatCompletionsRequestReasoning {
|
|
||||||
|
|
||||||
private ChatCompletionsRequestReasoningEffort effort;
|
|
||||||
|
|
||||||
private String summary;
|
|
||||||
|
|
||||||
@JsonProperty("effort")
|
|
||||||
public ChatCompletionsRequestReasoningEffort getEffort() {
|
|
||||||
return this.effort;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setEffort(final ChatCompletionsRequestReasoningEffort effort) {
|
|
||||||
this.effort = effort;
|
|
||||||
}
|
|
||||||
|
|
||||||
@JsonProperty("summary")
|
|
||||||
public String getSummary() {
|
|
||||||
return this.summary;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSummary(final String summary) {
|
|
||||||
this.summary = summary;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
static enum ChatCompletionsRequestReasoningEffort {
|
|
||||||
minimal, low, medium, high
|
|
||||||
}
|
|
||||||
|
|
||||||
static enum ChatCompletionsRequestReasoningSummaryDetails {
|
|
||||||
auto, concise, detailed
|
|
||||||
}
|
|
||||||
|
|
||||||
static final class ChatCompletionTool {
|
static final class ChatCompletionTool {
|
||||||
|
|
||||||
private String type;
|
private String type;
|
||||||
|
|||||||
@@ -53,19 +53,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy);
|
final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy);
|
||||||
|
final var request = this.createRequest(prompt, modelProfile, chunkConsumer != null);
|
||||||
final var request = new CompletionRequest();
|
|
||||||
request.setModel(this.getPrimaryModelName(modelProfile));
|
|
||||||
request.setPrompt(prompt);
|
|
||||||
request.setMaxTokens(modelProfile.maxTokens());
|
|
||||||
request.setTemperature(modelProfile.temperature());
|
|
||||||
request.setTopP(modelProfile.topP());
|
|
||||||
request.setStream(chunkConsumer != null);
|
|
||||||
if (chunkConsumer != null) {
|
|
||||||
this.configureStreamOptions(true, request::setStreamOptions);
|
|
||||||
} else {
|
|
||||||
request.setUsage(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
String requestBody;
|
String requestBody;
|
||||||
try {
|
try {
|
||||||
@@ -197,6 +185,23 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
|||||||
return new Completion(content, reasoning, finishReason, statistics);
|
return new Completion(content, reasoning, finishReason, statistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
CompletionRequest createRequest(final String prompt, final ModelProfile modelProfile, final boolean streaming) {
|
||||||
|
final var request = new CompletionRequest();
|
||||||
|
request.setModel(this.getPrimaryModelName(modelProfile));
|
||||||
|
request.setPrompt(prompt);
|
||||||
|
request.setMaxTokens(modelProfile.maxTokens());
|
||||||
|
request.setTemperature(modelProfile.temperature());
|
||||||
|
request.setTopP(modelProfile.topP());
|
||||||
|
request.setStream(streaming);
|
||||||
|
if (streaming) {
|
||||||
|
this.configureStreamOptions(true, request::setStreamOptions);
|
||||||
|
}
|
||||||
|
else if (this.sendUsage(modelProfile)) {
|
||||||
|
request.setUsage(true);
|
||||||
|
}
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||||
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
|
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
|
||||||
|
|
||||||
|
|||||||
@@ -18,8 +18,6 @@ public record ModelProfile(
|
|||||||
BigDecimal costPerCachedInToken,
|
BigDecimal costPerCachedInToken,
|
||||||
BigDecimal costPerOutToken,
|
BigDecimal costPerOutToken,
|
||||||
BigDecimal costPerReasoningOutToken,
|
BigDecimal costPerReasoningOutToken,
|
||||||
String reasoningEffort,
|
|
||||||
String reasoningSummaryDetails,
|
|
||||||
String category) {
|
String category) {
|
||||||
|
|
||||||
public ModelProfile {
|
public ModelProfile {
|
||||||
|
|||||||
@@ -90,12 +90,6 @@ public class ModelProfileService {
|
|||||||
.map(Object::toString)
|
.map(Object::toString)
|
||||||
.map(value -> new BigDecimal(value).divide(new BigDecimal(1000000.0)))
|
.map(value -> new BigDecimal(value).divide(new BigDecimal(1000000.0)))
|
||||||
.orElse(BigDecimal.ZERO);
|
.orElse(BigDecimal.ZERO);
|
||||||
final var reasoningEffort = Optional.ofNullable(resource.getProperty("reasoningEffort"))
|
|
||||||
.map(Object::toString)
|
|
||||||
.orElse(null);
|
|
||||||
final var reasoningSummaryDetails = Optional.ofNullable(resource.getProperty("reasoningSummaryDetails"))
|
|
||||||
.map(Object::toString)
|
|
||||||
.orElse(null);
|
|
||||||
final var category = Optional
|
final var category = Optional
|
||||||
.ofNullable(modelCategory)
|
.ofNullable(modelCategory)
|
||||||
.map(ModelCategory::getShortName)
|
.map(ModelCategory::getShortName)
|
||||||
@@ -112,7 +106,7 @@ public class ModelProfileService {
|
|||||||
.orElse(null);
|
.orElse(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category);
|
return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, category);
|
||||||
}
|
}
|
||||||
catch (final IOException e) {
|
catch (final IOException e) {
|
||||||
LOGGER.error("Error while reading model profile", e);
|
LOGGER.error("Error while reading model profile", e);
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package de.extio.lmlib.client.oai.completion.chat;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import de.extio.lmlib.client.oai.OpenAiProviderDialect;
|
||||||
|
import de.extio.lmlib.profile.ModelProfile;
|
||||||
|
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||||
|
|
||||||
|
class ChatCompletionClientRequestTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void defaultsToOfficialRequestFieldsWhenNoDialectIsConfigured() {
|
||||||
|
final var client = new TestChatCompletionClient();
|
||||||
|
final var request = client.createRequest(
|
||||||
|
List.of(new ChatMessage("user", "hello", null)),
|
||||||
|
this.createProfile(),
|
||||||
|
false,
|
||||||
|
null);
|
||||||
|
|
||||||
|
assertNull(request.getUsage());
|
||||||
|
assertEquals(OpenAiProviderDialect.ReasoningEffort.MEDIUM, request.getReasoningEffort());
|
||||||
|
assertEquals(2048, request.getMaxCompletionTokens());
|
||||||
|
assertNull(request.getMaxTokens());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void omitsDialectDisabledReasoningAndUsesLegacyMaxTokens() {
|
||||||
|
final var client = new TestChatCompletionClient();
|
||||||
|
client.setDialect(new OpenAiProviderDialect() {
|
||||||
|
@Override
|
||||||
|
public boolean sendReasoning(final ModelProfile modelProfile) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) {
|
||||||
|
return ChatTokenLimitParameterMode.MAX_TOKENS;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
final var request = client.createRequest(
|
||||||
|
List.of(new ChatMessage("user", "hello", null)),
|
||||||
|
this.createProfile(),
|
||||||
|
false,
|
||||||
|
null);
|
||||||
|
|
||||||
|
assertNull(request.getUsage());
|
||||||
|
assertNull(request.getReasoningEffort());
|
||||||
|
assertNull(request.getMaxCompletionTokens());
|
||||||
|
assertEquals(2048, request.getMaxTokens());
|
||||||
|
assertFalse(request.isStream());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void usesDialectProvidedReasoningEffort() {
|
||||||
|
final var client = new TestChatCompletionClient();
|
||||||
|
client.setDialect(new OpenAiProviderDialect() {
|
||||||
|
@Override
|
||||||
|
public ReasoningEffort reasoningEffort(final ModelProfile modelProfile) {
|
||||||
|
return ReasoningEffort.HIGH;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
final var request = client.createRequest(
|
||||||
|
List.of(new ChatMessage("user", "hello", null)),
|
||||||
|
this.createProfile(),
|
||||||
|
false,
|
||||||
|
null);
|
||||||
|
|
||||||
|
assertEquals(OpenAiProviderDialect.ReasoningEffort.HIGH, request.getReasoningEffort());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void canEnableUsageAndSendBothTokenLimitFields() {
|
||||||
|
final var client = new TestChatCompletionClient();
|
||||||
|
client.setDialect(new OpenAiProviderDialect() {
|
||||||
|
@Override
|
||||||
|
public boolean sendUsage(final ModelProfile modelProfile) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ChatTokenLimitParameterMode chatTokenLimitParameterMode(final ModelProfile modelProfile) {
|
||||||
|
return ChatTokenLimitParameterMode.BOTH;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
final var request = client.createRequest(
|
||||||
|
List.of(new ChatMessage("user", "hello", null)),
|
||||||
|
this.createProfile(),
|
||||||
|
false,
|
||||||
|
null);
|
||||||
|
|
||||||
|
assertTrue(request.getUsage());
|
||||||
|
assertEquals(2048, request.getMaxCompletionTokens());
|
||||||
|
assertEquals(2048, request.getMaxTokens());
|
||||||
|
}
|
||||||
|
|
||||||
|
private ModelProfile createProfile() {
|
||||||
|
return new ModelProfile(
|
||||||
|
"",
|
||||||
|
"fallback",
|
||||||
|
"cl100k_base",
|
||||||
|
2048,
|
||||||
|
8192,
|
||||||
|
0.4,
|
||||||
|
1.0,
|
||||||
|
ModelProvider.OAI_CHAT_COMPLETION,
|
||||||
|
"test-model",
|
||||||
|
"http://localhost:5001",
|
||||||
|
"",
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
"TEST");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class TestChatCompletionClient extends ChatCompletionClient {
|
||||||
|
|
||||||
|
private void setDialect(final OpenAiProviderDialect openAiProviderDialect) {
|
||||||
|
this.openAiProviderDialect = openAiProviderDialect;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package de.extio.lmlib.client.oai.completion.text;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import de.extio.lmlib.client.oai.OpenAiProviderDialect;
|
||||||
|
import de.extio.lmlib.profile.ModelProfile;
|
||||||
|
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||||
|
|
||||||
|
class TextCompletionClientRequestTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void omitsUsageWhenNoDialectIsConfigured() {
|
||||||
|
final var client = new TestTextCompletionClient();
|
||||||
|
final var request = client.createRequest("prompt", this.createProfile(), false);
|
||||||
|
|
||||||
|
assertNull(request.getUsage());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void includesUsageWhenDialectEnablesIt() {
|
||||||
|
final var client = new TestTextCompletionClient();
|
||||||
|
client.setDialect(new OpenAiProviderDialect() {
|
||||||
|
@Override
|
||||||
|
public boolean sendUsage(final ModelProfile modelProfile) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
final var request = client.createRequest("prompt", this.createProfile(), false);
|
||||||
|
|
||||||
|
assertTrue(request.getUsage());
|
||||||
|
}
|
||||||
|
|
||||||
|
private ModelProfile createProfile() {
|
||||||
|
return new ModelProfile(
|
||||||
|
"gpt-oss",
|
||||||
|
"fallback",
|
||||||
|
"cl100k_base",
|
||||||
|
2048,
|
||||||
|
8192,
|
||||||
|
0.4,
|
||||||
|
1.0,
|
||||||
|
ModelProvider.OAI_TEXT_COMPLETION,
|
||||||
|
"test-model",
|
||||||
|
"http://localhost:5001",
|
||||||
|
"",
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
BigDecimal.ZERO,
|
||||||
|
"TEST");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class TestTextCompletionClient extends TextCompletionClient {
|
||||||
|
|
||||||
|
private void setDialect(final OpenAiProviderDialect openAiProviderDialect) {
|
||||||
|
this.openAiProviderDialect = openAiProviderDialect;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -55,8 +55,6 @@ public class TokenizerResolverTest {
|
|||||||
BigDecimal.ZERO,
|
BigDecimal.ZERO,
|
||||||
BigDecimal.ZERO,
|
BigDecimal.ZERO,
|
||||||
BigDecimal.ZERO,
|
BigDecimal.ZERO,
|
||||||
null,
|
|
||||||
null,
|
|
||||||
"TEST");
|
"TEST");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,5 +14,3 @@ cost1MInTokens=0
|
|||||||
cost1MCachedInTokens=0
|
cost1MCachedInTokens=0
|
||||||
cost1MOutTokens=0
|
cost1MOutTokens=0
|
||||||
cost1MReasoningOutTokens=0
|
cost1MReasoningOutTokens=0
|
||||||
reasoningEffort=medium
|
|
||||||
reasoningSummaryDetails=auto
|
|
||||||
Reference in New Issue
Block a user