exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Introduce reasoning capabilities in chat and text completion

Browse Source 
- Added AbstractCompletionRequest and AbstractCompletionResponse classes to standardize request and response structures.
- Implemented StreamOptions for streaming responses in chat completions.
- Enhanced ChatCompletionClient and TextCompletionClient to support reasoning content in responses.
- Updated ModelProfile to include reasoning effort and summary details.
- Created GptOssPromptStrategy to handle specific prompt formatting and reasoning extraction.
- Modified existing tests to validate reasoning content in completions.
- Updated configuration properties for reasoning settings in local properties files.
This commit is contained in:
extio-de
2025-10-01 19:05:15 +02:00
parent 34c96f0147
commit 514dad1c7f
32 changed files with 582 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@@ -4,7 +4,7 @@
<artifactId>lm-lib</artifactId>
<groupId>de.extio</groupId>
<name>lm-lib</name>
<version>1.6.24</version>
<version>1.7.0</version>
<properties>
<java.version>21</java.version>

40
src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java
View File

@@ -10,10 +10,11 @@ import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.json.JsonReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import de.extio.lmlib.agent.AgentContext;
import de.extio.lmlib.client.Completion;
@@ -34,24 +35,25 @@ public class JsonAgentResponseHandler implements AgentResponseHandler {
public JsonAgentResponseHandler(final String prefixFields) {
this.prefixFields = prefixFields == null ? "" : prefixFields;
this.objectMapper = new ObjectMapper();
this.objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false);
this.objectMapper.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_MISSING_VALUES, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_COMMA, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_YAML_COMMENTS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
this.objectMapper = JsonMapper.builder()
.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false)
.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false)
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false)
.enable(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER)
.enable(JsonReadFeature.ALLOW_JAVA_COMMENTS)
.enable(JsonReadFeature.ALLOW_MISSING_VALUES)
.enable(JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS)
.enable(JsonReadFeature.ALLOW_SINGLE_QUOTES)
.enable(JsonReadFeature.ALLOW_TRAILING_COMMA)
.enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS)
.enable(JsonReadFeature.ALLOW_YAML_COMMENTS)
.enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES)
.build();
}
@Override

3
src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java
View File

@@ -1,11 +1,12 @@
package de.extio.lmlib.agent.responsehandler;
import de.extio.lmlib.agent.AgentContext;
import de.extio.lmlib.client.Chunk;
public interface StreamedAgentResponseHandler extends AgentResponseHandler {
void beforeStream(AgentContext context);
boolean handleChunk(String chunk, AgentContext context);
boolean handleChunk(Chunk chunk, AgentContext context);
}

46
src/main/java/de/extio/lmlib/agent/responsehandler/TextAgentResponseHandler.java
View File

@@ -1,6 +1,7 @@
package de.extio.lmlib.agent.responsehandler;
import java.util.Objects;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -8,33 +9,32 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.extio.lmlib.agent.AgentContext;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Completion;
public class TextAgentResponseHandler implements StreamedAgentResponseHandler {
public static final String UPDATE_KEY = "chunkUpdateKey";
private static final Logger LOGGER = LoggerFactory.getLogger(TextAgentResponseHandler.class);
private final String key;
private final String chunkKey;
private final String reasoningKey;
private final Function<String, String> transformer;
private final Consumer<AgentContext> beforeStream;
private final Function<String, String> chunkTransformer;
private final Consumer<AgentContext> afterChunkUpdate;
private final Function<Chunk, Chunk> chunkTransformer;
private final BiConsumer<AgentContext, Chunk> afterChunkUpdate;
public TextAgentResponseHandler(final String key) {
this(key, null, null, null, null);
}
public TextAgentResponseHandler(final String key, final Function<String, String> transformer, final Consumer<AgentContext> beforeStream, final Function<String, String> chunkTransformer, final Consumer<AgentContext> afterChunkUpdate) {
public TextAgentResponseHandler(final String key, final Function<String, String> transformer, final Consumer<AgentContext> beforeStream, final Function<Chunk, Chunk> chunkTransformer, final BiConsumer<AgentContext, Chunk> afterChunkUpdate) {
this.key = key;
this.chunkKey = key.concat("_chunk");
this.reasoningKey = key.concat("_reasoning");
this.transformer = transformer;
this.beforeStream = beforeStream;
this.chunkTransformer = chunkTransformer;
@@ -44,7 +44,14 @@ public class TextAgentResponseHandler implements StreamedAgentResponseHandler {
@Override
public boolean handle(final Completion completion, final AgentContext context) {
var response = completion.response();
LOGGER.debug("Response: {}", response);
final var reasoning = completion.reasoning();
if (reasoning != null && !reasoning.isBlank()) {
context.setStringValue(this.reasoningKey, reasoning);
LOGGER.debug("Reasoning: {}; Response: {}", reasoning, response);
}
else {
LOGGER.debug("Response: {}", response);
}
if (this.transformer != null) {
response = this.transformer.apply(response);
}
@@ -55,24 +62,27 @@ public class TextAgentResponseHandler implements StreamedAgentResponseHandler {
@Override
public void beforeStream(final AgentContext context) {
context.setStringValue(this.key, "");
context.getContext().remove(this.reasoningKey);
if (this.beforeStream != null) {
this.beforeStream.accept(context);
}
}
@Override
public boolean handleChunk(final String chunk_, final AgentContext context) {
public boolean handleChunk(final Chunk chunk_, final AgentContext context) {
var chunk = chunk_;
if (this.chunkTransformer != null) {
chunk = this.chunkTransformer.apply(chunk);
}
context.setStringValue(UPDATE_KEY, this.key);
if (afterChunkUpdate != null) {
context.setStringValue(this.chunkKey, chunk);
context.setStringValue(this.key, Objects.requireNonNullElse(context.getStringValue(this.key), "").concat(chunk));
afterChunkUpdate.accept(context);
context.getContext().remove(UPDATE_KEY);
context.getContext().remove(this.chunkKey);
if (chunk.content() != null) {
context.setStringValue(this.key, Objects.requireNonNullElseGet(context.getStringValue(this.key), () -> "").concat(chunk.content()));
}
if (chunk.reasoningContent() != null) {
context.setStringValue(this.reasoningKey, Objects.requireNonNullElseGet(context.getStringValue(this.reasoningKey), () -> "").concat(chunk.reasoningContent()));
}
afterChunkUpdate.accept(context, chunk);
}
return true;
}

5
src/main/java/de/extio/lmlib/client/Chunk.java Normal file
View File

@@ -0,0 +1,5 @@
package de.extio.lmlib.client;
public record Chunk(String content, String reasoningContent) {
}

2
src/main/java/de/extio/lmlib/client/Client.java
View File

@@ -15,7 +15,7 @@ public interface Client {
Completion conversation(ModelCategory modelCategory, Conversation conversation);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<String> chunkConsumer);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer);
ModelProvider getModelProvider();
}

2
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -69,7 +69,7 @@ public class ClientService {
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<String> chunkConsumer) {
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelCategory, conversation);

2
src/main/java/de/extio/lmlib/client/Completion.java
View File

@@ -1,5 +1,5 @@
package de.extio.lmlib.client;
public final record Completion(String response, CompletionFinishReason finishReason, CompletionStatistics statistics) {
public final record Completion(String response, String reasoning, CompletionFinishReason finishReason, CompletionStatistics statistics) {
}

6
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -31,6 +31,8 @@ import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
import com.azure.core.http.policy.ExponentialBackoffOptions;
import com.azure.core.http.policy.RetryOptions;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
@@ -91,7 +93,7 @@ public final class AzureAiClient implements Client {
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<String> chunkConsumer) {
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
throw new UnsupportedOperationException("Not implemented yet");
}
@@ -199,7 +201,7 @@ public final class AzureAiClient implements Client {
false);
LOGGER.debug("{}", statistics);
return new Completion(response.getChoice().getMessage().getContent(), finishReason, statistics);
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
}
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {

9
src/main/java/de/extio/lmlib/client/cached/CachedClient.java
View File

@@ -10,6 +10,7 @@ import java.util.Objects;
import java.util.function.Consumer;
import java.util.function.Supplier;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
@@ -44,10 +45,10 @@ public class CachedClient implements Client {
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<String> chunkConsumer) {
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(modelCategory, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(response.response());
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
return response;
}
@@ -62,12 +63,12 @@ public class CachedClient implements Client {
final var cachedCompletion = this.cacheRepository.get(key);
if (cachedCompletion != null) {
return new Completion(cachedCompletion.response(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true));
return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true));
}
final var completion = supplier.get();
if (completion != null) {
this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now()));
this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.reasoning(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now()));
}
return completion;
}

2
src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java
View File

@@ -4,6 +4,6 @@ import java.time.OffsetDateTime;
import de.extio.lmlib.client.CompletionFinishReason;
public record CachedCompletion(String response, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) {
public record CachedCompletion(String response, String reasoning, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) {
}

80
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
View File

@@ -15,12 +15,17 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.reactive.function.client.WebClient;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.json.JsonReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.oai.ModelNameSupplier;
import de.extio.lmlib.profile.ModelCategory;
@@ -28,6 +33,7 @@ import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
import reactor.util.retry.Retry;
public abstract class AbstractCompletionClient implements Client, DisposableBean {
@@ -52,28 +58,30 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
protected final CompletionStatistics totalStatistics = new CompletionStatistics();
protected final ObjectMapper objectMapper;
public AbstractCompletionClient() {
this.objectMapper = new ObjectMapper();
this.objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false);
this.objectMapper.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_MISSING_VALUES, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_COMMA, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_YAML_COMMENTS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS, true);
this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
this.objectMapper = JsonMapper.builder()
.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false)
.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false)
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false)
.serializationInclusion(JsonInclude.Include.NON_NULL)
.enable(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER)
.enable(JsonReadFeature.ALLOW_JAVA_COMMENTS)
.enable(JsonReadFeature.ALLOW_MISSING_VALUES)
.enable(JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS)
.enable(JsonReadFeature.ALLOW_SINGLE_QUOTES)
.enable(JsonReadFeature.ALLOW_TRAILING_COMMA)
.enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS)
.enable(JsonReadFeature.ALLOW_YAML_COMMENTS)
.enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES)
.build();
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.conversation(modelCategory, Conversation.create(system, text));
@@ -85,7 +93,7 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
}
@Override
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<String> chunkConsumer) {
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
@@ -95,7 +103,33 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
return this.requestCompletion(conversation, modelCategory, modelProfile, chunkConsumer);
}
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<String> chunkConsumer);
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer);
protected CompletionFinishReason mapFinishReason(final String finishReason) {
return switch (finishReason) {
case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
}
protected Retry createRetrySpec() {
return Retry.backoff(99, Duration.ofSeconds(1))
.maxBackoff(Duration.ofSeconds(10))
.jitter(0.25d)
.filter(throwable -> {
return throwable instanceof java.io.IOException || throwable instanceof java.net.ConnectException || throwable instanceof java.net.http.HttpTimeoutException;
})
.doAfterRetry(rs -> LOGGER.warn("Retrying failed request: " + rs.failure().getClass().getName() + " " + rs.failure().getMessage()));
}
protected void configureStreamOptions(final boolean enableStreaming, final Consumer<StreamOptions> streamOptionsConsumer) {
if (enableStreaming) {
final var streamOptions = new StreamOptions();
streamOptions.setIncludeUsage(true);
streamOptionsConsumer.accept(streamOptions);
}
}
protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) {
int inTokens = 0;

72
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java Normal file
View File

@@ -0,0 +1,72 @@
package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
public abstract class AbstractCompletionRequest {
protected String model;
protected Integer maxTokens;
protected Double temperature;
protected Double topP;
protected Boolean stream;
protected StreamOptions streamOptions;
@JsonProperty("model")
public String getModel() {
return this.model;
}
public void setModel(final String model) {
this.model = model;
}
@JsonProperty("max_tokens")
public Integer getMaxTokens() {
return this.maxTokens;
}
public void setMaxTokens(final Integer maxTokens) {
this.maxTokens = maxTokens;
}
@JsonProperty("temperature")
public Double getTemperature() {
return this.temperature;
}
public void setTemperature(final Double temperature) {
this.temperature = temperature;
}
@JsonProperty("top_p")
public Double getTopP() {
return this.topP;
}
public void setTopP(final Double topP) {
this.topP = topP;
}
@JsonProperty("stream")
public Boolean isStream() {
return this.stream;
}
public void setStream(final Boolean stream) {
this.stream = stream;
}
@JsonProperty("stream_options")
public StreamOptions getStreamOptions() {
return streamOptions;
}
public void setStreamOptions(final StreamOptions streamOptions) {
this.streamOptions = streamOptions;
}
}

50
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java Normal file
View File

@@ -0,0 +1,50 @@
package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
public abstract class AbstractCompletionResponse {
protected String id;
protected String object;
protected Integer created;
protected Usage usage;
@JsonProperty("id")
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("created")
public Integer getCreated() {
return this.created;
}
public void setCreated(final Integer created) {
this.created = created;
}
@JsonProperty("usage")
public Usage getUsage() {
return usage;
}
public void setUsage(final Usage usage) {
this.usage = usage;
}
}

6
src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionStreamOptions.java → src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java
View File

@@ -1,10 +1,10 @@
package de.extio.lmlib.client.oai.completion.text;
package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
public class CompletionStreamOptions {
public class StreamOptions {
private boolean includeUsage;
protected boolean includeUsage;
@JsonProperty("include_usage")
public boolean isIncludeUsage() {

90
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java
View File

@@ -1,6 +1,5 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -16,17 +15,15 @@ import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.client.oai.completion.FinishReasons;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import reactor.core.publisher.Mono;
import reactor.util.retry.Retry;
@Component
public class ChatCompletionClient extends AbstractCompletionClient {
@@ -43,7 +40,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<String> chunkConsumer) {
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
final var chat = createChats(conversation, modelProfile);
final var request = new ChatCompletionRequest();
@@ -53,13 +50,37 @@ public class ChatCompletionClient extends AbstractCompletionClient {
request.setMaxCompletionTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
if (chunkConsumer == null) {
request.setStream(false);
request.setStream(chunkConsumer != null);
if (chunkConsumer != null) {
this.configureStreamOptions(true, request::setStreamOptions);
}
else {
request.setStream(true);
request.setStreamOptions(new ChatCompletionStreamOptions());
request.getStreamOptions().setIncludeUsage(true);
if (modelProfile.reasoningEffort() != null && !modelProfile.reasoningEffort().isBlank()) {
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
try {
request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(modelProfile.reasoningEffort()));
}
catch (final IllegalArgumentException e) {
throw new IllegalArgumentException("Invalid reasoning effort in the model profile: " + modelProfile.reasoningEffort(), e);
}
}
if (modelProfile.reasoningSummaryDetails() != null && !modelProfile.reasoningSummaryDetails().isBlank()) {
if (request.getReasoning() == null) {
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
}
try {
request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(modelProfile.reasoningSummaryDetails()).name());
}
catch (final IllegalArgumentException e) {
throw new IllegalArgumentException("Invalid reasoning summary details in the model profile: " + modelProfile.reasoningSummaryDetails(), e);
}
}
String requestBody;
try {
requestBody = this.objectMapper.writeValueAsString(request);
}
catch (final JsonProcessingException e) {
throw new IllegalStateException("Cannot serialize request body", e);
}
LOGGER.debug("Requesting chat completion at {}", modelProfile.url());
@@ -70,19 +91,25 @@ public class ChatCompletionClient extends AbstractCompletionClient {
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.bodyValue(requestBody)
.exchangeToMono(clientResponse -> {
if (clientResponse.statusCode().isError()) {
return clientResponse.bodyToMono(String.class)
.flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody)));
}
if (chunkConsumer == null) {
return clientResponse.bodyToMono(ChatCompletionResponse.class);
}
else {
final var chatCompletionResponse = new ChatCompletionResponse();
chatCompletionResponse.setChoices(List.of(new ChatChoice()));
final StringBuilder contenStringBuilder = new StringBuilder();
final StringBuilder contentStringBuilder = new StringBuilder();
final StringBuilder reasoningStringBuilder = new StringBuilder();
return clientResponse.bodyToFlux(String.class)
.doOnNext(buffer -> {
try {
if (! "[DONE]".equals(buffer)) {
if (!"[DONE]".equals(buffer)) {
final var streamResponse = this.objectMapper.readValue(buffer, ChatCompletionResponse.class);
if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) {
@@ -101,9 +128,20 @@ public class ChatCompletionClient extends AbstractCompletionClient {
if (streamResponse.getChoices().getFirst().getFinishReason() != null) {
chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason());
}
if (streamResponse.getChoices().getFirst().getMessage() != null && streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) {
contenStringBuilder.append(streamResponse.getChoices().getFirst().getMessage().getContent());
chunkConsumer.accept(streamResponse.getChoices().getFirst().getMessage().getContent());
if (streamResponse.getChoices().getFirst().getMessage() != null) {
String content = null;
String reasoning = null;
if (streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) {
content = streamResponse.getChoices().getFirst().getMessage().getContent();
contentStringBuilder.append(content);
}
if (streamResponse.getChoices().getFirst().getMessage().getReasoningContent() != null && !streamResponse.getChoices().getFirst().getMessage().getReasoningContent().isEmpty()) {
reasoning = streamResponse.getChoices().getFirst().getMessage().getReasoningContent();
reasoningStringBuilder.append(reasoning);
}
if (content != null || reasoning != null) {
chunkConsumer.accept(new Chunk(content, reasoning));
}
}
}
}
@@ -113,27 +151,23 @@ public class ChatCompletionClient extends AbstractCompletionClient {
}
})
.then(Mono.<ChatCompletionResponse> fromSupplier(() -> {
chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contenStringBuilder.toString()));
chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contentStringBuilder.toString(), reasoningStringBuilder.toString()));
return chatCompletionResponse;
}));
}
})
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.retryWhen(this.createRetrySpec())
.block();
if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) {
throw new IllegalStateException("No parseable response content from server");
}
final var choice = response.getChoices().getFirst();
final var content = response.getChoices().getFirst().getMessage().getContent();
final var finishReason = switch (choice.getFinishReason()) {
case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
final var reasoning = response.getChoices().getFirst().getMessage().getReasoningContent();
final var finishReason = this.mapFinishReason(choice.getFinishReason());
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content);
return new Completion(content, finishReason, statistics);
return new Completion(content, reasoning, finishReason, statistics);
}
private List<ChatMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
@@ -165,7 +199,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
if (first && userTurn != null) {
if (systemTurn != null) {
chat.add(new ChatMessage("system", systemTurn.text()));
chat.add(new ChatMessage("system", systemTurn.text(), null));
}
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
break;
@@ -202,7 +236,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
trimmed = true;
}
chat.add(new ChatMessage(role, prompt));
chat.add(new ChatMessage(role, prompt, null));
return trimmed;
}

107
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java
View File

@@ -2,36 +2,17 @@ package de.extio.lmlib.client.oai.completion.chat;
import java.util.List;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
final class ChatCompletionRequest {
private String model;
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;
final class ChatCompletionRequest extends AbstractCompletionRequest {
private List<ChatMessage> messages;
private Integer maxTokens;
private Integer maxCompletionTokens;
private Double temperature;
private Double topP;
private Boolean stream;
private ChatCompletionStreamOptions streamOptions;
@JsonProperty("model")
public String getModel() {
return this.model;
}
public void setModel(final String model) {
this.model = model;
}
private ChatCompletionsRequestReasoning reasoning;
@JsonProperty("messages")
public List<ChatMessage> getMessages() {
@@ -42,15 +23,6 @@ final class ChatCompletionRequest {
this.messages = messages;
}
@JsonProperty("max_tokens")
public Integer getMaxTokens() {
return this.maxTokens;
}
public void setMaxTokens(final Integer maxTokens) {
this.maxTokens = maxTokens;
}
@JsonProperty("max_completion_tokens")
public Integer getMaxCompletionTokens() {
return this.maxCompletionTokens;
@@ -60,40 +32,47 @@ final class ChatCompletionRequest {
this.maxCompletionTokens = maxCompletionTokens;
}
@JsonProperty("temperature")
public Double getTemperature() {
return this.temperature;
@JsonProperty("reasoning")
public ChatCompletionsRequestReasoning getReasoning() {
return this.reasoning;
}
public void setTemperature(final Double temperature) {
this.temperature = temperature;
public void setReasoning(final ChatCompletionsRequestReasoning reasoning) {
this.reasoning = reasoning;
}
@JsonProperty("top_p")
public Double getTopP() {
return this.topP;
final static class ChatCompletionsRequestReasoning {
private ChatCompletionsRequestReasoningEffort effort;
private String summary;
@JsonProperty("effort")
public ChatCompletionsRequestReasoningEffort getEffort() {
return this.effort;
}
public void setEffort(final ChatCompletionsRequestReasoningEffort effort) {
this.effort = effort;
}
@JsonProperty("summary")
public String getSummary() {
return this.summary;
}
public void setSummary(final String summary) {
this.summary = summary;
}
}
public void setTopP(final Double topP) {
this.topP = topP;
static enum ChatCompletionsRequestReasoningEffort {
minimal, low, medium, high
}
@JsonProperty("stream")
public Boolean isStream() {
return this.stream;
static enum ChatCompletionsRequestReasoningSummaryDetails {
auto, concise, detailed
}
public void setStream(final Boolean stream) {
this.stream = stream;
}
@JsonProperty("stream_options")
public ChatCompletionStreamOptions getStreamOptions() {
return streamOptions;
}
public void setStreamOptions(ChatCompletionStreamOptions streamOptions) {
this.streamOptions = streamOptions;
}
}

48
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java
View File

@@ -4,47 +4,12 @@ import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.extio.lmlib.client.oai.completion.Usage;
import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse;
final class ChatCompletionResponse {
private String id;
private String object;
private Integer created;
final class ChatCompletionResponse extends AbstractCompletionResponse {
private List<ChatChoice> choices;
private Usage usage;
@JsonProperty("id")
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("created")
public Integer getCreated() {
return this.created;
}
public void setCreated(final Integer created) {
this.created = created;
}
@JsonProperty("choices")
public List<ChatChoice> getChoices() {
return this.choices;
@@ -53,15 +18,6 @@ final class ChatCompletionResponse {
public void setChoices(final List<ChatChoice> choices) {
this.choices = choices;
}
@JsonProperty("usage")
public Usage getUsage() {
return usage;
}
public void setUsage(final Usage usage) {
this.usage = usage;
}
@Override
public String toString() {

17
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionStreamOptions.java
View File

@@ -1,17 +0,0 @@
package de.extio.lmlib.client.oai.completion.chat;
import com.fasterxml.jackson.annotation.JsonProperty;
public class ChatCompletionStreamOptions {
private boolean includeUsage;
@JsonProperty("include_usage")
public boolean isIncludeUsage() {
return includeUsage;
}
public void setIncludeUsage(final boolean includeUsage) {
this.includeUsage = includeUsage;
}
}

15
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java
View File

@@ -8,14 +8,17 @@ final class ChatMessage {
private String content;
private String reasoningContent;
public ChatMessage() {
}
public ChatMessage(final String role, final String content) {
public ChatMessage(final String role, final String content, final String reasoningContent) {
this.role = role;
this.content = content;
this.reasoningContent = reasoningContent;
}
@JsonProperty("role")
public String getRole() {
return role;
@@ -34,4 +37,12 @@ final class ChatMessage {
this.content = content;
}
@JsonProperty("reasoning_content")
public String getReasoningContent() {
return reasoningContent;
}
public void setReasoningContent(final String reasoningContent) {
this.reasoningContent = reasoningContent;
}
}

70
src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java
View File

@@ -2,31 +2,12 @@ package de.extio.lmlib.client.oai.completion.text;
import com.fasterxml.jackson.annotation.JsonProperty;
final class CompletionRequest {
private String model;
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;
final class CompletionRequest extends AbstractCompletionRequest {
private String prompt;
private Integer maxTokens;
private Double temperature;
private Double topP;
private Boolean stream;
private CompletionStreamOptions streamOptions;
@JsonProperty("model")
public String getModel() {
return this.model;
}
public void setModel(final String model) {
this.model = model;
}
@JsonProperty("prompt")
public String getPrompt() {
return this.prompt;
@@ -35,49 +16,4 @@ final class CompletionRequest {
public void setPrompt(final String prompt) {
this.prompt = prompt;
}
@JsonProperty("max_tokens")
public Integer getMaxTokens() {
return this.maxTokens;
}
public void setMaxTokens(final Integer maxTokens) {
this.maxTokens = maxTokens;
}
@JsonProperty("temperature")
public Double getTemperature() {
return this.temperature;
}
public void setTemperature(final Double temperature) {
this.temperature = temperature;
}
@JsonProperty("top_p")
public Double getTopP() {
return this.topP;
}
public void setTopP(final Double topP) {
this.topP = topP;
}
@JsonProperty("stream")
public Boolean isStream() {
return this.stream;
}
public void setStream(final Boolean stream) {
this.stream = stream;
}
@JsonProperty("stream_options")
public CompletionStreamOptions getStreamOptions() {
return streamOptions;
}
public void setStreamOptions(final CompletionStreamOptions streamOptions) {
this.streamOptions = streamOptions;
}
}

48
src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java
View File

@@ -4,50 +4,15 @@ import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.extio.lmlib.client.oai.completion.Usage;
import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse;
final class CompletionResponse {
private String id;
private String object;
private Integer created;
final class CompletionResponse extends AbstractCompletionResponse {
private List<Choice> choices;
private String content;
private Boolean isStoppedEos;
private Usage usage;
@JsonProperty("id")
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("created")
public Integer getCreated() {
return this.created;
}
public void setCreated(final Integer created) {
this.created = created;
}
@JsonProperty("choices")
public List<Choice> getChoices() {
@@ -75,15 +40,6 @@ final class CompletionResponse {
public void setStoppedEos(final Boolean isStoppedEos) {
this.isStoppedEos = isStoppedEos;
}
@JsonProperty("usage")
public Usage getUsage() {
return usage;
}
public void setUsage(final Usage usage) {
this.usage = usage;
}
@Override
public String toString() {

53
src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java
View File

@@ -1,6 +1,5 @@
package de.extio.lmlib.client.oai.completion.text;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.Deque;
@@ -13,22 +12,20 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import com.azure.ai.inference.models.ChatChoice;
import com.fasterxml.jackson.core.JsonProcessingException;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.client.oai.completion.FinishReasons;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import reactor.core.publisher.Mono;
import reactor.util.retry.Retry;
@Component
public class TextCompletionClient extends AbstractCompletionClient {
@@ -44,7 +41,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<String> chunkConsumer) {
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
if (promptStrategy == null) {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
@@ -58,13 +55,17 @@ public class TextCompletionClient extends AbstractCompletionClient {
request.setMaxTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
if (chunkConsumer == null) {
request.setStream(false);
request.setStream(chunkConsumer != null);
if (chunkConsumer != null) {
this.configureStreamOptions(true, request::setStreamOptions);
}
else {
request.setStream(true);
request.setStreamOptions(new CompletionStreamOptions());
request.getStreamOptions().setIncludeUsage(true);
String requestBody;
try {
requestBody = this.objectMapper.writeValueAsString(request);
}
catch (final JsonProcessingException e) {
throw new IllegalStateException("Cannot serialize request body", e);
}
LOGGER.debug("Requesting completion at {}", modelProfile.url());
@@ -74,8 +75,13 @@ public class TextCompletionClient extends AbstractCompletionClient {
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.bodyValue(requestBody)
.exchangeToMono(clientResponse -> {
if (clientResponse.statusCode().isError()) {
return clientResponse.bodyToMono(String.class)
.flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody)));
}
if (chunkConsumer == null) {
return clientResponse.bodyToMono(CompletionResponse.class);
}
@@ -116,7 +122,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
}
if (streamResponse.getChoices().getFirst().getText() != null && !streamResponse.getChoices().getFirst().getText().isEmpty()) {
contentStringBuilder.append(streamResponse.getChoices().getFirst().getText());
chunkConsumer.accept(streamResponse.getChoices().getFirst().getText());
chunkConsumer.accept(new Chunk(streamResponse.getChoices().getFirst().getText(), null));
}
}
}
@@ -131,30 +137,33 @@ public class TextCompletionClient extends AbstractCompletionClient {
}));
}
})
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.retryWhen(this.createRetrySpec())
.block();
String content = null;
String reasoning = null;
CompletionFinishReason finishReason = null;
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
final var choice = response.getChoices().getFirst();
if (response.getChoices().getFirst().getText() != null) {
content = promptStrategy.removeEOT(choice.getText());
content = promptStrategy.getResponse(choice.getText());
reasoning = promptStrategy.getReasoning(choice.getText());
}
if (response.getChoices().getFirst().getFinishReason() != null) {
finishReason = switch (choice.getFinishReason()) {
case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
finishReason = this.mapFinishReason(choice.getFinishReason());
}
}
if (content == null && response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
content = promptStrategy.getResponse(response.getContent());
reasoning = promptStrategy.getReasoning(response.getContent());
}
if (content == null) {
throw new IllegalStateException("No parseable response content from server");
}
content = promptStrategy.removeEOT(content);
if (reasoning != null) {
reasoning = promptStrategy.removeEOT(reasoning);
}
if (finishReason == null) {
if (response.isStoppedEos() != null) {
finishReason = Boolean.TRUE.equals(response.isStoppedEos()) ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
@@ -165,7 +174,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
}
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent());
return new Completion(content, finishReason, statistics);
return new Completion(content, reasoning, finishReason, statistics);
}
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {

4
src/main/java/de/extio/lmlib/profile/ModelProfile.java
View File

@@ -14,7 +14,9 @@ public record ModelProfile(
String url,
String apiKey,
BigDecimal costInToken,
BigDecimal costOutToken) {
BigDecimal costOutToken,
String reasoningEffort,
String reasoningSummaryDetails) {
public static enum ModelProvider {
AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION;

10
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -85,8 +85,14 @@ public class ModelProfileService {
.map(value -> new BigDecimal(value))
.orElse(BigDecimal.ZERO)
.divide(new BigDecimal(1000000.0));
final var reasoningEffort = Optional.ofNullable(resource.getProperty("reasoningEffort"))
.map(Object::toString)
.orElse(null);
final var reasoningSummaryDetails = Optional.ofNullable(resource.getProperty("reasoningSummaryDetails"))
.map(Object::toString)
.orElse(null);
//To resolve api key placeholder ${exchangeautodocumentation.azure.apikey} in model-name.properties file
//To resolve api key placeholder ${apikey} in model-name.properties file
String apiKey = null;
synchronized (this.environment) {
if (this.environment instanceof final ConfigurableEnvironment configurableEnvironment) {
@@ -97,7 +103,7 @@ public class ModelProfileService {
.orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
}
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken);
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken, reasoningEffort, reasoningSummaryDetails);
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile", e);

110
src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java Normal file
View File

@@ -0,0 +1,110 @@
package de.extio.lmlib.prompt;
import org.springframework.stereotype.Component;
@Component
public class GptOssPromptStrategy implements PromptStrategy {
private static final String finalMarker = "<|channel|>final<|message|>";
private static final String startMarker = "<|start|>";
private static final String assistantStart = startMarker + "assistant";
private static final String analysisMarker = "<|channel|>analysis<|message|>";
@Override
public StringBuilder start(final String system, final String user) {
final StringBuilder prompt = new StringBuilder();
if (!system.isEmpty()) {
prompt.append("<|start|>system<|message|>");
prompt.append(system);
prompt.append("<|end|>");
}
prompt.append("<|start|>user<|message|>");
prompt.append(user);
prompt.append("<|end|>");
prompt.append(assistantStart);
return prompt;
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
final String assistantContent = assistant == null ? "" : assistant;
if (assistantContent.startsWith(assistantStart)) {
final int assistantStartLength = assistantStart.length();
if (prompt.length() >= assistantStartLength) {
boolean endsWithAssistantStart = true;
for (int i = 0; i < assistantStartLength; i++) {
if (prompt.charAt(prompt.length() - assistantStartLength + i) != assistantStart.charAt(i)) {
endsWithAssistantStart = false;
break;
}
}
if (endsWithAssistantStart) {
prompt.setLength(prompt.length() - assistantStartLength);
}
}
}
if (assistantContent.contains(finalMarker)) {
prompt.append(assistantContent);
if (!assistantContent.endsWith("<|end|>")) {
prompt.append("<|end|>");
}
}
else {
prompt.append("<|channel|>final<|message|>");
prompt.append(assistantContent);
prompt.append("<|end|>");
}
prompt.append("<|start|>user<|message|>");
prompt.append(user == null ? "" : user);
prompt.append("<|end|>");
prompt.append(assistantStart);
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<|end|>", "").replace("<|return|>", "");
}
@Override
public String getPromptName() {
return "gpt-oss";
}
@Override
public String getResponse(final String prompt) {
if (prompt == null) {
return null;
}
final int markerIndex = prompt.indexOf(finalMarker);
if (markerIndex == -1) {
return prompt;
}
final int start = markerIndex + finalMarker.length();
int end = prompt.indexOf(startMarker, start);
if (end == -1) {
end = prompt.length();
}
return prompt.substring(start, end).strip();
}
@Override
public String getReasoning(final String prompt) {
if (prompt == null) {
return null;
}
final int markerIndex = prompt.indexOf(analysisMarker);
if (markerIndex == -1) {
return null;
}
final int start = markerIndex + analysisMarker.length();
int end = prompt.indexOf(startMarker, start);
if (end == -1) {
end = prompt.length();
}
return prompt.substring(start, end).strip();
}
}

5
src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java
View File

@@ -32,9 +32,4 @@ public class NoPromptStrategy implements PromptStrategy {
prompt.append(user);
}
@Override
public String removeEOT(final String prompt) {
return prompt;
}
}

8
src/main/java/de/extio/lmlib/prompt/PromptStrategy.java
View File

@@ -12,4 +12,12 @@ public interface PromptStrategy {
return prompt.strip();
}
default String getResponse(final String prompt) {
return prompt;
}
default String getReasoning(final String prompt) {
return null;
}
}

16
src/test/java/de/extio/lmlib/agent/AgentTest.java
View File

@@ -29,7 +29,7 @@ import de.extio.lmlib.profile.ModelCategory;
import io.netty.util.internal.ThreadLocalRandom;
/**
* These tests have been verified with Llama 3.1 8B and Gemma2 27B models
* These tests have been verified with Llama 3.1 8B, Gemma2 27B, Gemma3 27B and GPT-OSS models
*/
@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@@ -137,11 +137,14 @@ public class AgentTest {
null,
null,
null,
context -> {
if ("summary".equals(context.getStringValue(TextAgentResponseHandler.UPDATE_KEY))) { // UPDATE_KEY helps you to identify the streamed chunk
System.out.print(context.getStringValue("summary_chunk")); // Single chunks can be accessed via _chunk suffix, depends on the StreamedAgentResponseHandler implementation
lastStreamedSummary.set(context.getStringValue("summary")); // The current key value can also be accessed (e.g. concatenated, depends on the StreamedAgentResponseHandler implementation)
}
(context, chunk) -> {
if (chunk.content() != null) { // Since chunk can contain both content and reasoning content, we check for null
context.setStringValue("summary_chunk", chunk.content());
}
if (chunk.reasoningContent() != null) {
// System.out.print(chunk.reasoningContent()); // The reasoning can also be accessed if available
}
lastStreamedSummary.set(context.getStringValue("summary")); // The current key value can also be accessed (e.g. concatenated, depends on the StreamedAgentResponseHandler implementation)
}
),
null,
@@ -157,6 +160,7 @@ public class AgentTest {
final var resultContexts = this.agentExecutor.walk(agents.get("CodeSummarizer"), context, contextUpdate -> {
// This is an optional consumer that can be used to receive updates from the agent context
System.out.print(contextUpdate.getStringValue("summary_chunk"));
});
assertEquals(1, resultContexts.size());

34
src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java
View File

@@ -1,6 +1,7 @@
package de.extio.lmlib.client.oai;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.LocalDateTime;
@@ -45,10 +46,11 @@ public class CompletionClientTest {
@Test
void completion() throws Exception {
final var completion = this.textCompletionClient.completion(
null,
ModelCategory.SMALL,
"You are a helpful assistant",
"Say the color is green");
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.chatCompletionclient));
@@ -61,6 +63,7 @@ public class CompletionClientTest {
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.textCompletionClient.conversation(ModelCategory.SMALL, conversation);
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.chatCompletionclient));
@@ -73,11 +76,24 @@ public class CompletionClientTest {
conversation.addTurn(new Turn(TurnType.USER, "You choose! But I need you to explain the character attributes to me."));
final StringBuilder sb = new StringBuilder();
final var completion = this.textCompletionClient.streamConversation(ModelCategory.SMALL, conversation, chunk -> {
System.out.print(chunk);
sb.append(chunk);
if (chunk.reasoningContent() != null) {
System.out.print(chunk.reasoningContent());
}
if (chunk.content() != null) {
System.out.print(chunk.content());
sb.append(chunk.content());
}
});
assertEquals(completion.response(), sb.toString());
if (completion.reasoning() != null) {
// Prompt template does not separate reasoning and response from the stream at the moment
assertTrue(sb.toString().contains(completion.reasoning()));
assertTrue(sb.toString().contains(completion.response()));
assertFalse(completion.response().contains(completion.reasoning()));
}
else {
assertEquals(completion.response(), sb.toString());
}
assertTrue(Grader.assessScoreBinary("Does the text mention a role-playing game character?", completion.response(), this.chatCompletionclient));
}
@@ -88,6 +104,7 @@ public class CompletionClientTest {
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.chatCompletionclient.conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.chatCompletionclient));
@@ -100,8 +117,13 @@ public class CompletionClientTest {
conversation.addTurn(new Turn(TurnType.USER, "You choose! But I need you to explain the opening to me."));
final StringBuilder sb = new StringBuilder();
final var completion = this.chatCompletionclient.streamConversation(ModelCategory.MEDIUM, conversation, chunk -> {
System.out.print(chunk);
sb.append(chunk);
if (chunk.reasoningContent() != null) {
System.out.print(chunk.reasoningContent());
}
if (chunk.content() != null) {
System.out.print(chunk.content());
sb.append(chunk.content());
}
});
assertEquals(completion.response(), sb.toString());

6
src/test/resources/local-chatcompletion.properties
View File

@@ -1,7 +1,7 @@
prompts=
tokenEncoding=cl100k_base
maxTokens=2000
maxContextLength=8000
maxTokens=2500
maxContextLength=16000
temperature=0.4
topP=1.0
modelProvider=OAI_CHAT_COMPLETION
@@ -10,3 +10,5 @@ url=http://localhost:5001
apiKey=
cost1MInTokens=0
cost1MOutTokens=0
reasoningEffort=medium
reasoningSummaryDetails=auto

6
src/test/resources/local-completion.properties
View File

@@ -1,7 +1,7 @@
prompts=gemma
prompts=gpt-oss
tokenEncoding=cl100k_base
maxTokens=2000
maxContextLength=8000
maxTokens=2500
maxContextLength=16000
temperature=0.4
topP=1.0
modelProvider=OAI_TEXT_COMPLETION