diff --git a/pom.xml b/pom.xml index a130545..ac75034 100755 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ lm-lib de.extio lm-lib - 1.6.24 + 1.7.0 21 diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java index a377efc..9738c7e 100755 --- a/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java @@ -10,10 +10,11 @@ import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.json.JsonReadFeature; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import de.extio.lmlib.agent.AgentContext; import de.extio.lmlib.client.Completion; @@ -34,24 +35,25 @@ public class JsonAgentResponseHandler implements AgentResponseHandler { public JsonAgentResponseHandler(final String prefixFields) { this.prefixFields = prefixFields == null ? "" : prefixFields; - this.objectMapper = new ObjectMapper(); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false); - this.objectMapper.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_MISSING_VALUES, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_COMMA, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_YAML_COMMENTS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + this.objectMapper = JsonMapper.builder() + .configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false) + .configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false) + .configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false) + .enable(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER) + .enable(JsonReadFeature.ALLOW_JAVA_COMMENTS) + .enable(JsonReadFeature.ALLOW_MISSING_VALUES) + .enable(JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS) + .enable(JsonReadFeature.ALLOW_SINGLE_QUOTES) + .enable(JsonReadFeature.ALLOW_TRAILING_COMMA) + .enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS) + .enable(JsonReadFeature.ALLOW_YAML_COMMENTS) + .enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS) + .enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS) + .enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS) + .enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES) + .build(); } @Override diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java index 7273697..f58cd24 100644 --- a/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java @@ -1,11 +1,12 @@ package de.extio.lmlib.agent.responsehandler; import de.extio.lmlib.agent.AgentContext; +import de.extio.lmlib.client.Chunk; public interface StreamedAgentResponseHandler extends AgentResponseHandler { void beforeStream(AgentContext context); - boolean handleChunk(String chunk, AgentContext context); + boolean handleChunk(Chunk chunk, AgentContext context); } diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/TextAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/TextAgentResponseHandler.java index f878132..81698ab 100755 --- a/src/main/java/de/extio/lmlib/agent/responsehandler/TextAgentResponseHandler.java +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/TextAgentResponseHandler.java @@ -1,6 +1,7 @@ package de.extio.lmlib.agent.responsehandler; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -8,33 +9,32 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import de.extio.lmlib.agent.AgentContext; +import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Completion; public class TextAgentResponseHandler implements StreamedAgentResponseHandler { - public static final String UPDATE_KEY = "chunkUpdateKey"; - private static final Logger LOGGER = LoggerFactory.getLogger(TextAgentResponseHandler.class); - + private final String key; - private final String chunkKey; + private final String reasoningKey; private final Function transformer; private final Consumer beforeStream; - - private final Function chunkTransformer; - - private final Consumer afterChunkUpdate; + + private final Function chunkTransformer; + + private final BiConsumer afterChunkUpdate; public TextAgentResponseHandler(final String key) { this(key, null, null, null, null); } - public TextAgentResponseHandler(final String key, final Function transformer, final Consumer beforeStream, final Function chunkTransformer, final Consumer afterChunkUpdate) { + public TextAgentResponseHandler(final String key, final Function transformer, final Consumer beforeStream, final Function chunkTransformer, final BiConsumer afterChunkUpdate) { this.key = key; - this.chunkKey = key.concat("_chunk"); + this.reasoningKey = key.concat("_reasoning"); this.transformer = transformer; this.beforeStream = beforeStream; this.chunkTransformer = chunkTransformer; @@ -44,7 +44,14 @@ public class TextAgentResponseHandler implements StreamedAgentResponseHandler { @Override public boolean handle(final Completion completion, final AgentContext context) { var response = completion.response(); - LOGGER.debug("Response: {}", response); + final var reasoning = completion.reasoning(); + if (reasoning != null && !reasoning.isBlank()) { + context.setStringValue(this.reasoningKey, reasoning); + LOGGER.debug("Reasoning: {}; Response: {}", reasoning, response); + } + else { + LOGGER.debug("Response: {}", response); + } if (this.transformer != null) { response = this.transformer.apply(response); } @@ -55,24 +62,27 @@ public class TextAgentResponseHandler implements StreamedAgentResponseHandler { @Override public void beforeStream(final AgentContext context) { context.setStringValue(this.key, ""); + context.getContext().remove(this.reasoningKey); if (this.beforeStream != null) { this.beforeStream.accept(context); } } @Override - public boolean handleChunk(final String chunk_, final AgentContext context) { + public boolean handleChunk(final Chunk chunk_, final AgentContext context) { var chunk = chunk_; if (this.chunkTransformer != null) { chunk = this.chunkTransformer.apply(chunk); } - context.setStringValue(UPDATE_KEY, this.key); if (afterChunkUpdate != null) { - context.setStringValue(this.chunkKey, chunk); - context.setStringValue(this.key, Objects.requireNonNullElse(context.getStringValue(this.key), "").concat(chunk)); - afterChunkUpdate.accept(context); - context.getContext().remove(UPDATE_KEY); - context.getContext().remove(this.chunkKey); + if (chunk.content() != null) { + context.setStringValue(this.key, Objects.requireNonNullElseGet(context.getStringValue(this.key), () -> "").concat(chunk.content())); + } + if (chunk.reasoningContent() != null) { + context.setStringValue(this.reasoningKey, Objects.requireNonNullElseGet(context.getStringValue(this.reasoningKey), () -> "").concat(chunk.reasoningContent())); + } + + afterChunkUpdate.accept(context, chunk); } return true; } diff --git a/src/main/java/de/extio/lmlib/client/Chunk.java b/src/main/java/de/extio/lmlib/client/Chunk.java new file mode 100644 index 0000000..aa87b7b --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/Chunk.java @@ -0,0 +1,5 @@ +package de.extio.lmlib.client; + +public record Chunk(String content, String reasoningContent) { + +} diff --git a/src/main/java/de/extio/lmlib/client/Client.java b/src/main/java/de/extio/lmlib/client/Client.java index 220f8c9..256547a 100755 --- a/src/main/java/de/extio/lmlib/client/Client.java +++ b/src/main/java/de/extio/lmlib/client/Client.java @@ -15,7 +15,7 @@ public interface Client { Completion conversation(ModelCategory modelCategory, Conversation conversation); - Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer chunkConsumer); + Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer chunkConsumer); ModelProvider getModelProvider(); } diff --git a/src/main/java/de/extio/lmlib/client/ClientService.java b/src/main/java/de/extio/lmlib/client/ClientService.java index 352ea03..42faa87 100755 --- a/src/main/java/de/extio/lmlib/client/ClientService.java +++ b/src/main/java/de/extio/lmlib/client/ClientService.java @@ -69,7 +69,7 @@ public class ClientService { } @Override - public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer chunkConsumer) { + public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer chunkConsumer) { Conversation conversation = conversation_; for (final var interceptor : this.completionInterceptors) { conversation = interceptor.before(modelCategory, conversation); diff --git a/src/main/java/de/extio/lmlib/client/Completion.java b/src/main/java/de/extio/lmlib/client/Completion.java index 3f87e32..c66c30c 100755 --- a/src/main/java/de/extio/lmlib/client/Completion.java +++ b/src/main/java/de/extio/lmlib/client/Completion.java @@ -1,5 +1,5 @@ package de.extio.lmlib.client; -public final record Completion(String response, CompletionFinishReason finishReason, CompletionStatistics statistics) { +public final record Completion(String response, String reasoning, CompletionFinishReason finishReason, CompletionStatistics statistics) { } diff --git a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java index dcdbfea..a689a49 100755 --- a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java +++ b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java @@ -31,6 +31,8 @@ import com.azure.core.http.netty.NettyAsyncHttpClientBuilder; import com.azure.core.http.policy.ExponentialBackoffOptions; import com.azure.core.http.policy.RetryOptions; +import de.extio.lmlib.client.Chunk; +import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Client; import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.CompletionFinishReason; @@ -91,7 +93,7 @@ public final class AzureAiClient implements Client { } @Override - public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer chunkConsumer) { + public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer chunkConsumer) { throw new UnsupportedOperationException("Not implemented yet"); } @@ -199,7 +201,7 @@ public final class AzureAiClient implements Client { false); LOGGER.debug("{}", statistics); - return new Completion(response.getChoice().getMessage().getContent(), finishReason, statistics); + return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics); } private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) { diff --git a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java index 5d2584e..2049611 100644 --- a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java +++ b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java @@ -10,6 +10,7 @@ import java.util.Objects; import java.util.function.Consumer; import java.util.function.Supplier; +import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Client; import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.CompletionFinishReason; @@ -44,10 +45,10 @@ public class CachedClient implements Client { } @Override - public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer chunkConsumer) { + public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer chunkConsumer) { final var response = this.getCachedResponse(modelCategory, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString()); if (response.statistics().cached()) { - chunkConsumer.accept(response.response()); + chunkConsumer.accept(new Chunk(response.response(), response.reasoning())); } return response; } @@ -62,12 +63,12 @@ public class CachedClient implements Client { final var cachedCompletion = this.cacheRepository.get(key); if (cachedCompletion != null) { - return new Completion(cachedCompletion.response(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true)); + return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true)); } final var completion = supplier.get(); if (completion != null) { - this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now())); + this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.reasoning(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now())); } return completion; } diff --git a/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java b/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java index c1431fa..f91149b 100644 --- a/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java +++ b/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java @@ -4,6 +4,6 @@ import java.time.OffsetDateTime; import de.extio.lmlib.client.CompletionFinishReason; -public record CachedCompletion(String response, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) { +public record CachedCompletion(String response, String reasoning, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) { } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index 2cba931..1c15f65 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -15,12 +15,17 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.web.reactive.function.client.WebClient; -import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.json.JsonReadFeature; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import de.extio.lmlib.client.Chunk; +import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Client; import de.extio.lmlib.client.Completion; +import de.extio.lmlib.client.CompletionFinishReason; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.oai.ModelNameSupplier; import de.extio.lmlib.profile.ModelCategory; @@ -28,6 +33,7 @@ import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.profile.ModelProfileService; import de.extio.lmlib.token.Tokenizer; +import reactor.util.retry.Retry; public abstract class AbstractCompletionClient implements Client, DisposableBean { @@ -52,28 +58,30 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean protected final CompletionStatistics totalStatistics = new CompletionStatistics(); protected final ObjectMapper objectMapper; - + public AbstractCompletionClient() { - this.objectMapper = new ObjectMapper(); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false); - this.objectMapper.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_MISSING_VALUES, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_COMMA, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_YAML_COMMENTS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS, true); - this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + this.objectMapper = JsonMapper.builder() + .configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false) + .configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false) + .configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .enable(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER) + .enable(JsonReadFeature.ALLOW_JAVA_COMMENTS) + .enable(JsonReadFeature.ALLOW_MISSING_VALUES) + .enable(JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS) + .enable(JsonReadFeature.ALLOW_SINGLE_QUOTES) + .enable(JsonReadFeature.ALLOW_TRAILING_COMMA) + .enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS) + .enable(JsonReadFeature.ALLOW_YAML_COMMENTS) + .enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS) + .enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS) + .enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS) + .enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES) + .build(); } - + @Override public Completion completion(final ModelCategory modelCategory, final String system, final String text) { return this.conversation(modelCategory, Conversation.create(system, text)); @@ -85,7 +93,7 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean } @Override - public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer chunkConsumer) { + public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer chunkConsumer) { final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_; final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile()); if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) { @@ -95,7 +103,33 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean return this.requestCompletion(conversation, modelCategory, modelProfile, chunkConsumer); } - protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer chunkConsumer); + protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer chunkConsumer); + + protected CompletionFinishReason mapFinishReason(final String finishReason) { + return switch (finishReason) { + case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED; + case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED; + default -> CompletionFinishReason.DONE; + }; + } + + protected Retry createRetrySpec() { + return Retry.backoff(99, Duration.ofSeconds(1)) + .maxBackoff(Duration.ofSeconds(10)) + .jitter(0.25d) + .filter(throwable -> { + return throwable instanceof java.io.IOException || throwable instanceof java.net.ConnectException || throwable instanceof java.net.http.HttpTimeoutException; + }) + .doAfterRetry(rs -> LOGGER.warn("Retrying failed request: " + rs.failure().getClass().getName() + " " + rs.failure().getMessage())); + } + + protected void configureStreamOptions(final boolean enableStreaming, final Consumer streamOptionsConsumer) { + if (enableStreaming) { + final var streamOptions = new StreamOptions(); + streamOptions.setIncludeUsage(true); + streamOptionsConsumer.accept(streamOptions); + } + } protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) { int inTokens = 0; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java new file mode 100644 index 0000000..8e5531d --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java @@ -0,0 +1,72 @@ +package de.extio.lmlib.client.oai.completion; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public abstract class AbstractCompletionRequest { + + protected String model; + + protected Integer maxTokens; + + protected Double temperature; + + protected Double topP; + + protected Boolean stream; + + protected StreamOptions streamOptions; + + @JsonProperty("model") + public String getModel() { + return this.model; + } + + public void setModel(final String model) { + this.model = model; + } + + @JsonProperty("max_tokens") + public Integer getMaxTokens() { + return this.maxTokens; + } + + public void setMaxTokens(final Integer maxTokens) { + this.maxTokens = maxTokens; + } + + @JsonProperty("temperature") + public Double getTemperature() { + return this.temperature; + } + + public void setTemperature(final Double temperature) { + this.temperature = temperature; + } + + @JsonProperty("top_p") + public Double getTopP() { + return this.topP; + } + + public void setTopP(final Double topP) { + this.topP = topP; + } + + @JsonProperty("stream") + public Boolean isStream() { + return this.stream; + } + + public void setStream(final Boolean stream) { + this.stream = stream; + } + + @JsonProperty("stream_options") + public StreamOptions getStreamOptions() { + return streamOptions; + } + + public void setStreamOptions(final StreamOptions streamOptions) { + this.streamOptions = streamOptions; + } +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java new file mode 100644 index 0000000..275b7c8 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java @@ -0,0 +1,50 @@ +package de.extio.lmlib.client.oai.completion; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public abstract class AbstractCompletionResponse { + + protected String id; + + protected String object; + + protected Integer created; + + protected Usage usage; + + @JsonProperty("id") + public String getId() { + return this.id; + } + + public void setId(final String id) { + this.id = id; + } + + @JsonProperty("object") + public String getObject() { + return this.object; + } + + public void setObject(final String object) { + this.object = object; + } + + @JsonProperty("created") + public Integer getCreated() { + return this.created; + } + + public void setCreated(final Integer created) { + this.created = created; + } + + @JsonProperty("usage") + public Usage getUsage() { + return usage; + } + + public void setUsage(final Usage usage) { + this.usage = usage; + } +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionStreamOptions.java b/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java similarity index 69% rename from src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionStreamOptions.java rename to src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java index 6ecd9b5..b9da1ed 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionStreamOptions.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java @@ -1,10 +1,10 @@ -package de.extio.lmlib.client.oai.completion.text; +package de.extio.lmlib.client.oai.completion; import com.fasterxml.jackson.annotation.JsonProperty; -public class CompletionStreamOptions { +public class StreamOptions { - private boolean includeUsage; + protected boolean includeUsage; @JsonProperty("include_usage") public boolean isIncludeUsage() { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index 321b208..073398e 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -1,6 +1,5 @@ package de.extio.lmlib.client.oai.completion.chat; -import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayDeque; import java.util.ArrayList; @@ -16,17 +15,15 @@ import org.springframework.stereotype.Component; import com.fasterxml.jackson.core.JsonProcessingException; +import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Completion; -import de.extio.lmlib.client.CompletionFinishReason; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; -import de.extio.lmlib.client.oai.completion.FinishReasons; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import reactor.core.publisher.Mono; -import reactor.util.retry.Retry; @Component public class ChatCompletionClient extends AbstractCompletionClient { @@ -43,7 +40,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { } @Override - protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer chunkConsumer) { + protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer chunkConsumer) { final var chat = createChats(conversation, modelProfile); final var request = new ChatCompletionRequest(); @@ -53,13 +50,37 @@ public class ChatCompletionClient extends AbstractCompletionClient { request.setMaxCompletionTokens(modelProfile.maxTokens()); request.setTemperature(modelProfile.temperature()); request.setTopP(modelProfile.topP()); - if (chunkConsumer == null) { - request.setStream(false); + request.setStream(chunkConsumer != null); + if (chunkConsumer != null) { + this.configureStreamOptions(true, request::setStreamOptions); } - else { - request.setStream(true); - request.setStreamOptions(new ChatCompletionStreamOptions()); - request.getStreamOptions().setIncludeUsage(true); + if (modelProfile.reasoningEffort() != null && !modelProfile.reasoningEffort().isBlank()) { + request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); + try { + request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(modelProfile.reasoningEffort())); + } + catch (final IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid reasoning effort in the model profile: " + modelProfile.reasoningEffort(), e); + } + } + if (modelProfile.reasoningSummaryDetails() != null && !modelProfile.reasoningSummaryDetails().isBlank()) { + if (request.getReasoning() == null) { + request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); + } + try { + request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(modelProfile.reasoningSummaryDetails()).name()); + } + catch (final IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid reasoning summary details in the model profile: " + modelProfile.reasoningSummaryDetails(), e); + } + } + + String requestBody; + try { + requestBody = this.objectMapper.writeValueAsString(request); + } + catch (final JsonProcessingException e) { + throw new IllegalStateException("Cannot serialize request body", e); } LOGGER.debug("Requesting chat completion at {}", modelProfile.url()); @@ -70,19 +91,25 @@ public class ChatCompletionClient extends AbstractCompletionClient { .method(HttpMethod.POST) .uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build()) .header("Content-Type", "application/json") - .bodyValue(request) + .bodyValue(requestBody) .exchangeToMono(clientResponse -> { + if (clientResponse.statusCode().isError()) { + return clientResponse.bodyToMono(String.class) + .flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody))); + } + if (chunkConsumer == null) { return clientResponse.bodyToMono(ChatCompletionResponse.class); } else { final var chatCompletionResponse = new ChatCompletionResponse(); chatCompletionResponse.setChoices(List.of(new ChatChoice())); - final StringBuilder contenStringBuilder = new StringBuilder(); + final StringBuilder contentStringBuilder = new StringBuilder(); + final StringBuilder reasoningStringBuilder = new StringBuilder(); return clientResponse.bodyToFlux(String.class) .doOnNext(buffer -> { try { - if (! "[DONE]".equals(buffer)) { + if (!"[DONE]".equals(buffer)) { final var streamResponse = this.objectMapper.readValue(buffer, ChatCompletionResponse.class); if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) { @@ -101,9 +128,20 @@ public class ChatCompletionClient extends AbstractCompletionClient { if (streamResponse.getChoices().getFirst().getFinishReason() != null) { chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason()); } - if (streamResponse.getChoices().getFirst().getMessage() != null && streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) { - contenStringBuilder.append(streamResponse.getChoices().getFirst().getMessage().getContent()); - chunkConsumer.accept(streamResponse.getChoices().getFirst().getMessage().getContent()); + if (streamResponse.getChoices().getFirst().getMessage() != null) { + String content = null; + String reasoning = null; + if (streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) { + content = streamResponse.getChoices().getFirst().getMessage().getContent(); + contentStringBuilder.append(content); + } + if (streamResponse.getChoices().getFirst().getMessage().getReasoningContent() != null && !streamResponse.getChoices().getFirst().getMessage().getReasoningContent().isEmpty()) { + reasoning = streamResponse.getChoices().getFirst().getMessage().getReasoningContent(); + reasoningStringBuilder.append(reasoning); + } + if (content != null || reasoning != null) { + chunkConsumer.accept(new Chunk(content, reasoning)); + } } } } @@ -113,27 +151,23 @@ public class ChatCompletionClient extends AbstractCompletionClient { } }) .then(Mono. fromSupplier(() -> { - chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contenStringBuilder.toString())); + chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contentStringBuilder.toString(), reasoningStringBuilder.toString())); return chatCompletionResponse; })); } }) - .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) + .retryWhen(this.createRetrySpec()) .block(); if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) { throw new IllegalStateException("No parseable response content from server"); } - final var choice = response.getChoices().getFirst(); final var content = response.getChoices().getFirst().getMessage().getContent(); - final var finishReason = switch (choice.getFinishReason()) { - case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED; - case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED; - default -> CompletionFinishReason.DONE; - }; + final var reasoning = response.getChoices().getFirst().getMessage().getReasoningContent(); + final var finishReason = this.mapFinishReason(choice.getFinishReason()); final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content); - return new Completion(content, finishReason, statistics); + return new Completion(content, reasoning, finishReason, statistics); } private List createChats(final Conversation conversation, final ModelProfile modelProfile) { @@ -165,7 +199,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { if (first && userTurn != null) { if (systemTurn != null) { - chat.add(new ChatMessage("system", systemTurn.text())); + chat.add(new ChatMessage("system", systemTurn.text(), null)); } if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) { break; @@ -202,7 +236,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { trimmed = true; } - chat.add(new ChatMessage(role, prompt)); + chat.add(new ChatMessage(role, prompt, null)); return trimmed; } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java index 691c9c4..facb6af 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java @@ -2,36 +2,17 @@ package de.extio.lmlib.client.oai.completion.chat; import java.util.List; -import java.util.List; - import com.fasterxml.jackson.annotation.JsonProperty; -final class ChatCompletionRequest { - - private String model; +import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest; + +final class ChatCompletionRequest extends AbstractCompletionRequest { private List messages; - private Integer maxTokens; - private Integer maxCompletionTokens; - - private Double temperature; - - private Double topP; - - private Boolean stream; - - private ChatCompletionStreamOptions streamOptions; - - @JsonProperty("model") - public String getModel() { - return this.model; - } - - public void setModel(final String model) { - this.model = model; - } + + private ChatCompletionsRequestReasoning reasoning; @JsonProperty("messages") public List getMessages() { @@ -42,15 +23,6 @@ final class ChatCompletionRequest { this.messages = messages; } - @JsonProperty("max_tokens") - public Integer getMaxTokens() { - return this.maxTokens; - } - - public void setMaxTokens(final Integer maxTokens) { - this.maxTokens = maxTokens; - } - @JsonProperty("max_completion_tokens") public Integer getMaxCompletionTokens() { return this.maxCompletionTokens; @@ -60,40 +32,47 @@ final class ChatCompletionRequest { this.maxCompletionTokens = maxCompletionTokens; } - @JsonProperty("temperature") - public Double getTemperature() { - return this.temperature; + @JsonProperty("reasoning") + public ChatCompletionsRequestReasoning getReasoning() { + return this.reasoning; } - - public void setTemperature(final Double temperature) { - this.temperature = temperature; + + public void setReasoning(final ChatCompletionsRequestReasoning reasoning) { + this.reasoning = reasoning; } - - @JsonProperty("top_p") - public Double getTopP() { - return this.topP; + + final static class ChatCompletionsRequestReasoning { + + private ChatCompletionsRequestReasoningEffort effort; + + private String summary; + + @JsonProperty("effort") + public ChatCompletionsRequestReasoningEffort getEffort() { + return this.effort; + } + + public void setEffort(final ChatCompletionsRequestReasoningEffort effort) { + this.effort = effort; + } + + @JsonProperty("summary") + public String getSummary() { + return this.summary; + } + + public void setSummary(final String summary) { + this.summary = summary; + } + } - - public void setTopP(final Double topP) { - this.topP = topP; + + static enum ChatCompletionsRequestReasoningEffort { + minimal, low, medium, high } - - @JsonProperty("stream") - public Boolean isStream() { - return this.stream; + + static enum ChatCompletionsRequestReasoningSummaryDetails { + auto, concise, detailed } - - public void setStream(final Boolean stream) { - this.stream = stream; - } - - @JsonProperty("stream_options") - public ChatCompletionStreamOptions getStreamOptions() { - return streamOptions; - } - - public void setStreamOptions(ChatCompletionStreamOptions streamOptions) { - this.streamOptions = streamOptions; - } - + } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java index 12bcd0a..34c7dbd 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java @@ -4,47 +4,12 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonProperty; -import de.extio.lmlib.client.oai.completion.Usage; +import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse; -final class ChatCompletionResponse { - - private String id; - - private String object; - - private Integer created; +final class ChatCompletionResponse extends AbstractCompletionResponse { private List choices; - private Usage usage; - - @JsonProperty("id") - public String getId() { - return this.id; - } - - public void setId(final String id) { - this.id = id; - } - - @JsonProperty("object") - public String getObject() { - return this.object; - } - - public void setObject(final String object) { - this.object = object; - } - - @JsonProperty("created") - public Integer getCreated() { - return this.created; - } - - public void setCreated(final Integer created) { - this.created = created; - } - @JsonProperty("choices") public List getChoices() { return this.choices; @@ -53,15 +18,6 @@ final class ChatCompletionResponse { public void setChoices(final List choices) { this.choices = choices; } - - @JsonProperty("usage") - public Usage getUsage() { - return usage; - } - - public void setUsage(final Usage usage) { - this.usage = usage; - } @Override public String toString() { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionStreamOptions.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionStreamOptions.java deleted file mode 100644 index 15e4417..0000000 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionStreamOptions.java +++ /dev/null @@ -1,17 +0,0 @@ -package de.extio.lmlib.client.oai.completion.chat; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class ChatCompletionStreamOptions { - - private boolean includeUsage; - - @JsonProperty("include_usage") - public boolean isIncludeUsage() { - return includeUsage; - } - - public void setIncludeUsage(final boolean includeUsage) { - this.includeUsage = includeUsage; - } -} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java index af8fdd8..c15f1b1 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java @@ -8,14 +8,17 @@ final class ChatMessage { private String content; + private String reasoningContent; + public ChatMessage() { } - public ChatMessage(final String role, final String content) { + public ChatMessage(final String role, final String content, final String reasoningContent) { this.role = role; this.content = content; + this.reasoningContent = reasoningContent; } - + @JsonProperty("role") public String getRole() { return role; @@ -34,4 +37,12 @@ final class ChatMessage { this.content = content; } + @JsonProperty("reasoning_content") + public String getReasoningContent() { + return reasoningContent; + } + + public void setReasoningContent(final String reasoningContent) { + this.reasoningContent = reasoningContent; + } } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java index 70437fe..4617e26 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java @@ -2,31 +2,12 @@ package de.extio.lmlib.client.oai.completion.text; import com.fasterxml.jackson.annotation.JsonProperty; -final class CompletionRequest { - - private String model; +import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest; + +final class CompletionRequest extends AbstractCompletionRequest { private String prompt; - private Integer maxTokens; - - private Double temperature; - - private Double topP; - - private Boolean stream; - - private CompletionStreamOptions streamOptions; - - @JsonProperty("model") - public String getModel() { - return this.model; - } - - public void setModel(final String model) { - this.model = model; - } - @JsonProperty("prompt") public String getPrompt() { return this.prompt; @@ -35,49 +16,4 @@ final class CompletionRequest { public void setPrompt(final String prompt) { this.prompt = prompt; } - - @JsonProperty("max_tokens") - public Integer getMaxTokens() { - return this.maxTokens; - } - - public void setMaxTokens(final Integer maxTokens) { - this.maxTokens = maxTokens; - } - - @JsonProperty("temperature") - public Double getTemperature() { - return this.temperature; - } - - public void setTemperature(final Double temperature) { - this.temperature = temperature; - } - - @JsonProperty("top_p") - public Double getTopP() { - return this.topP; - } - - public void setTopP(final Double topP) { - this.topP = topP; - } - - @JsonProperty("stream") - public Boolean isStream() { - return this.stream; - } - - public void setStream(final Boolean stream) { - this.stream = stream; - } - - @JsonProperty("stream_options") - public CompletionStreamOptions getStreamOptions() { - return streamOptions; - } - - public void setStreamOptions(final CompletionStreamOptions streamOptions) { - this.streamOptions = streamOptions; - } } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java index 5df5537..e6acd28 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java @@ -4,50 +4,15 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonProperty; -import de.extio.lmlib.client.oai.completion.Usage; +import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse; -final class CompletionResponse { - - private String id; - - private String object; - - private Integer created; +final class CompletionResponse extends AbstractCompletionResponse { private List choices; private String content; private Boolean isStoppedEos; - - private Usage usage; - - @JsonProperty("id") - public String getId() { - return this.id; - } - - public void setId(final String id) { - this.id = id; - } - - @JsonProperty("object") - public String getObject() { - return this.object; - } - - public void setObject(final String object) { - this.object = object; - } - - @JsonProperty("created") - public Integer getCreated() { - return this.created; - } - - public void setCreated(final Integer created) { - this.created = created; - } @JsonProperty("choices") public List getChoices() { @@ -75,15 +40,6 @@ final class CompletionResponse { public void setStoppedEos(final Boolean isStoppedEos) { this.isStoppedEos = isStoppedEos; } - - @JsonProperty("usage") - public Usage getUsage() { - return usage; - } - - public void setUsage(final Usage usage) { - this.usage = usage; - } @Override public String toString() { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 53d988f..745de29 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -1,6 +1,5 @@ package de.extio.lmlib.client.oai.completion.text; -import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayDeque; import java.util.Deque; @@ -13,22 +12,20 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Component; -import com.azure.ai.inference.models.ChatChoice; import com.fasterxml.jackson.core.JsonProcessingException; +import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.CompletionFinishReason; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; -import de.extio.lmlib.client.oai.completion.FinishReasons; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.prompt.PromptStrategy; import de.extio.lmlib.prompt.PromptStrategyFactory; import reactor.core.publisher.Mono; -import reactor.util.retry.Retry; @Component public class TextCompletionClient extends AbstractCompletionClient { @@ -44,7 +41,7 @@ public class TextCompletionClient extends AbstractCompletionClient { } @Override - protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer chunkConsumer) { + protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer chunkConsumer) { final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt()); if (promptStrategy == null) { throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt()); @@ -58,13 +55,17 @@ public class TextCompletionClient extends AbstractCompletionClient { request.setMaxTokens(modelProfile.maxTokens()); request.setTemperature(modelProfile.temperature()); request.setTopP(modelProfile.topP()); - if (chunkConsumer == null) { - request.setStream(false); + request.setStream(chunkConsumer != null); + if (chunkConsumer != null) { + this.configureStreamOptions(true, request::setStreamOptions); } - else { - request.setStream(true); - request.setStreamOptions(new CompletionStreamOptions()); - request.getStreamOptions().setIncludeUsage(true); + + String requestBody; + try { + requestBody = this.objectMapper.writeValueAsString(request); + } + catch (final JsonProcessingException e) { + throw new IllegalStateException("Cannot serialize request body", e); } LOGGER.debug("Requesting completion at {}", modelProfile.url()); @@ -74,8 +75,13 @@ public class TextCompletionClient extends AbstractCompletionClient { .method(HttpMethod.POST) .uri(uriBuilder -> uriBuilder.path("/v1/completions").build()) .header("Content-Type", "application/json") - .bodyValue(request) + .bodyValue(requestBody) .exchangeToMono(clientResponse -> { + if (clientResponse.statusCode().isError()) { + return clientResponse.bodyToMono(String.class) + .flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody))); + } + if (chunkConsumer == null) { return clientResponse.bodyToMono(CompletionResponse.class); } @@ -116,7 +122,7 @@ public class TextCompletionClient extends AbstractCompletionClient { } if (streamResponse.getChoices().getFirst().getText() != null && !streamResponse.getChoices().getFirst().getText().isEmpty()) { contentStringBuilder.append(streamResponse.getChoices().getFirst().getText()); - chunkConsumer.accept(streamResponse.getChoices().getFirst().getText()); + chunkConsumer.accept(new Chunk(streamResponse.getChoices().getFirst().getText(), null)); } } } @@ -131,30 +137,33 @@ public class TextCompletionClient extends AbstractCompletionClient { })); } }) - .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) + .retryWhen(this.createRetrySpec()) .block(); String content = null; + String reasoning = null; CompletionFinishReason finishReason = null; if (response.getChoices() != null && !response.getChoices().isEmpty()) { final var choice = response.getChoices().getFirst(); if (response.getChoices().getFirst().getText() != null) { - content = promptStrategy.removeEOT(choice.getText()); + content = promptStrategy.getResponse(choice.getText()); + reasoning = promptStrategy.getReasoning(choice.getText()); } if (response.getChoices().getFirst().getFinishReason() != null) { - finishReason = switch (choice.getFinishReason()) { - case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED; - case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED; - default -> CompletionFinishReason.DONE; - }; + finishReason = this.mapFinishReason(choice.getFinishReason()); } } if (content == null && response.getContent() != null) { - content = promptStrategy.removeEOT(response.getContent()); + content = promptStrategy.getResponse(response.getContent()); + reasoning = promptStrategy.getReasoning(response.getContent()); } if (content == null) { throw new IllegalStateException("No parseable response content from server"); } + content = promptStrategy.removeEOT(content); + if (reasoning != null) { + reasoning = promptStrategy.removeEOT(reasoning); + } if (finishReason == null) { if (response.isStoppedEos() != null) { finishReason = Boolean.TRUE.equals(response.isStoppedEos()) ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED; @@ -165,7 +174,7 @@ public class TextCompletionClient extends AbstractCompletionClient { } final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent()); - return new Completion(content, finishReason, statistics); + return new Completion(content, reasoning, finishReason, statistics); } private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) { diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfile.java b/src/main/java/de/extio/lmlib/profile/ModelProfile.java index 75fd19d..adb817c 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfile.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfile.java @@ -14,7 +14,9 @@ public record ModelProfile( String url, String apiKey, BigDecimal costInToken, - BigDecimal costOutToken) { + BigDecimal costOutToken, + String reasoningEffort, + String reasoningSummaryDetails) { public static enum ModelProvider { AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION; diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java index bb324b8..6a79aec 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java @@ -85,8 +85,14 @@ public class ModelProfileService { .map(value -> new BigDecimal(value)) .orElse(BigDecimal.ZERO) .divide(new BigDecimal(1000000.0)); + final var reasoningEffort = Optional.ofNullable(resource.getProperty("reasoningEffort")) + .map(Object::toString) + .orElse(null); + final var reasoningSummaryDetails = Optional.ofNullable(resource.getProperty("reasoningSummaryDetails")) + .map(Object::toString) + .orElse(null); - //To resolve api key placeholder ${exchangeautodocumentation.azure.apikey} in model-name.properties file + //To resolve api key placeholder ${apikey} in model-name.properties file String apiKey = null; synchronized (this.environment) { if (this.environment instanceof final ConfigurableEnvironment configurableEnvironment) { @@ -97,7 +103,7 @@ public class ModelProfileService { .orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName)); } - return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken); + return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken, reasoningEffort, reasoningSummaryDetails); } catch (final IOException e) { LOGGER.error("Error while reading model profile", e); diff --git a/src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java b/src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java new file mode 100644 index 0000000..5a4052e --- /dev/null +++ b/src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java @@ -0,0 +1,110 @@ +package de.extio.lmlib.prompt; + +import org.springframework.stereotype.Component; + +@Component +public class GptOssPromptStrategy implements PromptStrategy { + + private static final String finalMarker = "<|channel|>final<|message|>"; + + private static final String startMarker = "<|start|>"; + + private static final String assistantStart = startMarker + "assistant"; + + private static final String analysisMarker = "<|channel|>analysis<|message|>"; + + @Override + public StringBuilder start(final String system, final String user) { + final StringBuilder prompt = new StringBuilder(); + if (!system.isEmpty()) { + prompt.append("<|start|>system<|message|>"); + prompt.append(system); + prompt.append("<|end|>"); + } + prompt.append("<|start|>user<|message|>"); + prompt.append(user); + prompt.append("<|end|>"); + prompt.append(assistantStart); + + return prompt; + } + + @Override + public void next(final StringBuilder prompt, final String assistant, final String user) { + final String assistantContent = assistant == null ? "" : assistant; + if (assistantContent.startsWith(assistantStart)) { + final int assistantStartLength = assistantStart.length(); + if (prompt.length() >= assistantStartLength) { + boolean endsWithAssistantStart = true; + for (int i = 0; i < assistantStartLength; i++) { + if (prompt.charAt(prompt.length() - assistantStartLength + i) != assistantStart.charAt(i)) { + endsWithAssistantStart = false; + break; + } + } + if (endsWithAssistantStart) { + prompt.setLength(prompt.length() - assistantStartLength); + } + } + } + if (assistantContent.contains(finalMarker)) { + prompt.append(assistantContent); + if (!assistantContent.endsWith("<|end|>")) { + prompt.append("<|end|>"); + } + } + else { + prompt.append("<|channel|>final<|message|>"); + prompt.append(assistantContent); + prompt.append("<|end|>"); + } + prompt.append("<|start|>user<|message|>"); + prompt.append(user == null ? "" : user); + prompt.append("<|end|>"); + prompt.append(assistantStart); + } + + @Override + public String removeEOT(final String prompt) { + return prompt.strip().replace("<|end|>", "").replace("<|return|>", ""); + } + + @Override + public String getPromptName() { + return "gpt-oss"; + } + + @Override + public String getResponse(final String prompt) { + if (prompt == null) { + return null; + } + final int markerIndex = prompt.indexOf(finalMarker); + if (markerIndex == -1) { + return prompt; + } + final int start = markerIndex + finalMarker.length(); + int end = prompt.indexOf(startMarker, start); + if (end == -1) { + end = prompt.length(); + } + return prompt.substring(start, end).strip(); + } + + @Override + public String getReasoning(final String prompt) { + if (prompt == null) { + return null; + } + final int markerIndex = prompt.indexOf(analysisMarker); + if (markerIndex == -1) { + return null; + } + final int start = markerIndex + analysisMarker.length(); + int end = prompt.indexOf(startMarker, start); + if (end == -1) { + end = prompt.length(); + } + return prompt.substring(start, end).strip(); + } +} diff --git a/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java b/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java index b04f574..5ee4312 100644 --- a/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java +++ b/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java @@ -32,9 +32,4 @@ public class NoPromptStrategy implements PromptStrategy { prompt.append(user); } - @Override - public String removeEOT(final String prompt) { - return prompt; - } - } diff --git a/src/main/java/de/extio/lmlib/prompt/PromptStrategy.java b/src/main/java/de/extio/lmlib/prompt/PromptStrategy.java index ca47a86..11fd823 100755 --- a/src/main/java/de/extio/lmlib/prompt/PromptStrategy.java +++ b/src/main/java/de/extio/lmlib/prompt/PromptStrategy.java @@ -12,4 +12,12 @@ public interface PromptStrategy { return prompt.strip(); } + default String getResponse(final String prompt) { + return prompt; + } + + default String getReasoning(final String prompt) { + return null; + } + } diff --git a/src/test/java/de/extio/lmlib/agent/AgentTest.java b/src/test/java/de/extio/lmlib/agent/AgentTest.java index 1faa3d8..e78ea36 100644 --- a/src/test/java/de/extio/lmlib/agent/AgentTest.java +++ b/src/test/java/de/extio/lmlib/agent/AgentTest.java @@ -29,7 +29,7 @@ import de.extio.lmlib.profile.ModelCategory; import io.netty.util.internal.ThreadLocalRandom; /** - * These tests have been verified with Llama 3.1 8B and Gemma2 27B models + * These tests have been verified with Llama 3.1 8B, Gemma2 27B, Gemma3 27B and GPT-OSS models */ @Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)") @SpringBootTest(webEnvironment = WebEnvironment.NONE) @@ -137,11 +137,14 @@ public class AgentTest { null, null, null, - context -> { - if ("summary".equals(context.getStringValue(TextAgentResponseHandler.UPDATE_KEY))) { // UPDATE_KEY helps you to identify the streamed chunk - System.out.print(context.getStringValue("summary_chunk")); // Single chunks can be accessed via _chunk suffix, depends on the StreamedAgentResponseHandler implementation - lastStreamedSummary.set(context.getStringValue("summary")); // The current key value can also be accessed (e.g. concatenated, depends on the StreamedAgentResponseHandler implementation) - } + (context, chunk) -> { + if (chunk.content() != null) { // Since chunk can contain both content and reasoning content, we check for null + context.setStringValue("summary_chunk", chunk.content()); + } + if (chunk.reasoningContent() != null) { + // System.out.print(chunk.reasoningContent()); // The reasoning can also be accessed if available + } + lastStreamedSummary.set(context.getStringValue("summary")); // The current key value can also be accessed (e.g. concatenated, depends on the StreamedAgentResponseHandler implementation) } ), null, @@ -157,6 +160,7 @@ public class AgentTest { final var resultContexts = this.agentExecutor.walk(agents.get("CodeSummarizer"), context, contextUpdate -> { // This is an optional consumer that can be used to receive updates from the agent context + System.out.print(contextUpdate.getStringValue("summary_chunk")); }); assertEquals(1, resultContexts.size()); diff --git a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java index 900afce..0949e44 100755 --- a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java +++ b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java @@ -1,6 +1,7 @@ package de.extio.lmlib.client.oai; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.LocalDateTime; @@ -45,10 +46,11 @@ public class CompletionClientTest { @Test void completion() throws Exception { final var completion = this.textCompletionClient.completion( - null, + ModelCategory.SMALL, "You are a helpful assistant", "Say the color is green"); + LOGGER.info(completion.reasoning()); LOGGER.info(completion.response()); assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.chatCompletionclient)); @@ -61,6 +63,7 @@ public class CompletionClientTest { conversation.addTurn(new Turn(TurnType.USER, "You choose!")); final var completion = this.textCompletionClient.conversation(ModelCategory.SMALL, conversation); + LOGGER.info(completion.reasoning()); LOGGER.info(completion.response()); assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.chatCompletionclient)); @@ -73,11 +76,24 @@ public class CompletionClientTest { conversation.addTurn(new Turn(TurnType.USER, "You choose! But I need you to explain the character attributes to me.")); final StringBuilder sb = new StringBuilder(); final var completion = this.textCompletionClient.streamConversation(ModelCategory.SMALL, conversation, chunk -> { - System.out.print(chunk); - sb.append(chunk); + if (chunk.reasoningContent() != null) { + System.out.print(chunk.reasoningContent()); + } + if (chunk.content() != null) { + System.out.print(chunk.content()); + sb.append(chunk.content()); + } }); - assertEquals(completion.response(), sb.toString()); + if (completion.reasoning() != null) { + // Prompt template does not separate reasoning and response from the stream at the moment + assertTrue(sb.toString().contains(completion.reasoning())); + assertTrue(sb.toString().contains(completion.response())); + assertFalse(completion.response().contains(completion.reasoning())); + } + else { + assertEquals(completion.response(), sb.toString()); + } assertTrue(Grader.assessScoreBinary("Does the text mention a role-playing game character?", completion.response(), this.chatCompletionclient)); } @@ -88,6 +104,7 @@ public class CompletionClientTest { conversation.addTurn(new Turn(TurnType.USER, "You choose!")); final var completion = this.chatCompletionclient.conversation(ModelCategory.MEDIUM, conversation); + LOGGER.info(completion.reasoning()); LOGGER.info(completion.response()); assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.chatCompletionclient)); @@ -100,8 +117,13 @@ public class CompletionClientTest { conversation.addTurn(new Turn(TurnType.USER, "You choose! But I need you to explain the opening to me.")); final StringBuilder sb = new StringBuilder(); final var completion = this.chatCompletionclient.streamConversation(ModelCategory.MEDIUM, conversation, chunk -> { - System.out.print(chunk); - sb.append(chunk); + if (chunk.reasoningContent() != null) { + System.out.print(chunk.reasoningContent()); + } + if (chunk.content() != null) { + System.out.print(chunk.content()); + sb.append(chunk.content()); + } }); assertEquals(completion.response(), sb.toString()); diff --git a/src/test/resources/local-chatcompletion.properties b/src/test/resources/local-chatcompletion.properties index 7b0f810..67e3d7a 100755 --- a/src/test/resources/local-chatcompletion.properties +++ b/src/test/resources/local-chatcompletion.properties @@ -1,7 +1,7 @@ prompts= tokenEncoding=cl100k_base -maxTokens=2000 -maxContextLength=8000 +maxTokens=2500 +maxContextLength=16000 temperature=0.4 topP=1.0 modelProvider=OAI_CHAT_COMPLETION @@ -10,3 +10,5 @@ url=http://localhost:5001 apiKey= cost1MInTokens=0 cost1MOutTokens=0 +reasoningEffort=medium +reasoningSummaryDetails=auto \ No newline at end of file diff --git a/src/test/resources/local-completion.properties b/src/test/resources/local-completion.properties index 161698f..ee4f9ee 100755 --- a/src/test/resources/local-completion.properties +++ b/src/test/resources/local-completion.properties @@ -1,7 +1,7 @@ -prompts=gemma +prompts=gpt-oss tokenEncoding=cl100k_base -maxTokens=2000 -maxContextLength=8000 +maxTokens=2500 +maxContextLength=16000 temperature=0.4 topP=1.0 modelProvider=OAI_TEXT_COMPLETION