feat: Introduce reasoning capabilities in chat and text completion
- Added AbstractCompletionRequest and AbstractCompletionResponse classes to standardize request and response structures. - Implemented StreamOptions for streaming responses in chat completions. - Enhanced ChatCompletionClient and TextCompletionClient to support reasoning content in responses. - Updated ModelProfile to include reasoning effort and summary details. - Created GptOssPromptStrategy to handle specific prompt formatting and reasoning extraction. - Modified existing tests to validate reasoning content in completions. - Updated configuration properties for reasoning settings in local properties files.
This commit is contained in:
extio-de
5
src/main/java/de/extio/lmlib/client/Chunk.java
Normal file
5
src/main/java/de/extio/lmlib/client/Chunk.java
Normal file
@@ -0,0 +1,5 @@
|
||||
package de.extio.lmlib.client;
|
||||
|
||||
public record Chunk(String content, String reasoningContent) {
|
||||
|
||||
}
|
||||
@@ -15,7 +15,7 @@ public interface Client {
|
||||
|
||||
Completion conversation(ModelCategory modelCategory, Conversation conversation);
|
||||
|
||||
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<String> chunkConsumer);
|
||||
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer);
|
||||
|
||||
ModelProvider getModelProvider();
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ public class ClientService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<String> chunkConsumer) {
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
|
||||
Conversation conversation = conversation_;
|
||||
for (final var interceptor : this.completionInterceptors) {
|
||||
conversation = interceptor.before(modelCategory, conversation);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
package de.extio.lmlib.client;
|
||||
|
||||
public final record Completion(String response, CompletionFinishReason finishReason, CompletionStatistics statistics) {
|
||||
public final record Completion(String response, String reasoning, CompletionFinishReason finishReason, CompletionStatistics statistics) {
|
||||
|
||||
}
|
||||
|
||||
@@ -31,6 +31,8 @@ import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
|
||||
import com.azure.core.http.policy.ExponentialBackoffOptions;
|
||||
import com.azure.core.http.policy.RetryOptions;
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
@@ -91,7 +93,7 @@ public final class AzureAiClient implements Client {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<String> chunkConsumer) {
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
throw new UnsupportedOperationException("Not implemented yet");
|
||||
}
|
||||
|
||||
@@ -199,7 +201,7 @@ public final class AzureAiClient implements Client {
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
|
||||
return new Completion(response.getChoice().getMessage().getContent(), finishReason, statistics);
|
||||
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
|
||||
}
|
||||
|
||||
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
|
||||
@@ -10,6 +10,7 @@ import java.util.Objects;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
@@ -44,10 +45,10 @@ public class CachedClient implements Client {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<String> chunkConsumer) {
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
final var response = this.getCachedResponse(modelCategory, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
|
||||
if (response.statistics().cached()) {
|
||||
chunkConsumer.accept(response.response());
|
||||
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
|
||||
}
|
||||
return response;
|
||||
}
|
||||
@@ -62,12 +63,12 @@ public class CachedClient implements Client {
|
||||
|
||||
final var cachedCompletion = this.cacheRepository.get(key);
|
||||
if (cachedCompletion != null) {
|
||||
return new Completion(cachedCompletion.response(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true));
|
||||
return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true));
|
||||
}
|
||||
|
||||
final var completion = supplier.get();
|
||||
if (completion != null) {
|
||||
this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now()));
|
||||
this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.reasoning(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now()));
|
||||
}
|
||||
return completion;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,6 @@ import java.time.OffsetDateTime;
|
||||
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
|
||||
public record CachedCompletion(String response, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) {
|
||||
public record CachedCompletion(String response, String reasoning, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) {
|
||||
|
||||
}
|
||||
|
||||
@@ -15,12 +15,17 @@ import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.core.json.JsonReadFeature;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.json.JsonMapper;
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.oai.ModelNameSupplier;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
@@ -28,6 +33,7 @@ import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
public abstract class AbstractCompletionClient implements Client, DisposableBean {
|
||||
|
||||
@@ -52,28 +58,30 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
||||
protected final CompletionStatistics totalStatistics = new CompletionStatistics();
|
||||
|
||||
protected final ObjectMapper objectMapper;
|
||||
|
||||
|
||||
public AbstractCompletionClient() {
|
||||
this.objectMapper = new ObjectMapper();
|
||||
this.objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
|
||||
this.objectMapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false);
|
||||
this.objectMapper.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false);
|
||||
this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_MISSING_VALUES, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_COMMA, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_YAML_COMMENTS, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS, true);
|
||||
this.objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
|
||||
this.objectMapper = JsonMapper.builder()
|
||||
.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
|
||||
.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false)
|
||||
.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false)
|
||||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
||||
.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false)
|
||||
.serializationInclusion(JsonInclude.Include.NON_NULL)
|
||||
.enable(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER)
|
||||
.enable(JsonReadFeature.ALLOW_JAVA_COMMENTS)
|
||||
.enable(JsonReadFeature.ALLOW_MISSING_VALUES)
|
||||
.enable(JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS)
|
||||
.enable(JsonReadFeature.ALLOW_SINGLE_QUOTES)
|
||||
.enable(JsonReadFeature.ALLOW_TRAILING_COMMA)
|
||||
.enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS)
|
||||
.enable(JsonReadFeature.ALLOW_YAML_COMMENTS)
|
||||
.enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS)
|
||||
.enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS)
|
||||
.enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS)
|
||||
.enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
|
||||
return this.conversation(modelCategory, Conversation.create(system, text));
|
||||
@@ -85,7 +93,7 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<String> chunkConsumer) {
|
||||
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
|
||||
@@ -95,7 +103,33 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
||||
return this.requestCompletion(conversation, modelCategory, modelProfile, chunkConsumer);
|
||||
}
|
||||
|
||||
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<String> chunkConsumer);
|
||||
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer);
|
||||
|
||||
protected CompletionFinishReason mapFinishReason(final String finishReason) {
|
||||
return switch (finishReason) {
|
||||
case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
}
|
||||
|
||||
protected Retry createRetrySpec() {
|
||||
return Retry.backoff(99, Duration.ofSeconds(1))
|
||||
.maxBackoff(Duration.ofSeconds(10))
|
||||
.jitter(0.25d)
|
||||
.filter(throwable -> {
|
||||
return throwable instanceof java.io.IOException || throwable instanceof java.net.ConnectException || throwable instanceof java.net.http.HttpTimeoutException;
|
||||
})
|
||||
.doAfterRetry(rs -> LOGGER.warn("Retrying failed request: " + rs.failure().getClass().getName() + " " + rs.failure().getMessage()));
|
||||
}
|
||||
|
||||
protected void configureStreamOptions(final boolean enableStreaming, final Consumer<StreamOptions> streamOptionsConsumer) {
|
||||
if (enableStreaming) {
|
||||
final var streamOptions = new StreamOptions();
|
||||
streamOptions.setIncludeUsage(true);
|
||||
streamOptionsConsumer.accept(streamOptions);
|
||||
}
|
||||
}
|
||||
|
||||
protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) {
|
||||
int inTokens = 0;
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public abstract class AbstractCompletionRequest {
|
||||
|
||||
protected String model;
|
||||
|
||||
protected Integer maxTokens;
|
||||
|
||||
protected Double temperature;
|
||||
|
||||
protected Double topP;
|
||||
|
||||
protected Boolean stream;
|
||||
|
||||
protected StreamOptions streamOptions;
|
||||
|
||||
@JsonProperty("model")
|
||||
public String getModel() {
|
||||
return this.model;
|
||||
}
|
||||
|
||||
public void setModel(final String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
@JsonProperty("max_tokens")
|
||||
public Integer getMaxTokens() {
|
||||
return this.maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokens(final Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("temperature")
|
||||
public Double getTemperature() {
|
||||
return this.temperature;
|
||||
}
|
||||
|
||||
public void setTemperature(final Double temperature) {
|
||||
this.temperature = temperature;
|
||||
}
|
||||
|
||||
@JsonProperty("top_p")
|
||||
public Double getTopP() {
|
||||
return this.topP;
|
||||
}
|
||||
|
||||
public void setTopP(final Double topP) {
|
||||
this.topP = topP;
|
||||
}
|
||||
|
||||
@JsonProperty("stream")
|
||||
public Boolean isStream() {
|
||||
return this.stream;
|
||||
}
|
||||
|
||||
public void setStream(final Boolean stream) {
|
||||
this.stream = stream;
|
||||
}
|
||||
|
||||
@JsonProperty("stream_options")
|
||||
public StreamOptions getStreamOptions() {
|
||||
return streamOptions;
|
||||
}
|
||||
|
||||
public void setStreamOptions(final StreamOptions streamOptions) {
|
||||
this.streamOptions = streamOptions;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public abstract class AbstractCompletionResponse {
|
||||
|
||||
protected String id;
|
||||
|
||||
protected String object;
|
||||
|
||||
protected Integer created;
|
||||
|
||||
protected Usage usage;
|
||||
|
||||
@JsonProperty("id")
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public void setId(final String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@JsonProperty("object")
|
||||
public String getObject() {
|
||||
return this.object;
|
||||
}
|
||||
|
||||
public void setObject(final String object) {
|
||||
this.object = object;
|
||||
}
|
||||
|
||||
@JsonProperty("created")
|
||||
public Integer getCreated() {
|
||||
return this.created;
|
||||
}
|
||||
|
||||
public void setCreated(final Integer created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
@JsonProperty("usage")
|
||||
public Usage getUsage() {
|
||||
return usage;
|
||||
}
|
||||
|
||||
public void setUsage(final Usage usage) {
|
||||
this.usage = usage;
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public class CompletionStreamOptions {
|
||||
public class StreamOptions {
|
||||
|
||||
private boolean includeUsage;
|
||||
protected boolean includeUsage;
|
||||
|
||||
@JsonProperty("include_usage")
|
||||
public boolean isIncludeUsage() {
|
||||
@@ -1,6 +1,5 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
@@ -16,17 +15,15 @@ import org.springframework.stereotype.Component;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
|
||||
import de.extio.lmlib.client.oai.completion.FinishReasons;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
@@ -43,7 +40,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<String> chunkConsumer) {
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
|
||||
final var chat = createChats(conversation, modelProfile);
|
||||
|
||||
final var request = new ChatCompletionRequest();
|
||||
@@ -53,13 +50,37 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
if (chunkConsumer == null) {
|
||||
request.setStream(false);
|
||||
request.setStream(chunkConsumer != null);
|
||||
if (chunkConsumer != null) {
|
||||
this.configureStreamOptions(true, request::setStreamOptions);
|
||||
}
|
||||
else {
|
||||
request.setStream(true);
|
||||
request.setStreamOptions(new ChatCompletionStreamOptions());
|
||||
request.getStreamOptions().setIncludeUsage(true);
|
||||
if (modelProfile.reasoningEffort() != null && !modelProfile.reasoningEffort().isBlank()) {
|
||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
||||
try {
|
||||
request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(modelProfile.reasoningEffort()));
|
||||
}
|
||||
catch (final IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("Invalid reasoning effort in the model profile: " + modelProfile.reasoningEffort(), e);
|
||||
}
|
||||
}
|
||||
if (modelProfile.reasoningSummaryDetails() != null && !modelProfile.reasoningSummaryDetails().isBlank()) {
|
||||
if (request.getReasoning() == null) {
|
||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
||||
}
|
||||
try {
|
||||
request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(modelProfile.reasoningSummaryDetails()).name());
|
||||
}
|
||||
catch (final IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("Invalid reasoning summary details in the model profile: " + modelProfile.reasoningSummaryDetails(), e);
|
||||
}
|
||||
}
|
||||
|
||||
String requestBody;
|
||||
try {
|
||||
requestBody = this.objectMapper.writeValueAsString(request);
|
||||
}
|
||||
catch (final JsonProcessingException e) {
|
||||
throw new IllegalStateException("Cannot serialize request body", e);
|
||||
}
|
||||
|
||||
LOGGER.debug("Requesting chat completion at {}", modelProfile.url());
|
||||
@@ -70,19 +91,25 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.bodyValue(requestBody)
|
||||
.exchangeToMono(clientResponse -> {
|
||||
if (clientResponse.statusCode().isError()) {
|
||||
return clientResponse.bodyToMono(String.class)
|
||||
.flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody)));
|
||||
}
|
||||
|
||||
if (chunkConsumer == null) {
|
||||
return clientResponse.bodyToMono(ChatCompletionResponse.class);
|
||||
}
|
||||
else {
|
||||
final var chatCompletionResponse = new ChatCompletionResponse();
|
||||
chatCompletionResponse.setChoices(List.of(new ChatChoice()));
|
||||
final StringBuilder contenStringBuilder = new StringBuilder();
|
||||
final StringBuilder contentStringBuilder = new StringBuilder();
|
||||
final StringBuilder reasoningStringBuilder = new StringBuilder();
|
||||
return clientResponse.bodyToFlux(String.class)
|
||||
.doOnNext(buffer -> {
|
||||
try {
|
||||
if (! "[DONE]".equals(buffer)) {
|
||||
if (!"[DONE]".equals(buffer)) {
|
||||
final var streamResponse = this.objectMapper.readValue(buffer, ChatCompletionResponse.class);
|
||||
|
||||
if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) {
|
||||
@@ -101,9 +128,20 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
if (streamResponse.getChoices().getFirst().getFinishReason() != null) {
|
||||
chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason());
|
||||
}
|
||||
if (streamResponse.getChoices().getFirst().getMessage() != null && streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) {
|
||||
contenStringBuilder.append(streamResponse.getChoices().getFirst().getMessage().getContent());
|
||||
chunkConsumer.accept(streamResponse.getChoices().getFirst().getMessage().getContent());
|
||||
if (streamResponse.getChoices().getFirst().getMessage() != null) {
|
||||
String content = null;
|
||||
String reasoning = null;
|
||||
if (streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) {
|
||||
content = streamResponse.getChoices().getFirst().getMessage().getContent();
|
||||
contentStringBuilder.append(content);
|
||||
}
|
||||
if (streamResponse.getChoices().getFirst().getMessage().getReasoningContent() != null && !streamResponse.getChoices().getFirst().getMessage().getReasoningContent().isEmpty()) {
|
||||
reasoning = streamResponse.getChoices().getFirst().getMessage().getReasoningContent();
|
||||
reasoningStringBuilder.append(reasoning);
|
||||
}
|
||||
if (content != null || reasoning != null) {
|
||||
chunkConsumer.accept(new Chunk(content, reasoning));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -113,27 +151,23 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
}
|
||||
})
|
||||
.then(Mono.<ChatCompletionResponse> fromSupplier(() -> {
|
||||
chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contenStringBuilder.toString()));
|
||||
chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contentStringBuilder.toString(), reasoningStringBuilder.toString()));
|
||||
return chatCompletionResponse;
|
||||
}));
|
||||
}
|
||||
})
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.retryWhen(this.createRetrySpec())
|
||||
.block();
|
||||
|
||||
if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
|
||||
final var choice = response.getChoices().getFirst();
|
||||
final var content = response.getChoices().getFirst().getMessage().getContent();
|
||||
final var finishReason = switch (choice.getFinishReason()) {
|
||||
case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
final var reasoning = response.getChoices().getFirst().getMessage().getReasoningContent();
|
||||
final var finishReason = this.mapFinishReason(choice.getFinishReason());
|
||||
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content);
|
||||
return new Completion(content, finishReason, statistics);
|
||||
return new Completion(content, reasoning, finishReason, statistics);
|
||||
}
|
||||
|
||||
private List<ChatMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
@@ -165,7 +199,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
|
||||
if (first && userTurn != null) {
|
||||
if (systemTurn != null) {
|
||||
chat.add(new ChatMessage("system", systemTurn.text()));
|
||||
chat.add(new ChatMessage("system", systemTurn.text(), null));
|
||||
}
|
||||
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
|
||||
break;
|
||||
@@ -202,7 +236,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
trimmed = true;
|
||||
}
|
||||
|
||||
chat.add(new ChatMessage(role, prompt));
|
||||
chat.add(new ChatMessage(role, prompt, null));
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
@@ -2,36 +2,17 @@ package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class ChatCompletionRequest {
|
||||
|
||||
private String model;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;
|
||||
|
||||
final class ChatCompletionRequest extends AbstractCompletionRequest {
|
||||
|
||||
private List<ChatMessage> messages;
|
||||
|
||||
private Integer maxTokens;
|
||||
|
||||
private Integer maxCompletionTokens;
|
||||
|
||||
private Double temperature;
|
||||
|
||||
private Double topP;
|
||||
|
||||
private Boolean stream;
|
||||
|
||||
private ChatCompletionStreamOptions streamOptions;
|
||||
|
||||
@JsonProperty("model")
|
||||
public String getModel() {
|
||||
return this.model;
|
||||
}
|
||||
|
||||
public void setModel(final String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
private ChatCompletionsRequestReasoning reasoning;
|
||||
|
||||
@JsonProperty("messages")
|
||||
public List<ChatMessage> getMessages() {
|
||||
@@ -42,15 +23,6 @@ final class ChatCompletionRequest {
|
||||
this.messages = messages;
|
||||
}
|
||||
|
||||
@JsonProperty("max_tokens")
|
||||
public Integer getMaxTokens() {
|
||||
return this.maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokens(final Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("max_completion_tokens")
|
||||
public Integer getMaxCompletionTokens() {
|
||||
return this.maxCompletionTokens;
|
||||
@@ -60,40 +32,47 @@ final class ChatCompletionRequest {
|
||||
this.maxCompletionTokens = maxCompletionTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("temperature")
|
||||
public Double getTemperature() {
|
||||
return this.temperature;
|
||||
@JsonProperty("reasoning")
|
||||
public ChatCompletionsRequestReasoning getReasoning() {
|
||||
return this.reasoning;
|
||||
}
|
||||
|
||||
public void setTemperature(final Double temperature) {
|
||||
this.temperature = temperature;
|
||||
|
||||
public void setReasoning(final ChatCompletionsRequestReasoning reasoning) {
|
||||
this.reasoning = reasoning;
|
||||
}
|
||||
|
||||
@JsonProperty("top_p")
|
||||
public Double getTopP() {
|
||||
return this.topP;
|
||||
|
||||
final static class ChatCompletionsRequestReasoning {
|
||||
|
||||
private ChatCompletionsRequestReasoningEffort effort;
|
||||
|
||||
private String summary;
|
||||
|
||||
@JsonProperty("effort")
|
||||
public ChatCompletionsRequestReasoningEffort getEffort() {
|
||||
return this.effort;
|
||||
}
|
||||
|
||||
public void setEffort(final ChatCompletionsRequestReasoningEffort effort) {
|
||||
this.effort = effort;
|
||||
}
|
||||
|
||||
@JsonProperty("summary")
|
||||
public String getSummary() {
|
||||
return this.summary;
|
||||
}
|
||||
|
||||
public void setSummary(final String summary) {
|
||||
this.summary = summary;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void setTopP(final Double topP) {
|
||||
this.topP = topP;
|
||||
|
||||
static enum ChatCompletionsRequestReasoningEffort {
|
||||
minimal, low, medium, high
|
||||
}
|
||||
|
||||
@JsonProperty("stream")
|
||||
public Boolean isStream() {
|
||||
return this.stream;
|
||||
|
||||
static enum ChatCompletionsRequestReasoningSummaryDetails {
|
||||
auto, concise, detailed
|
||||
}
|
||||
|
||||
public void setStream(final Boolean stream) {
|
||||
this.stream = stream;
|
||||
}
|
||||
|
||||
@JsonProperty("stream_options")
|
||||
public ChatCompletionStreamOptions getStreamOptions() {
|
||||
return streamOptions;
|
||||
}
|
||||
|
||||
public void setStreamOptions(ChatCompletionStreamOptions streamOptions) {
|
||||
this.streamOptions = streamOptions;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -4,47 +4,12 @@ import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import de.extio.lmlib.client.oai.completion.Usage;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse;
|
||||
|
||||
final class ChatCompletionResponse {
|
||||
|
||||
private String id;
|
||||
|
||||
private String object;
|
||||
|
||||
private Integer created;
|
||||
final class ChatCompletionResponse extends AbstractCompletionResponse {
|
||||
|
||||
private List<ChatChoice> choices;
|
||||
|
||||
private Usage usage;
|
||||
|
||||
@JsonProperty("id")
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public void setId(final String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@JsonProperty("object")
|
||||
public String getObject() {
|
||||
return this.object;
|
||||
}
|
||||
|
||||
public void setObject(final String object) {
|
||||
this.object = object;
|
||||
}
|
||||
|
||||
@JsonProperty("created")
|
||||
public Integer getCreated() {
|
||||
return this.created;
|
||||
}
|
||||
|
||||
public void setCreated(final Integer created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
@JsonProperty("choices")
|
||||
public List<ChatChoice> getChoices() {
|
||||
return this.choices;
|
||||
@@ -53,15 +18,6 @@ final class ChatCompletionResponse {
|
||||
public void setChoices(final List<ChatChoice> choices) {
|
||||
this.choices = choices;
|
||||
}
|
||||
|
||||
@JsonProperty("usage")
|
||||
public Usage getUsage() {
|
||||
return usage;
|
||||
}
|
||||
|
||||
public void setUsage(final Usage usage) {
|
||||
this.usage = usage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public class ChatCompletionStreamOptions {
|
||||
|
||||
private boolean includeUsage;
|
||||
|
||||
@JsonProperty("include_usage")
|
||||
public boolean isIncludeUsage() {
|
||||
return includeUsage;
|
||||
}
|
||||
|
||||
public void setIncludeUsage(final boolean includeUsage) {
|
||||
this.includeUsage = includeUsage;
|
||||
}
|
||||
}
|
||||
@@ -8,14 +8,17 @@ final class ChatMessage {
|
||||
|
||||
private String content;
|
||||
|
||||
private String reasoningContent;
|
||||
|
||||
public ChatMessage() {
|
||||
}
|
||||
|
||||
public ChatMessage(final String role, final String content) {
|
||||
public ChatMessage(final String role, final String content, final String reasoningContent) {
|
||||
this.role = role;
|
||||
this.content = content;
|
||||
this.reasoningContent = reasoningContent;
|
||||
}
|
||||
|
||||
|
||||
@JsonProperty("role")
|
||||
public String getRole() {
|
||||
return role;
|
||||
@@ -34,4 +37,12 @@ final class ChatMessage {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
@JsonProperty("reasoning_content")
|
||||
public String getReasoningContent() {
|
||||
return reasoningContent;
|
||||
}
|
||||
|
||||
public void setReasoningContent(final String reasoningContent) {
|
||||
this.reasoningContent = reasoningContent;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,31 +2,12 @@ package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class CompletionRequest {
|
||||
|
||||
private String model;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;
|
||||
|
||||
final class CompletionRequest extends AbstractCompletionRequest {
|
||||
|
||||
private String prompt;
|
||||
|
||||
private Integer maxTokens;
|
||||
|
||||
private Double temperature;
|
||||
|
||||
private Double topP;
|
||||
|
||||
private Boolean stream;
|
||||
|
||||
private CompletionStreamOptions streamOptions;
|
||||
|
||||
@JsonProperty("model")
|
||||
public String getModel() {
|
||||
return this.model;
|
||||
}
|
||||
|
||||
public void setModel(final String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
@JsonProperty("prompt")
|
||||
public String getPrompt() {
|
||||
return this.prompt;
|
||||
@@ -35,49 +16,4 @@ final class CompletionRequest {
|
||||
public void setPrompt(final String prompt) {
|
||||
this.prompt = prompt;
|
||||
}
|
||||
|
||||
@JsonProperty("max_tokens")
|
||||
public Integer getMaxTokens() {
|
||||
return this.maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokens(final Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("temperature")
|
||||
public Double getTemperature() {
|
||||
return this.temperature;
|
||||
}
|
||||
|
||||
public void setTemperature(final Double temperature) {
|
||||
this.temperature = temperature;
|
||||
}
|
||||
|
||||
@JsonProperty("top_p")
|
||||
public Double getTopP() {
|
||||
return this.topP;
|
||||
}
|
||||
|
||||
public void setTopP(final Double topP) {
|
||||
this.topP = topP;
|
||||
}
|
||||
|
||||
@JsonProperty("stream")
|
||||
public Boolean isStream() {
|
||||
return this.stream;
|
||||
}
|
||||
|
||||
public void setStream(final Boolean stream) {
|
||||
this.stream = stream;
|
||||
}
|
||||
|
||||
@JsonProperty("stream_options")
|
||||
public CompletionStreamOptions getStreamOptions() {
|
||||
return streamOptions;
|
||||
}
|
||||
|
||||
public void setStreamOptions(final CompletionStreamOptions streamOptions) {
|
||||
this.streamOptions = streamOptions;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,50 +4,15 @@ import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import de.extio.lmlib.client.oai.completion.Usage;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse;
|
||||
|
||||
final class CompletionResponse {
|
||||
|
||||
private String id;
|
||||
|
||||
private String object;
|
||||
|
||||
private Integer created;
|
||||
final class CompletionResponse extends AbstractCompletionResponse {
|
||||
|
||||
private List<Choice> choices;
|
||||
|
||||
private String content;
|
||||
|
||||
private Boolean isStoppedEos;
|
||||
|
||||
private Usage usage;
|
||||
|
||||
@JsonProperty("id")
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public void setId(final String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@JsonProperty("object")
|
||||
public String getObject() {
|
||||
return this.object;
|
||||
}
|
||||
|
||||
public void setObject(final String object) {
|
||||
this.object = object;
|
||||
}
|
||||
|
||||
@JsonProperty("created")
|
||||
public Integer getCreated() {
|
||||
return this.created;
|
||||
}
|
||||
|
||||
public void setCreated(final Integer created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
@JsonProperty("choices")
|
||||
public List<Choice> getChoices() {
|
||||
@@ -75,15 +40,6 @@ final class CompletionResponse {
|
||||
public void setStoppedEos(final Boolean isStoppedEos) {
|
||||
this.isStoppedEos = isStoppedEos;
|
||||
}
|
||||
|
||||
@JsonProperty("usage")
|
||||
public Usage getUsage() {
|
||||
return usage;
|
||||
}
|
||||
|
||||
public void setUsage(final Usage usage) {
|
||||
this.usage = usage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
@@ -13,22 +12,20 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.azure.ai.inference.models.ChatChoice;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
|
||||
import de.extio.lmlib.client.oai.completion.FinishReasons;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.prompt.PromptStrategy;
|
||||
import de.extio.lmlib.prompt.PromptStrategyFactory;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class TextCompletionClient extends AbstractCompletionClient {
|
||||
@@ -44,7 +41,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<String> chunkConsumer) {
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
|
||||
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
|
||||
if (promptStrategy == null) {
|
||||
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
|
||||
@@ -58,13 +55,17 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
if (chunkConsumer == null) {
|
||||
request.setStream(false);
|
||||
request.setStream(chunkConsumer != null);
|
||||
if (chunkConsumer != null) {
|
||||
this.configureStreamOptions(true, request::setStreamOptions);
|
||||
}
|
||||
else {
|
||||
request.setStream(true);
|
||||
request.setStreamOptions(new CompletionStreamOptions());
|
||||
request.getStreamOptions().setIncludeUsage(true);
|
||||
|
||||
String requestBody;
|
||||
try {
|
||||
requestBody = this.objectMapper.writeValueAsString(request);
|
||||
}
|
||||
catch (final JsonProcessingException e) {
|
||||
throw new IllegalStateException("Cannot serialize request body", e);
|
||||
}
|
||||
|
||||
LOGGER.debug("Requesting completion at {}", modelProfile.url());
|
||||
@@ -74,8 +75,13 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.bodyValue(requestBody)
|
||||
.exchangeToMono(clientResponse -> {
|
||||
if (clientResponse.statusCode().isError()) {
|
||||
return clientResponse.bodyToMono(String.class)
|
||||
.flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody)));
|
||||
}
|
||||
|
||||
if (chunkConsumer == null) {
|
||||
return clientResponse.bodyToMono(CompletionResponse.class);
|
||||
}
|
||||
@@ -116,7 +122,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
}
|
||||
if (streamResponse.getChoices().getFirst().getText() != null && !streamResponse.getChoices().getFirst().getText().isEmpty()) {
|
||||
contentStringBuilder.append(streamResponse.getChoices().getFirst().getText());
|
||||
chunkConsumer.accept(streamResponse.getChoices().getFirst().getText());
|
||||
chunkConsumer.accept(new Chunk(streamResponse.getChoices().getFirst().getText(), null));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -131,30 +137,33 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
}));
|
||||
}
|
||||
})
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.retryWhen(this.createRetrySpec())
|
||||
.block();
|
||||
|
||||
String content = null;
|
||||
String reasoning = null;
|
||||
CompletionFinishReason finishReason = null;
|
||||
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
|
||||
final var choice = response.getChoices().getFirst();
|
||||
if (response.getChoices().getFirst().getText() != null) {
|
||||
content = promptStrategy.removeEOT(choice.getText());
|
||||
content = promptStrategy.getResponse(choice.getText());
|
||||
reasoning = promptStrategy.getReasoning(choice.getText());
|
||||
}
|
||||
if (response.getChoices().getFirst().getFinishReason() != null) {
|
||||
finishReason = switch (choice.getFinishReason()) {
|
||||
case FinishReasons.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case FinishReasons.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
finishReason = this.mapFinishReason(choice.getFinishReason());
|
||||
}
|
||||
}
|
||||
if (content == null && response.getContent() != null) {
|
||||
content = promptStrategy.removeEOT(response.getContent());
|
||||
content = promptStrategy.getResponse(response.getContent());
|
||||
reasoning = promptStrategy.getReasoning(response.getContent());
|
||||
}
|
||||
if (content == null) {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
content = promptStrategy.removeEOT(content);
|
||||
if (reasoning != null) {
|
||||
reasoning = promptStrategy.removeEOT(reasoning);
|
||||
}
|
||||
if (finishReason == null) {
|
||||
if (response.isStoppedEos() != null) {
|
||||
finishReason = Boolean.TRUE.equals(response.isStoppedEos()) ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
@@ -165,7 +174,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
}
|
||||
|
||||
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent());
|
||||
return new Completion(content, finishReason, statistics);
|
||||
return new Completion(content, reasoning, finishReason, statistics);
|
||||
}
|
||||
|
||||
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
|
||||
Reference in New Issue
Block a user