From 4ebb8640c7a5adb9c531b6fee528b428d7de4d58 Mon Sep 17 00:00:00 2001 From: extio-de Date: Fri, 19 Dec 2025 17:58:56 +0100 Subject: [PATCH] Version 2.0 Java 25, Spring Boot 4, Use RestClient with JDK client backend --- README.md | 23 ++- pom.xml | 12 +- .../java/de/extio/lmlib/agent/BaseAgent.java | 6 +- .../JsonAgentResponseHandler.java | 14 +- .../lmlib/client/RestClientConfiguration.java | 139 ++++++++++++++++++ .../lmlib/client/WebClientConfiguration.java | 40 ----- .../java/de/extio/lmlib/client/oai/Model.java | 2 +- .../lmlib/client/oai/ModelNameLoader.java | 13 +- .../lmlib/client/oai/ModelNameSupplier.java | 10 +- .../lmlib/client/oai/ModelsResponse.java | 2 +- .../completion/AbstractCompletionClient.java | 30 ++-- .../AbstractCompletionResponse.java | 2 +- .../client/oai/completion/StreamOptions.java | 2 +- .../lmlib/client/oai/completion/Timings.java | 2 +- .../lmlib/client/oai/completion/Usage.java | 2 +- .../oai/completion/chat/ChatChoice.java | 2 +- .../completion/chat/ChatCompletionClient.java | 132 +++++++++-------- .../chat/ChatCompletionRequest.java | 2 +- .../chat/ChatCompletionResponse.java | 2 +- .../oai/completion/chat/ChatMessage.java | 2 +- .../client/oai/completion/text/Choice.java | 2 +- .../completion/text/CompletionRequest.java | 2 +- .../completion/text/CompletionResponse.java | 2 +- .../completion/text/TextCompletionClient.java | 137 +++++++++-------- .../lmlib/token/LlamaServerTokenizer.java | 35 ++--- .../token/TokenizerAutoConfiguration.java | 6 +- .../java/de/extio/lmlib/agent/AgentTest.java | 2 +- .../lmlib/client/cached/CachedClientTest.java | 18 ++- .../de/extio/lmlib/token/TokenizerTest.java | 8 +- .../de/extio/lmlib/util/TextUtilsTest.java | 2 - .../resources/local-completion.properties | 2 +- 31 files changed, 392 insertions(+), 263 deletions(-) create mode 100644 src/main/java/de/extio/lmlib/client/RestClientConfiguration.java delete mode 100755 src/main/java/de/extio/lmlib/client/WebClientConfiguration.java diff --git a/README.md b/README.md index 20ad1f5..06093b3 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ The libraries's key features include: - Modular architecture for easy extension and customization - Spring integration for dependency injection and configuration +# Requirements + +- **Java 25** or higher +- **Spring Boot 4.0** or higher + # Setup To keep this library lean and compatible with different dependency trees, all dependencies must be provided by the application. @@ -24,9 +29,25 @@ To keep this library lean and compatible with different dependency trees, all de org.springframework.boot - spring-boot-starter-webflux + spring-boot-starter-restclient +## Configuration + +The library can be configured using the following application properties: + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `lmlib.client.proxy.enabled` | Boolean | `false` | Enable HTTP proxy for LLM requests. | +| `lmlib.client.proxy.host` | String | | HTTP proxy host. | +| `lmlib.client.proxy.port` | Integer | `0` | HTTP proxy port. | +| `lmlib.client.proxy.user` | String | | HTTP proxy username (optional). | +| `lmlib.client.proxy.password` | String | | HTTP proxy password (optional). | +| `lmlib.client.tls.verification.disabled` | Boolean | `false` | Disable TLS certificate verification (not recommended for production). | +| `lmlib.client.retry.max-attempts` | Integer | `5` | Maximum number of retry attempts for failed requests. | +| `lmlib.client.retry.backoff-interval-min` | Long | `200` | Minimum backoff interval in milliseconds between retry attempts. | +| `lmlib.client.retry.backoff-interval-max` | Long | `2500` | Maximum backoff interval in milliseconds between retry attempts. | + ## Optional dependencies Some services benefit from Spring cache framework, e.g. the model profile service: diff --git a/pom.xml b/pom.xml index 3f87870..44dca56 100755 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,13 @@ lm-lib de.extio lm-lib - 1.10.0 + 2.0 - 21 - 21 - 21 - 3.5.6 + 25 + 25 + 25 + 4.0.1 1.1.0 @@ -48,7 +48,7 @@ org.springframework.boot - spring-boot-starter-webflux + spring-boot-starter-restclient provided diff --git a/src/main/java/de/extio/lmlib/agent/BaseAgent.java b/src/main/java/de/extio/lmlib/agent/BaseAgent.java index 3441ea6..4f1bdfe 100644 --- a/src/main/java/de/extio/lmlib/agent/BaseAgent.java +++ b/src/main/java/de/extio/lmlib/agent/BaseAgent.java @@ -12,8 +12,8 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; import de.extio.lmlib.agent.responsehandler.AgentResponseHandler; import de.extio.lmlib.agent.responsehandler.StreamedAgentResponseHandler; @@ -239,7 +239,7 @@ public interface BaseAgent { try { text = text.replace(k, objectMapper.writeValueAsString(entry.getValue())); } - catch (final JsonProcessingException e) { + catch (final JacksonException e) { LOGGER.warn("Cannot write json", e); } } diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java index 7883722..7177424 100755 --- a/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java @@ -10,11 +10,11 @@ import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.json.JsonReadFeature; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.json.JsonMapper; +import tools.jackson.core.json.JsonReadFeature; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; import de.extio.lmlib.agent.AgentContext; import de.extio.lmlib.client.Completion; @@ -52,7 +52,7 @@ public class JsonAgentResponseHandler implements AgentResponseHandler { .enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS) .enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS) .enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS) - .enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES) + .enable(JsonReadFeature.ALLOW_UNQUOTED_PROPERTY_NAMES) .build(); } @@ -94,7 +94,7 @@ public class JsonAgentResponseHandler implements AgentResponseHandler { private void parseEntry(final LinkedHashMap> items, final Entry entry) { if (entry.getValue().isArray()) { - entry.getValue().elements().forEachRemaining(nestedArrayValue -> { + entry.getValue().values().forEach(nestedArrayValue -> { if (nestedArrayValue.properties().isEmpty()) { items.computeIfAbsent(this.prefixFields + entry.getKey(), k -> new ArrayList<>()).add(nestedArrayValue.asText()); } diff --git a/src/main/java/de/extio/lmlib/client/RestClientConfiguration.java b/src/main/java/de/extio/lmlib/client/RestClientConfiguration.java new file mode 100644 index 0000000..714209a --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/RestClientConfiguration.java @@ -0,0 +1,139 @@ +package de.extio.lmlib.client; + +import java.net.Authenticator; +import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; +import java.net.ProxySelector; +import java.net.http.HttpClient; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.time.Duration; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.util.backoff.BackOff; +import org.springframework.util.backoff.BackOffExecution; +import org.springframework.util.backoff.ExponentialBackOff; +import org.springframework.web.client.RestClient; + +@Configuration +public class RestClientConfiguration { + + private static final Logger LOGGER = LoggerFactory.getLogger(RestClientConfiguration.class); + + private static final int TIMEOUT = 300000; + + private static final int CONNECT_TIMEOUT = 10000; + + @Bean + @Qualifier("lmLibRestClientBuilder") + RestClient.Builder lmLibRestClientBuilder( + @Value("${lmlib.client.proxy.enabled:false}") final boolean proxyEnabled, + @Value("${lmlib.client.proxy.host:}") final String proxyHost, + @Value("${lmlib.client.proxy.port:0}") final int proxyPort, + @Value("${lmlib.client.proxy.user:}") final String proxyUser, + @Value("${lmlib.client.proxy.password:}") final String proxyPassword, + @Value("${lmlib.client.tls.verification.disabled:false}") final boolean tlsVerificationDisabled, + @Value("${lmlib.client.retry.max-attempts:5}") final int maxAttempts, + @Value("${lmlib.client.retry.backoff-interval-min:125}") final long backoffIntervalMin, + @Value("${lmlib.client.retry.backoff-interval-max:2500}") final long backoffIntervalMax) { + + final HttpClient.Builder httpClientBuilder = HttpClient.newBuilder() + .connectTimeout(Duration.ofMillis(CONNECT_TIMEOUT)); + + if (proxyEnabled && proxyHost != null && !proxyHost.isBlank()) { + httpClientBuilder.proxy(ProxySelector.of(new InetSocketAddress(proxyHost, proxyPort))); + + if (proxyUser != null && !proxyUser.isBlank()) { + httpClientBuilder.authenticator(new Authenticator() { + + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUser, proxyPassword.toCharArray()); + } + }); + } + } + + if (tlsVerificationDisabled) { + try { + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new TrustManager[] { new X509TrustManager() { + + @Override + public void checkClientTrusted(final X509Certificate[] chain, final String authType) { + } + + @Override + public void checkServerTrusted(final X509Certificate[] chain, final String authType) { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } }, new SecureRandom()); + httpClientBuilder.sslContext(sslContext); + } + catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new RuntimeException("Failed to disable TLS verification", e); + } + } + + final JdkClientHttpRequestFactory requestFactory = new JdkClientHttpRequestFactory(httpClientBuilder.build()); + requestFactory.setReadTimeout(Duration.ofMillis(TIMEOUT)); + + return RestClient.builder() + .requestFactory(requestFactory) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) + .requestInterceptor(createRetryInterceptor(maxAttempts, backoffIntervalMin, backoffIntervalMax)); + } + + private ClientHttpRequestInterceptor createRetryInterceptor(final int maxAttempts, final long backoffIntervalMin, final long backoffIntervalMax) { + return (request, body, execution) -> { + final BackOff backOff = new ExponentialBackOff(); + ((ExponentialBackOff) backOff).setInitialInterval(backoffIntervalMin); + ((ExponentialBackOff) backOff).setMaxInterval(backoffIntervalMax); + ((ExponentialBackOff) backOff).setMaxElapsedTime(maxAttempts * backoffIntervalMax); + ((ExponentialBackOff) backOff).setMultiplier(2.0); + ((ExponentialBackOff) backOff).setJitter(backoffIntervalMin / 2); + final BackOffExecution backOffExecution = backOff.start(); + + while (true) { + try { + return execution.execute(request, body); + } + catch (final java.io.IOException e) { + final long waitTime = backOffExecution.nextBackOff(); + if (waitTime == BackOffExecution.STOP) { + throw e; + } + LOGGER.warn("Request failed, retrying in {} ms: {} {}", waitTime, e.getClass().getSimpleName(), e.getMessage() != null ? e.getMessage() : ""); + try { + Thread.sleep(waitTime); + } + catch (final InterruptedException ie) { + Thread.currentThread().interrupt(); + throw e; + } + } + } + }; + } + +} diff --git a/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java b/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java deleted file mode 100755 index 84835d5..0000000 --- a/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java +++ /dev/null @@ -1,40 +0,0 @@ -package de.extio.lmlib.client; - -import java.time.Duration; -import java.util.concurrent.TimeUnit; - -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; -import org.springframework.web.reactive.function.client.WebClient; - -import io.netty.channel.ChannelOption; -import io.netty.handler.timeout.ReadTimeoutHandler; -import io.netty.handler.timeout.WriteTimeoutHandler; -import reactor.netty.http.client.HttpClient; - -@Configuration -public class WebClientConfiguration { - - private static final int TIMEOUT = 300000; - - private static final int CONNECT_TIMEOUT = 10000; - - @Bean - @Qualifier("lmLibWebClientBuilder") - WebClient.Builder lmLibWebClientBuilder() { - final HttpClient httpClient = HttpClient.create() - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT) - .responseTimeout(Duration.ofMillis(TIMEOUT)) - .doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS))); - - return WebClient.builder() - .clientConnector(new ReactorClientHttpConnector(httpClient)) - .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) - .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE); - } - -} diff --git a/src/main/java/de/extio/lmlib/client/oai/Model.java b/src/main/java/de/extio/lmlib/client/oai/Model.java index c70619d..52b6f21 100755 --- a/src/main/java/de/extio/lmlib/client/oai/Model.java +++ b/src/main/java/de/extio/lmlib/client/oai/Model.java @@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai; import java.util.List; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; public final class Model { diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java b/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java index 9093042..dce4f5b 100755 --- a/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java +++ b/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java @@ -5,28 +5,27 @@ import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpMethod; -import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.client.RestClient; class ModelNameLoader { private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class); - private final WebClient webClient; + private final RestClient restClient; - public ModelNameLoader(final WebClient webClient) { - this.webClient = webClient; + public ModelNameLoader(final RestClient restClient) { + this.restClient = restClient; } public Optional loadModelName() { try { LOGGER.info("Querying model name"); - final var modelName = this.webClient + final var modelName = this.restClient .method(HttpMethod.GET) .uri(uriBuilder -> uriBuilder.path("/v1/models").build()) .retrieve() - .bodyToMono(ModelsResponse.class) - .block() + .body(ModelsResponse.class) .getData() .stream() .map(Model::getId) diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java b/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java index 35db717..9e43fe2 100755 --- a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java +++ b/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java @@ -6,7 +6,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.client.RestClient; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; @@ -15,8 +15,8 @@ import de.extio.lmlib.profile.ModelProfile.ModelProvider; public class ModelNameSupplier { @Autowired - @Qualifier("lmLibWebClientBuilder") - private WebClient.Builder webClientBuilder; + @Qualifier("lmLibRestClientBuilder") + private RestClient.Builder restClientBuilder; private final Map resolvedModelNames = new ConcurrentHashMap<>(); @@ -31,8 +31,8 @@ public class ModelNameSupplier { final var cacheKey = modelProfile.category() + "|" + modelProfile.url(); return this.resolvedModelNames.computeIfAbsent(cacheKey, key -> { - final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); - final var loader = new ModelNameLoader(webClient); + final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); + final var loader = new ModelNameLoader(restClient); return loader.loadModelName().orElse(""); }); } diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelsResponse.java b/src/main/java/de/extio/lmlib/client/oai/ModelsResponse.java index fb66ef9..ce0b3fd 100755 --- a/src/main/java/de/extio/lmlib/client/oai/ModelsResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/ModelsResponse.java @@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai; import java.util.List; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; public final class ModelsResponse { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index 048a656..373856e 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -13,13 +13,13 @@ import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; -import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.client.RestClient; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.json.JsonReadFeature; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.json.JsonMapper; +import tools.jackson.core.json.JsonReadFeature; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Client; @@ -32,15 +32,14 @@ import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.profile.ModelProfileService; import de.extio.lmlib.token.Tokenizer; -import reactor.util.retry.Retry; public abstract class AbstractCompletionClient implements Client, DisposableBean { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class); @Autowired - @Qualifier("lmLibWebClientBuilder") - protected WebClient.Builder webClientBuilder; + @Qualifier("lmLibRestClientBuilder") + protected RestClient.Builder restClientBuilder; @Autowired protected ModelNameSupplier modelNameSupplier; @@ -65,7 +64,8 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean .configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false) - .serializationInclusion(JsonInclude.Include.NON_NULL) + .changeDefaultPropertyInclusion(incl -> incl.withValueInclusion(JsonInclude.Include.NON_NULL)) + .changeDefaultPropertyInclusion(incl -> incl.withContentInclusion(JsonInclude.Include.NON_NULL)) .enable(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER) .enable(JsonReadFeature.ALLOW_JAVA_COMMENTS) .enable(JsonReadFeature.ALLOW_MISSING_VALUES) @@ -77,7 +77,7 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean .enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS) .enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS) .enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS) - .enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES) + .enable(JsonReadFeature.ALLOW_UNQUOTED_PROPERTY_NAMES) .build(); } @@ -122,16 +122,6 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean }; } - protected Retry createRetrySpec() { - return Retry.backoff(99, Duration.ofSeconds(1)) - .maxBackoff(Duration.ofSeconds(10)) - .jitter(0.25d) - .filter(throwable -> { - return throwable instanceof java.io.IOException || throwable instanceof java.net.ConnectException || throwable instanceof java.net.http.HttpTimeoutException; - }) - .doAfterRetry(rs -> LOGGER.warn("Retrying failed request: " + rs.failure().getClass().getName() + " " + rs.failure().getMessage())); - } - protected void configureStreamOptions(final boolean enableStreaming, final Consumer streamOptionsConsumer) { if (enableStreaming) { final var streamOptions = new StreamOptions(); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java index a9f9a13..abb7e1d 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; public abstract class AbstractCompletionResponse { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java b/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java index 465a641..985fe02 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; public class StreamOptions { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java b/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java index 0b3e004..98859c8 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; public final class Timings { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java index 27419fe..db191a3 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; public final class Usage { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java index aeae9f0..9774d87 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion.chat; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; class ChatChoice { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index c054f7f..05656b5 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -1,5 +1,8 @@ package de.extio.lmlib.client.oai.completion.chat; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.util.ArrayDeque; import java.util.ArrayList; @@ -12,8 +15,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClient; -import com.fasterxml.jackson.core.JsonProcessingException; +import tools.jackson.core.JacksonException; import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Completion; @@ -22,7 +26,6 @@ import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; -import reactor.core.publisher.Mono; @Component public class ChatCompletionClient extends AbstractCompletionClient { @@ -80,92 +83,99 @@ public class ChatCompletionClient extends AbstractCompletionClient { try { requestBody = this.objectMapper.writeValueAsString(request); } - catch (final JsonProcessingException e) { + catch (final JacksonException e) { throw new IllegalStateException("Cannot serialize request body", e); } LOGGER.debug("Requesting chat completion at {}", modelProfile.url()); final LocalDateTime start = LocalDateTime.now(); - final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); + final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); - var requestSpec = webClient + var requestSpec = restClient .method(HttpMethod.POST) - .uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build()) + .uri("/v1/chat/completions") .header("Content-Type", "application/json"); if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + final var response = requestSpec - .bodyValue(requestBody) - .exchangeToMono(clientResponse -> { - if (clientResponse.statusCode().isError()) { - return clientResponse.bodyToMono(String.class) - .flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody))); + .body(requestBody) + .exchange((clientRequest, clientResponse) -> { + if (clientResponse.getStatusCode().isError()) { + throw new IllegalStateException("Error response from server: " + clientResponse.getStatusCode()); } if (chunkConsumer == null) { - return clientResponse.bodyToMono(ChatCompletionResponse.class); + return this.objectMapper.readValue(clientResponse.getBody(), ChatCompletionResponse.class); } else { final var chatCompletionResponse = new ChatCompletionResponse(); chatCompletionResponse.setChoices(List.of(new ChatChoice())); final StringBuilder contentStringBuilder = new StringBuilder(); final StringBuilder reasoningStringBuilder = new StringBuilder(); - return clientResponse.bodyToFlux(String.class) - .doOnNext(buffer -> { - try { - if (!"[DONE]".equals(buffer)) { - final var streamResponse = this.objectMapper.readValue(buffer, ChatCompletionResponse.class); - - if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) { - chatCompletionResponse.setId(streamResponse.getId()); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(clientResponse.getBody(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("data: ")) { + line = line.substring(6).trim(); + } + if (line.isEmpty()) { + continue; + } + if ("[DONE]".equals(line)) { + break; + } + + try { + final var streamResponse = this.objectMapper.readValue(line, ChatCompletionResponse.class); + + if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) { + chatCompletionResponse.setId(streamResponse.getId()); + } + if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) { + chatCompletionResponse.setObject(streamResponse.getObject()); + } + if (streamResponse.getCreated() != null) { + chatCompletionResponse.setCreated(streamResponse.getCreated()); + } + if (streamResponse.getUsage() != null) { + chatCompletionResponse.setUsage(streamResponse.getUsage()); + } + if (streamResponse.getTimings() != null) { + chatCompletionResponse.setTimings(streamResponse.getTimings()); + } + if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) { + if (streamResponse.getChoices().getFirst().getFinishReason() != null) { + chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason()); + } + if (streamResponse.getChoices().getFirst().getMessage() != null) { + String content = null; + String reasoning = null; + if (streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) { + content = streamResponse.getChoices().getFirst().getMessage().getContent(); + contentStringBuilder.append(content); } - if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) { - chatCompletionResponse.setObject(streamResponse.getObject()); + if (streamResponse.getChoices().getFirst().getMessage().getReasoningContent() != null && !streamResponse.getChoices().getFirst().getMessage().getReasoningContent().isEmpty()) { + reasoning = streamResponse.getChoices().getFirst().getMessage().getReasoningContent(); + reasoningStringBuilder.append(reasoning); } - if (streamResponse.getCreated() != null) { - chatCompletionResponse.setCreated(streamResponse.getCreated()); - } - if (streamResponse.getUsage() != null) { - chatCompletionResponse.setUsage(streamResponse.getUsage()); - } - if (streamResponse.getTimings() != null) { - chatCompletionResponse.setTimings(streamResponse.getTimings()); - } - if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) { - if (streamResponse.getChoices().getFirst().getFinishReason() != null) { - chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason()); - } - if (streamResponse.getChoices().getFirst().getMessage() != null) { - String content = null; - String reasoning = null; - if (streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) { - content = streamResponse.getChoices().getFirst().getMessage().getContent(); - contentStringBuilder.append(content); - } - if (streamResponse.getChoices().getFirst().getMessage().getReasoningContent() != null && !streamResponse.getChoices().getFirst().getMessage().getReasoningContent().isEmpty()) { - reasoning = streamResponse.getChoices().getFirst().getMessage().getReasoningContent(); - reasoningStringBuilder.append(reasoning); - } - if (content != null || reasoning != null) { - chunkConsumer.accept(new Chunk(content, reasoning)); - } - } + if (content != null || reasoning != null) { + chunkConsumer.accept(new Chunk(content, reasoning)); } } } - catch (final JsonProcessingException e) { - LOGGER.warn("Streamed chunk is not parseable", e); - } - }) - .then(Mono. fromSupplier(() -> { - chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contentStringBuilder.toString(), reasoningStringBuilder.toString())); - return chatCompletionResponse; - })); + } + catch (final JacksonException e) { + LOGGER.warn("Streamed chunk is not parseable: {}", line, e); + } + } + } + chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contentStringBuilder.toString(), reasoningStringBuilder.toString())); + return chatCompletionResponse; } - }) - .retryWhen(this.createRetrySpec()) - .block(); + }); if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) { throw new IllegalStateException("No parseable response content from server"); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java index facb6af..613ef61 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java @@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion.chat; import java.util.List; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java index 34c7dbd..94f62d1 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java @@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion.chat; import java.util.List; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java index c33d684..81b3b4f 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion.chat; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; final class ChatMessage { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java index 777cd59..a9ba9c9 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion.text; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; final class Choice { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java index 4617e26..3b82cc7 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java @@ -1,6 +1,6 @@ package de.extio.lmlib.client.oai.completion.text; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java index e6acd28..e9e6a3a 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java @@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion.text; import java.util.List; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 35824fb..bd48ab1 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -1,5 +1,8 @@ package de.extio.lmlib.client.oai.completion.text; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.util.ArrayDeque; import java.util.Deque; @@ -11,8 +14,9 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClient; -import com.fasterxml.jackson.core.JsonProcessingException; +import tools.jackson.core.JacksonException; import de.extio.lmlib.client.Chunk; import de.extio.lmlib.client.Completion; @@ -24,7 +28,6 @@ import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.prompt.PromptStrategy; import de.extio.lmlib.prompt.PromptStrategyFactory; -import reactor.core.publisher.Mono; @Component public class TextCompletionClient extends AbstractCompletionClient { @@ -65,88 +68,94 @@ public class TextCompletionClient extends AbstractCompletionClient { try { requestBody = this.objectMapper.writeValueAsString(request); } - catch (final JsonProcessingException e) { + catch (final JacksonException e) { throw new IllegalStateException("Cannot serialize request body", e); } LOGGER.debug("Requesting completion at {}", modelProfile.url()); final LocalDateTime start = LocalDateTime.now(); - final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); - var requestSpec = webClient + final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); + var requestSpec = restClient .method(HttpMethod.POST) - .uri(uriBuilder -> uriBuilder.path("/v1/completions").build()) + .uri("/v1/completions") .header("Content-Type", "application/json"); if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } final var response = requestSpec - .bodyValue(requestBody) - .exchangeToMono(clientResponse -> { - if (clientResponse.statusCode().isError()) { - return clientResponse.bodyToMono(String.class) - .flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody))); + .body(requestBody) + .exchange((clientRequest, clientResponse) -> { + if (clientResponse.getStatusCode().isError()) { + throw new IllegalStateException("Error response from server: " + clientResponse.getStatusCode()); } if (chunkConsumer == null) { - return clientResponse.bodyToMono(CompletionResponse.class); + return this.objectMapper.readValue(clientResponse.getBody(), CompletionResponse.class); } else { - final var CompletionResponse = new CompletionResponse(); - CompletionResponse.setChoices(List.of(new Choice())); + final var completionResponse = new CompletionResponse(); + completionResponse.setChoices(List.of(new Choice())); final StringBuilder contentStringBuilder = new StringBuilder(); - return clientResponse.bodyToFlux(String.class) - .doOnNext(buffer -> { - try { - if (!"[DONE]".equals(buffer)) { - final var streamResponse = this.objectMapper.readValue(buffer, CompletionResponse.class); - - if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) { - CompletionResponse.setId(streamResponse.getId()); - } - if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) { - CompletionResponse.setObject(streamResponse.getObject()); - } - if (streamResponse.getCreated() != null) { - CompletionResponse.setCreated(streamResponse.getCreated()); - } - if (streamResponse.isStoppedEos() != null) { - CompletionResponse.setStoppedEos(streamResponse.isStoppedEos()); - } - if (streamResponse.getContent() != null && !streamResponse.getContent().isEmpty()) { - contentStringBuilder.append(streamResponse.getContent()); - } - if (streamResponse.getUsage() != null) { - CompletionResponse.setUsage(streamResponse.getUsage()); - } - if (streamResponse.getTimings() != null) { - CompletionResponse.setTimings(streamResponse.getTimings()); - } - if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) { - if (streamResponse.getChoices().getFirst().getFinishReason() != null) { - CompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason()); - } - if (streamResponse.getChoices().getFirst().getScore() != null) { - CompletionResponse.getChoices().getFirst().setScore(streamResponse.getChoices().getFirst().getScore()); - } - if (streamResponse.getChoices().getFirst().getText() != null && !streamResponse.getChoices().getFirst().getText().isEmpty()) { - contentStringBuilder.append(streamResponse.getChoices().getFirst().getText()); - chunkConsumer.accept(new Chunk(streamResponse.getChoices().getFirst().getText(), null)); - } - } + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(clientResponse.getBody(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("data: ")) { + line = line.substring(6).trim(); + } + if (line.isEmpty()) { + continue; + } + if ("[DONE]".equals(line)) { + break; + } + + try { + final var streamResponse = this.objectMapper.readValue(line, CompletionResponse.class); + + if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) { + completionResponse.setId(streamResponse.getId()); + } + if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) { + completionResponse.setObject(streamResponse.getObject()); + } + if (streamResponse.getCreated() != null) { + completionResponse.setCreated(streamResponse.getCreated()); + } + if (streamResponse.isStoppedEos() != null) { + completionResponse.setStoppedEos(streamResponse.isStoppedEos()); + } + if (streamResponse.getContent() != null && !streamResponse.getContent().isEmpty()) { + contentStringBuilder.append(streamResponse.getContent()); + } + if (streamResponse.getUsage() != null) { + completionResponse.setUsage(streamResponse.getUsage()); + } + if (streamResponse.getTimings() != null) { + completionResponse.setTimings(streamResponse.getTimings()); + } + if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) { + if (streamResponse.getChoices().getFirst().getFinishReason() != null) { + completionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason()); + } + if (streamResponse.getChoices().getFirst().getScore() != null) { + completionResponse.getChoices().getFirst().setScore(streamResponse.getChoices().getFirst().getScore()); + } + if (streamResponse.getChoices().getFirst().getText() != null && !streamResponse.getChoices().getFirst().getText().isEmpty()) { + contentStringBuilder.append(streamResponse.getChoices().getFirst().getText()); + chunkConsumer.accept(new Chunk(streamResponse.getChoices().getFirst().getText(), null)); } } - catch (final JsonProcessingException e) { - LOGGER.warn("Streamed chunk is not parseable", e); - } - }) - .then(Mono. fromSupplier(() -> { - CompletionResponse.getChoices().getFirst().setText(contentStringBuilder.toString()); - return CompletionResponse; - })); + } + catch (final JacksonException e) { + LOGGER.warn("Streamed chunk is not parseable: {}", line, e); + } + } + } + completionResponse.getChoices().getFirst().setText(contentStringBuilder.toString()); + return completionResponse; } - }) - .retryWhen(this.createRetrySpec()) - .block(); + }); String content = null; String reasoning = null; diff --git a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java index bfa69db..803dd00 100755 --- a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java +++ b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java @@ -6,36 +6,31 @@ import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpMethod; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClient.Builder; +import org.springframework.web.client.RestClient; import de.extio.lmlib.profile.ModelProfile; -import reactor.util.retry.Retry; - final class LlamaServerTokenizer implements Tokenizer { private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class); - private final WebClient.Builder webClientBuilder; + private final RestClient.Builder restClientBuilder; - public LlamaServerTokenizer(final Builder webClientBuilder) { - this.webClientBuilder = webClientBuilder; + public LlamaServerTokenizer(final RestClient.Builder restClientBuilder) { + this.restClientBuilder = restClientBuilder; } @Override public List tokenize(final String txt, final ModelProfile modelProfile) { LOGGER.debug("Requesting tokenizing"); final var request = new TokenizeRequest(txt); - final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); - final var response = webClient + final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); + final var response = restClient .method(HttpMethod.POST) - .uri(uriBuilder -> uriBuilder.path("/tokenize").build()) - .bodyValue(request) + .uri("/tokenize") + .body(request) .retrieve() - .bodyToMono(TokenizeResponse.class) - .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) - .block(); + .body(TokenizeResponse.class); return response.tokens(); } @@ -48,15 +43,13 @@ final class LlamaServerTokenizer implements Tokenizer { public String detokenize(final List tokens, final ModelProfile modelProfile) { LOGGER.debug("Requesting detokenizing"); final var request = new DetokenizeRequest(tokens); - final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); - final var response = webClient + final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); + final var response = restClient .method(HttpMethod.POST) - .uri(uriBuilder -> uriBuilder.path("/detokenize").build()) - .bodyValue(request) + .uri("/detokenize") + .body(request) .retrieve() - .bodyToMono(DetokenizeResponse.class) - .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) - .block(); + .body(DetokenizeResponse.class); return response.content(); } diff --git a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java index 81d9872..10cca8f 100644 --- a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java +++ b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java @@ -7,7 +7,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.client.RestClient; @AutoConfiguration public class TokenizerAutoConfiguration { @@ -15,8 +15,8 @@ public class TokenizerAutoConfiguration { @Bean @ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer") @ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer") - Tokenizer llamaServerTokenizer(@Qualifier("lmLibWebClientBuilder") final WebClient.Builder webClientBuilder) { - return new LlamaServerTokenizer(webClientBuilder); + Tokenizer llamaServerTokenizer(@Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder) { + return new LlamaServerTokenizer(restClientBuilder); } @Configuration(proxyBeanMethods = false) diff --git a/src/test/java/de/extio/lmlib/agent/AgentTest.java b/src/test/java/de/extio/lmlib/agent/AgentTest.java index 69e0b2f..2b83f64 100644 --- a/src/test/java/de/extio/lmlib/agent/AgentTest.java +++ b/src/test/java/de/extio/lmlib/agent/AgentTest.java @@ -8,6 +8,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Disabled; @@ -26,7 +27,6 @@ import de.extio.lmlib.agent.responsehandler.TextAgentResponseHandler; import de.extio.lmlib.client.ClientService; import de.extio.lmlib.grader.Grader; import de.extio.lmlib.profile.ModelCategory; -import io.netty.util.internal.ThreadLocalRandom; /** * These tests have been verified with Llama 3.1 8B, Gemma2 27B, Gemma3 27B and GPT-OSS models diff --git a/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java index 724e49b..be9cc91 100755 --- a/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java +++ b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java @@ -107,8 +107,13 @@ public class CachedClientTest { final var sb = new java.lang.StringBuilder(); var completion = this.clientService.getClient(ModelCategory.MEDIUM).streamConversation(ModelCategory.MEDIUM, conversation, chunk -> { - sb.append(chunk); - System.out.print(chunk); + if (chunk.reasoningContent() != null) { + System.out.print(chunk.reasoningContent()); + } + if (chunk.content() != null) { + sb.append(chunk.content()); + System.out.print(chunk.content()); + } }); assertFalse(completion.statistics().cached()); assertEquals(1, completion.statistics().requests()); @@ -120,8 +125,13 @@ public class CachedClientTest { final var sb2 = new java.lang.StringBuilder(); completion = this.clientService.getClient(ModelCategory.MEDIUM).streamConversation(ModelCategory.MEDIUM, conversation, chunk -> { - sb2.append(chunk); - System.out.print(chunk); + if (chunk.reasoningContent() != null) { + System.out.print(chunk.reasoningContent()); + } + if (chunk.content() != null) { + sb2.append(chunk.content()); + System.out.print(chunk.content()); + } }); assertTrue(completion.statistics().cached()); assertEquals(sb.toString(), sb2.toString()); diff --git a/src/test/java/de/extio/lmlib/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/token/TokenizerTest.java index 9f03191..0669f57 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerTest.java @@ -11,7 +11,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.PropertySource; -import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.client.RestClient; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfileService; @@ -28,8 +28,8 @@ public class TokenizerTest { private ModelProfileService modelProfileService; @Autowired - @Qualifier("lmLibWebClientBuilder") - private WebClient.Builder webClientBuilder; + @Qualifier("lmLibRestClientBuilder") + private RestClient.Builder restClientBuilder; @Test void jTokkit() throws Exception { @@ -48,7 +48,7 @@ public class TokenizerTest { @Disabled("Requires llama.cpp model profile") @Test void llamaServer() throws Exception { - final var tokenizer = new LlamaServerTokenizer(this.webClientBuilder); + final var tokenizer = new LlamaServerTokenizer(this.restClientBuilder); final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile()); assertEquals(21, tokenizer.count(TEXT, profile)); diff --git a/src/test/java/de/extio/lmlib/util/TextUtilsTest.java b/src/test/java/de/extio/lmlib/util/TextUtilsTest.java index 4ea76e9..47e208d 100755 --- a/src/test/java/de/extio/lmlib/util/TextUtilsTest.java +++ b/src/test/java/de/extio/lmlib/util/TextUtilsTest.java @@ -215,8 +215,6 @@ public class TextUtilsTest { "'This is a test with special characters', 'test{2}', false", "'This is a test with special characters', 'test[abc]', false", - "'bean:exchangeSenderProducer?method=sendToNamedConnection(${exchange}, \\'HttpsOutbound\\')', 'exchangeSenderProducer', true", - "'Service.process(${param}, [1,2,3])', 'Service', true", "'Service.process(${param}, [1,2,3])', 'process', true", "'Service.process(${param}, [1,2,3])', 'param', true", diff --git a/src/test/resources/local-completion.properties b/src/test/resources/local-completion.properties index a2d560f..276f0ac 100755 --- a/src/test/resources/local-completion.properties +++ b/src/test/resources/local-completion.properties @@ -1,5 +1,5 @@ category=TEXT -prompts=gemma +prompts=gpt-oss tokenEncoding=cl100k_base maxTokens=2500 maxContextLength=16000