exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 2.0

Browse Source 
Java 25, Spring Boot 4, Use RestClient with JDK client backend
This commit is contained in:
extio-de
2025-12-19 17:58:56 +01:00
parent 4c0c30974e
commit 4ebb8640c7
31 changed files with 392 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/de/extio/lmlib/agent/BaseAgent.java
View File

@@ -12,8 +12,8 @@ import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.ObjectMapper;
import de.extio.lmlib.agent.responsehandler.AgentResponseHandler;
import de.extio.lmlib.agent.responsehandler.StreamedAgentResponseHandler;
@@ -239,7 +239,7 @@ public interface BaseAgent {
try {
text = text.replace(k, objectMapper.writeValueAsString(entry.getValue()));
}
catch (final JsonProcessingException e) {
catch (final JacksonException e) {
LOGGER.warn("Cannot write json", e);
}
}

14
src/main/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandler.java
View File

@@ -10,11 +10,11 @@ import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.json.JsonReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import tools.jackson.core.json.JsonReadFeature;
import tools.jackson.databind.DeserializationFeature;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.json.JsonMapper;
import de.extio.lmlib.agent.AgentContext;
import de.extio.lmlib.client.Completion;
@@ -52,7 +52,7 @@ public class JsonAgentResponseHandler implements AgentResponseHandler {
.enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES)
.enable(JsonReadFeature.ALLOW_UNQUOTED_PROPERTY_NAMES)
.build();
}
@@ -94,7 +94,7 @@ public class JsonAgentResponseHandler implements AgentResponseHandler {
private void parseEntry(final LinkedHashMap<String, List<String>> items, final Entry<String, JsonNode> entry) {
if (entry.getValue().isArray()) {
entry.getValue().elements().forEachRemaining(nestedArrayValue -> {
entry.getValue().values().forEach(nestedArrayValue -> {
if (nestedArrayValue.properties().isEmpty()) {
items.computeIfAbsent(this.prefixFields + entry.getKey(), k -> new ArrayList<>()).add(nestedArrayValue.asText());
}

139
src/main/java/de/extio/lmlib/client/RestClientConfiguration.java Normal file
View File

@@ -0,0 +1,139 @@
package de.extio.lmlib.client;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.ProxySelector;
import java.net.http.HttpClient;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.time.Duration;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.util.backoff.BackOff;
import org.springframework.util.backoff.BackOffExecution;
import org.springframework.util.backoff.ExponentialBackOff;
import org.springframework.web.client.RestClient;
@Configuration
public class RestClientConfiguration {
private static final Logger LOGGER = LoggerFactory.getLogger(RestClientConfiguration.class);
private static final int TIMEOUT = 300000;
private static final int CONNECT_TIMEOUT = 10000;
@Bean
@Qualifier("lmLibRestClientBuilder")
RestClient.Builder lmLibRestClientBuilder(
@Value("${lmlib.client.proxy.enabled:false}") final boolean proxyEnabled,
@Value("${lmlib.client.proxy.host:}") final String proxyHost,
@Value("${lmlib.client.proxy.port:0}") final int proxyPort,
@Value("${lmlib.client.proxy.user:}") final String proxyUser,
@Value("${lmlib.client.proxy.password:}") final String proxyPassword,
@Value("${lmlib.client.tls.verification.disabled:false}") final boolean tlsVerificationDisabled,
@Value("${lmlib.client.retry.max-attempts:5}") final int maxAttempts,
@Value("${lmlib.client.retry.backoff-interval-min:125}") final long backoffIntervalMin,
@Value("${lmlib.client.retry.backoff-interval-max:2500}") final long backoffIntervalMax) {
final HttpClient.Builder httpClientBuilder = HttpClient.newBuilder()
.connectTimeout(Duration.ofMillis(CONNECT_TIMEOUT));
if (proxyEnabled && proxyHost != null && !proxyHost.isBlank()) {
httpClientBuilder.proxy(ProxySelector.of(new InetSocketAddress(proxyHost, proxyPort)));
if (proxyUser != null && !proxyUser.isBlank()) {
httpClientBuilder.authenticator(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUser, proxyPassword.toCharArray());
}
});
}
}
if (tlsVerificationDisabled) {
try {
final SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, new TrustManager[] { new X509TrustManager() {
@Override
public void checkClientTrusted(final X509Certificate[] chain, final String authType) {
}
@Override
public void checkServerTrusted(final X509Certificate[] chain, final String authType) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
} }, new SecureRandom());
httpClientBuilder.sslContext(sslContext);
}
catch (NoSuchAlgorithmException | KeyManagementException e) {
throw new RuntimeException("Failed to disable TLS verification", e);
}
}
final JdkClientHttpRequestFactory requestFactory = new JdkClientHttpRequestFactory(httpClientBuilder.build());
requestFactory.setReadTimeout(Duration.ofMillis(TIMEOUT));
return RestClient.builder()
.requestFactory(requestFactory)
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
.defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
.requestInterceptor(createRetryInterceptor(maxAttempts, backoffIntervalMin, backoffIntervalMax));
}
private ClientHttpRequestInterceptor createRetryInterceptor(final int maxAttempts, final long backoffIntervalMin, final long backoffIntervalMax) {
return (request, body, execution) -> {
final BackOff backOff = new ExponentialBackOff();
((ExponentialBackOff) backOff).setInitialInterval(backoffIntervalMin);
((ExponentialBackOff) backOff).setMaxInterval(backoffIntervalMax);
((ExponentialBackOff) backOff).setMaxElapsedTime(maxAttempts * backoffIntervalMax);
((ExponentialBackOff) backOff).setMultiplier(2.0);
((ExponentialBackOff) backOff).setJitter(backoffIntervalMin / 2);
final BackOffExecution backOffExecution = backOff.start();
while (true) {
try {
return execution.execute(request, body);
}
catch (final java.io.IOException e) {
final long waitTime = backOffExecution.nextBackOff();
if (waitTime == BackOffExecution.STOP) {
throw e;
}
LOGGER.warn("Request failed, retrying in {} ms: {} {}", waitTime, e.getClass().getSimpleName(), e.getMessage() != null ? e.getMessage() : "");
try {
Thread.sleep(waitTime);
}
catch (final InterruptedException ie) {
Thread.currentThread().interrupt();
throw e;
}
}
}
};
}
}

40
src/main/java/de/extio/lmlib/client/WebClientConfiguration.java
View File

@@ -1,40 +0,0 @@
package de.extio.lmlib.client;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import io.netty.channel.ChannelOption;
import io.netty.handler.timeout.ReadTimeoutHandler;
import io.netty.handler.timeout.WriteTimeoutHandler;
import reactor.netty.http.client.HttpClient;
@Configuration
public class WebClientConfiguration {
private static final int TIMEOUT = 300000;
private static final int CONNECT_TIMEOUT = 10000;
@Bean
@Qualifier("lmLibWebClientBuilder")
WebClient.Builder lmLibWebClientBuilder() {
final HttpClient httpClient = HttpClient.create()
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
.responseTimeout(Duration.ofMillis(TIMEOUT))
.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
return WebClient.builder()
.clientConnector(new ReactorClientHttpConnector(httpClient))
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
.defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
}
}

2
src/main/java/de/extio/lmlib/client/oai/Model.java
View File

@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
public final class Model {

13
src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java
View File

@@ -5,28 +5,27 @@ import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.client.RestClient;
class ModelNameLoader {
private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class);
private final WebClient webClient;
private final RestClient restClient;
public ModelNameLoader(final WebClient webClient) {
this.webClient = webClient;
public ModelNameLoader(final RestClient restClient) {
this.restClient = restClient;
}
public Optional<String> loadModelName() {
try {
LOGGER.info("Querying model name");
final var modelName = this.webClient
final var modelName = this.restClient
.method(HttpMethod.GET)
.uri(uriBuilder -> uriBuilder.path("/v1/models").build())
.retrieve()
.bodyToMono(ModelsResponse.class)
.block()
.body(ModelsResponse.class)
.getData()
.stream()
.map(Model::getId)

10
src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java
View File

@@ -6,7 +6,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.client.RestClient;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
@@ -15,8 +15,8 @@ import de.extio.lmlib.profile.ModelProfile.ModelProvider;
public class ModelNameSupplier {
@Autowired
@Qualifier("lmLibWebClientBuilder")
private WebClient.Builder webClientBuilder;
@Qualifier("lmLibRestClientBuilder")
private RestClient.Builder restClientBuilder;
private final Map<String, String> resolvedModelNames = new ConcurrentHashMap<>();
@@ -31,8 +31,8 @@ public class ModelNameSupplier {
final var cacheKey = modelProfile.category() + "|" + modelProfile.url();
return this.resolvedModelNames.computeIfAbsent(cacheKey, key -> {
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var loader = new ModelNameLoader(webClient);
final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build();
final var loader = new ModelNameLoader(restClient);
return loader.loadModelName().orElse("");
});
}

2
src/main/java/de/extio/lmlib/client/oai/ModelsResponse.java
View File

@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
public final class ModelsResponse {

30
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
View File

@@ -13,13 +13,13 @@ import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.client.RestClient;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.json.JsonReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import tools.jackson.core.json.JsonReadFeature;
import tools.jackson.databind.DeserializationFeature;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.json.JsonMapper;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Client;
@@ -32,15 +32,14 @@ import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
import reactor.util.retry.Retry;
public abstract class AbstractCompletionClient implements Client, DisposableBean {
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
@Autowired
@Qualifier("lmLibWebClientBuilder")
protected WebClient.Builder webClientBuilder;
@Qualifier("lmLibRestClientBuilder")
protected RestClient.Builder restClientBuilder;
@Autowired
protected ModelNameSupplier modelNameSupplier;
@@ -65,7 +64,8 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
.configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false)
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS, false)
.serializationInclusion(JsonInclude.Include.NON_NULL)
.changeDefaultPropertyInclusion(incl -> incl.withValueInclusion(JsonInclude.Include.NON_NULL))
.changeDefaultPropertyInclusion(incl -> incl.withContentInclusion(JsonInclude.Include.NON_NULL))
.enable(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER)
.enable(JsonReadFeature.ALLOW_JAVA_COMMENTS)
.enable(JsonReadFeature.ALLOW_MISSING_VALUES)
@@ -77,7 +77,7 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
.enable(JsonReadFeature.ALLOW_LEADING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_TRAILING_DECIMAL_POINT_FOR_NUMBERS)
.enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES)
.enable(JsonReadFeature.ALLOW_UNQUOTED_PROPERTY_NAMES)
.build();
}
@@ -122,16 +122,6 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
};
}
protected Retry createRetrySpec() {
return Retry.backoff(99, Duration.ofSeconds(1))
.maxBackoff(Duration.ofSeconds(10))
.jitter(0.25d)
.filter(throwable -> {
return throwable instanceof java.io.IOException || throwable instanceof java.net.ConnectException || throwable instanceof java.net.http.HttpTimeoutException;
})
.doAfterRetry(rs -> LOGGER.warn("Retrying failed request: " + rs.failure().getClass().getName() + " " + rs.failure().getMessage()));
}
protected void configureStreamOptions(final boolean enableStreaming, final Consumer<StreamOptions> streamOptionsConsumer) {
if (enableStreaming) {
final var streamOptions = new StreamOptions();

2
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
public abstract class AbstractCompletionResponse {

2
src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
public class StreamOptions {

2
src/main/java/de/extio/lmlib/client/oai/completion/Timings.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
public final class Timings {

2
src/main/java/de/extio/lmlib/client/oai/completion/Usage.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
public final class Usage {

2
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion.chat;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
class ChatChoice {

132
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java
View File

@@ -1,5 +1,8 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -12,8 +15,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestClient;
import com.fasterxml.jackson.core.JsonProcessingException;
import tools.jackson.core.JacksonException;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Completion;
@@ -22,7 +26,6 @@ import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import reactor.core.publisher.Mono;
@Component
public class ChatCompletionClient extends AbstractCompletionClient {
@@ -80,92 +83,99 @@ public class ChatCompletionClient extends AbstractCompletionClient {
try {
requestBody = this.objectMapper.writeValueAsString(request);
}
catch (final JsonProcessingException e) {
catch (final JacksonException e) {
throw new IllegalStateException("Cannot serialize request body", e);
}
LOGGER.debug("Requesting chat completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build();
var requestSpec = webClient
var requestSpec = restClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build())
.uri("/v1/chat/completions")
.header("Content-Type", "application/json");
if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) {
requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey());
}
final var response = requestSpec
.bodyValue(requestBody)
.exchangeToMono(clientResponse -> {
if (clientResponse.statusCode().isError()) {
return clientResponse.bodyToMono(String.class)
.flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody)));
.body(requestBody)
.exchange((clientRequest, clientResponse) -> {
if (clientResponse.getStatusCode().isError()) {
throw new IllegalStateException("Error response from server: " + clientResponse.getStatusCode());
}
if (chunkConsumer == null) {
return clientResponse.bodyToMono(ChatCompletionResponse.class);
return this.objectMapper.readValue(clientResponse.getBody(), ChatCompletionResponse.class);
}
else {
final var chatCompletionResponse = new ChatCompletionResponse();
chatCompletionResponse.setChoices(List.of(new ChatChoice()));
final StringBuilder contentStringBuilder = new StringBuilder();
final StringBuilder reasoningStringBuilder = new StringBuilder();
return clientResponse.bodyToFlux(String.class)
.doOnNext(buffer -> {
try {
if (!"[DONE]".equals(buffer)) {
final var streamResponse = this.objectMapper.readValue(buffer, ChatCompletionResponse.class);
if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) {
chatCompletionResponse.setId(streamResponse.getId());
try (BufferedReader reader = new BufferedReader(new InputStreamReader(clientResponse.getBody(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
if (line.startsWith("data: ")) {
line = line.substring(6).trim();
}
if (line.isEmpty()) {
continue;
}
if ("[DONE]".equals(line)) {
break;
}
try {
final var streamResponse = this.objectMapper.readValue(line, ChatCompletionResponse.class);
if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) {
chatCompletionResponse.setId(streamResponse.getId());
}
if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) {
chatCompletionResponse.setObject(streamResponse.getObject());
}
if (streamResponse.getCreated() != null) {
chatCompletionResponse.setCreated(streamResponse.getCreated());
}
if (streamResponse.getUsage() != null) {
chatCompletionResponse.setUsage(streamResponse.getUsage());
}
if (streamResponse.getTimings() != null) {
chatCompletionResponse.setTimings(streamResponse.getTimings());
}
if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) {
if (streamResponse.getChoices().getFirst().getFinishReason() != null) {
chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason());
}
if (streamResponse.getChoices().getFirst().getMessage() != null) {
String content = null;
String reasoning = null;
if (streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) {
content = streamResponse.getChoices().getFirst().getMessage().getContent();
contentStringBuilder.append(content);
}
if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) {
chatCompletionResponse.setObject(streamResponse.getObject());
if (streamResponse.getChoices().getFirst().getMessage().getReasoningContent() != null && !streamResponse.getChoices().getFirst().getMessage().getReasoningContent().isEmpty()) {
reasoning = streamResponse.getChoices().getFirst().getMessage().getReasoningContent();
reasoningStringBuilder.append(reasoning);
}
if (streamResponse.getCreated() != null) {
chatCompletionResponse.setCreated(streamResponse.getCreated());
}
if (streamResponse.getUsage() != null) {
chatCompletionResponse.setUsage(streamResponse.getUsage());
}
if (streamResponse.getTimings() != null) {
chatCompletionResponse.setTimings(streamResponse.getTimings());
}
if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) {
if (streamResponse.getChoices().getFirst().getFinishReason() != null) {
chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason());
}
if (streamResponse.getChoices().getFirst().getMessage() != null) {
String content = null;
String reasoning = null;
if (streamResponse.getChoices().getFirst().getMessage().getContent() != null && !streamResponse.getChoices().getFirst().getMessage().getContent().isEmpty()) {
content = streamResponse.getChoices().getFirst().getMessage().getContent();
contentStringBuilder.append(content);
}
if (streamResponse.getChoices().getFirst().getMessage().getReasoningContent() != null && !streamResponse.getChoices().getFirst().getMessage().getReasoningContent().isEmpty()) {
reasoning = streamResponse.getChoices().getFirst().getMessage().getReasoningContent();
reasoningStringBuilder.append(reasoning);
}
if (content != null || reasoning != null) {
chunkConsumer.accept(new Chunk(content, reasoning));
}
}
if (content != null || reasoning != null) {
chunkConsumer.accept(new Chunk(content, reasoning));
}
}
}
catch (final JsonProcessingException e) {
LOGGER.warn("Streamed chunk is not parseable", e);
}
})
.then(Mono.<ChatCompletionResponse> fromSupplier(() -> {
chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contentStringBuilder.toString(), reasoningStringBuilder.toString()));
return chatCompletionResponse;
}));
}
catch (final JacksonException e) {
LOGGER.warn("Streamed chunk is not parseable: {}", line, e);
}
}
}
chatCompletionResponse.getChoices().getFirst().setMessage(new ChatMessage("assistant", contentStringBuilder.toString(), reasoningStringBuilder.toString()));
return chatCompletionResponse;
}
})
.retryWhen(this.createRetrySpec())
.block();
});
if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) {
throw new IllegalStateException("No parseable response content from server");

2
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java
View File

@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion.chat;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;

2
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java
View File

@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion.chat;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse;

2
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion.chat;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
final class ChatMessage {

2
src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion.text;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
final class Choice {

2
src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java
View File

@@ -1,6 +1,6 @@
package de.extio.lmlib.client.oai.completion.text;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
import de.extio.lmlib.client.oai.completion.AbstractCompletionRequest;

2
src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java
View File

@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion.text;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.*;
import de.extio.lmlib.client.oai.completion.AbstractCompletionResponse;

137
src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java
View File

@@ -1,5 +1,8 @@
package de.extio.lmlib.client.oai.completion.text;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.Deque;
@@ -11,8 +14,9 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestClient;
import com.fasterxml.jackson.core.JsonProcessingException;
import tools.jackson.core.JacksonException;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Completion;
@@ -24,7 +28,6 @@ import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import reactor.core.publisher.Mono;
@Component
public class TextCompletionClient extends AbstractCompletionClient {
@@ -65,88 +68,94 @@ public class TextCompletionClient extends AbstractCompletionClient {
try {
requestBody = this.objectMapper.writeValueAsString(request);
}
catch (final JsonProcessingException e) {
catch (final JacksonException e) {
throw new IllegalStateException("Cannot serialize request body", e);
}
LOGGER.debug("Requesting completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
var requestSpec = webClient
final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build();
var requestSpec = restClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.uri("/v1/completions")
.header("Content-Type", "application/json");
if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) {
requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey());
}
final var response = requestSpec
.bodyValue(requestBody)
.exchangeToMono(clientResponse -> {
if (clientResponse.statusCode().isError()) {
return clientResponse.bodyToMono(String.class)
.flatMap(errorBody -> Mono.error(new IllegalStateException("Error response from server: " + clientResponse.statusCode() + " - " + errorBody)));
.body(requestBody)
.exchange((clientRequest, clientResponse) -> {
if (clientResponse.getStatusCode().isError()) {
throw new IllegalStateException("Error response from server: " + clientResponse.getStatusCode());
}
if (chunkConsumer == null) {
return clientResponse.bodyToMono(CompletionResponse.class);
return this.objectMapper.readValue(clientResponse.getBody(), CompletionResponse.class);
}
else {
final var CompletionResponse = new CompletionResponse();
CompletionResponse.setChoices(List.of(new Choice()));
final var completionResponse = new CompletionResponse();
completionResponse.setChoices(List.of(new Choice()));
final StringBuilder contentStringBuilder = new StringBuilder();
return clientResponse.bodyToFlux(String.class)
.doOnNext(buffer -> {
try {
if (!"[DONE]".equals(buffer)) {
final var streamResponse = this.objectMapper.readValue(buffer, CompletionResponse.class);
if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) {
CompletionResponse.setId(streamResponse.getId());
}
if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) {
CompletionResponse.setObject(streamResponse.getObject());
}
if (streamResponse.getCreated() != null) {
CompletionResponse.setCreated(streamResponse.getCreated());
}
if (streamResponse.isStoppedEos() != null) {
CompletionResponse.setStoppedEos(streamResponse.isStoppedEos());
}
if (streamResponse.getContent() != null && !streamResponse.getContent().isEmpty()) {
contentStringBuilder.append(streamResponse.getContent());
}
if (streamResponse.getUsage() != null) {
CompletionResponse.setUsage(streamResponse.getUsage());
}
if (streamResponse.getTimings() != null) {
CompletionResponse.setTimings(streamResponse.getTimings());
}
if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) {
if (streamResponse.getChoices().getFirst().getFinishReason() != null) {
CompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason());
}
if (streamResponse.getChoices().getFirst().getScore() != null) {
CompletionResponse.getChoices().getFirst().setScore(streamResponse.getChoices().getFirst().getScore());
}
if (streamResponse.getChoices().getFirst().getText() != null && !streamResponse.getChoices().getFirst().getText().isEmpty()) {
contentStringBuilder.append(streamResponse.getChoices().getFirst().getText());
chunkConsumer.accept(new Chunk(streamResponse.getChoices().getFirst().getText(), null));
}
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(clientResponse.getBody(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
if (line.startsWith("data: ")) {
line = line.substring(6).trim();
}
if (line.isEmpty()) {
continue;
}
if ("[DONE]".equals(line)) {
break;
}
try {
final var streamResponse = this.objectMapper.readValue(line, CompletionResponse.class);
if (streamResponse.getId() != null && !streamResponse.getId().isEmpty()) {
completionResponse.setId(streamResponse.getId());
}
if (streamResponse.getObject() != null && !streamResponse.getObject().isEmpty()) {
completionResponse.setObject(streamResponse.getObject());
}
if (streamResponse.getCreated() != null) {
completionResponse.setCreated(streamResponse.getCreated());
}
if (streamResponse.isStoppedEos() != null) {
completionResponse.setStoppedEos(streamResponse.isStoppedEos());
}
if (streamResponse.getContent() != null && !streamResponse.getContent().isEmpty()) {
contentStringBuilder.append(streamResponse.getContent());
}
if (streamResponse.getUsage() != null) {
completionResponse.setUsage(streamResponse.getUsage());
}
if (streamResponse.getTimings() != null) {
completionResponse.setTimings(streamResponse.getTimings());
}
if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) {
if (streamResponse.getChoices().getFirst().getFinishReason() != null) {
completionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason());
}
if (streamResponse.getChoices().getFirst().getScore() != null) {
completionResponse.getChoices().getFirst().setScore(streamResponse.getChoices().getFirst().getScore());
}
if (streamResponse.getChoices().getFirst().getText() != null && !streamResponse.getChoices().getFirst().getText().isEmpty()) {
contentStringBuilder.append(streamResponse.getChoices().getFirst().getText());
chunkConsumer.accept(new Chunk(streamResponse.getChoices().getFirst().getText(), null));
}
}
catch (final JsonProcessingException e) {
LOGGER.warn("Streamed chunk is not parseable", e);
}
})
.then(Mono.<CompletionResponse> fromSupplier(() -> {
CompletionResponse.getChoices().getFirst().setText(contentStringBuilder.toString());
return CompletionResponse;
}));
}
catch (final JacksonException e) {
LOGGER.warn("Streamed chunk is not parseable: {}", line, e);
}
}
}
completionResponse.getChoices().getFirst().setText(contentStringBuilder.toString());
return completionResponse;
}
})
.retryWhen(this.createRetrySpec())
.block();
});
String content = null;
String reasoning = null;

35
src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java
View File

@@ -6,36 +6,31 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClient.Builder;
import org.springframework.web.client.RestClient;
import de.extio.lmlib.profile.ModelProfile;
import reactor.util.retry.Retry;
final class LlamaServerTokenizer implements Tokenizer {
private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class);
private final WebClient.Builder webClientBuilder;
private final RestClient.Builder restClientBuilder;
public LlamaServerTokenizer(final Builder webClientBuilder) {
this.webClientBuilder = webClientBuilder;
public LlamaServerTokenizer(final RestClient.Builder restClientBuilder) {
this.restClientBuilder = restClientBuilder;
}
@Override
public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
LOGGER.debug("Requesting tokenizing");
final var request = new TokenizeRequest(txt);
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build();
final var response = restClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/tokenize").build())
.bodyValue(request)
.uri("/tokenize")
.body(request)
.retrieve()
.bodyToMono(TokenizeResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
.body(TokenizeResponse.class);
return response.tokens();
}
@@ -48,15 +43,13 @@ final class LlamaServerTokenizer implements Tokenizer {
public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
LOGGER.debug("Requesting detokenizing");
final var request = new DetokenizeRequest(tokens);
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build();
final var response = restClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/detokenize").build())
.bodyValue(request)
.uri("/detokenize")
.body(request)
.retrieve()
.bodyToMono(DetokenizeResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
.body(DetokenizeResponse.class);
return response.content();
}

6
src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java
View File

@@ -7,7 +7,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.client.RestClient;
@AutoConfiguration
public class TokenizerAutoConfiguration {
@@ -15,8 +15,8 @@ public class TokenizerAutoConfiguration {
@Bean
@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer")
@ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer")
Tokenizer llamaServerTokenizer(@Qualifier("lmLibWebClientBuilder") final WebClient.Builder webClientBuilder) {
return new LlamaServerTokenizer(webClientBuilder);
Tokenizer llamaServerTokenizer(@Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder) {
return new LlamaServerTokenizer(restClientBuilder);
}
@Configuration(proxyBeanMethods = false)

2
src/test/java/de/extio/lmlib/agent/AgentTest.java
View File

@@ -8,6 +8,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.Disabled;
@@ -26,7 +27,6 @@ import de.extio.lmlib.agent.responsehandler.TextAgentResponseHandler;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.grader.Grader;
import de.extio.lmlib.profile.ModelCategory;
import io.netty.util.internal.ThreadLocalRandom;
/**
* These tests have been verified with Llama 3.1 8B, Gemma2 27B, Gemma3 27B and GPT-OSS models

18
src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java
View File

@@ -107,8 +107,13 @@ public class CachedClientTest {
final var sb = new java.lang.StringBuilder();
var completion = this.clientService.getClient(ModelCategory.MEDIUM).streamConversation(ModelCategory.MEDIUM, conversation, chunk -> {
sb.append(chunk);
System.out.print(chunk);
if (chunk.reasoningContent() != null) {
System.out.print(chunk.reasoningContent());
}
if (chunk.content() != null) {
sb.append(chunk.content());
System.out.print(chunk.content());
}
});
assertFalse(completion.statistics().cached());
assertEquals(1, completion.statistics().requests());
@@ -120,8 +125,13 @@ public class CachedClientTest {
final var sb2 = new java.lang.StringBuilder();
completion = this.clientService.getClient(ModelCategory.MEDIUM).streamConversation(ModelCategory.MEDIUM, conversation, chunk -> {
sb2.append(chunk);
System.out.print(chunk);
if (chunk.reasoningContent() != null) {
System.out.print(chunk.reasoningContent());
}
if (chunk.content() != null) {
sb2.append(chunk.content());
System.out.print(chunk.content());
}
});
assertTrue(completion.statistics().cached());
assertEquals(sb.toString(), sb2.toString());

8
src/test/java/de/extio/lmlib/token/TokenizerTest.java
View File

@@ -11,7 +11,7 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.client.RestClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfileService;
@@ -28,8 +28,8 @@ public class TokenizerTest {
private ModelProfileService modelProfileService;
@Autowired
@Qualifier("lmLibWebClientBuilder")
private WebClient.Builder webClientBuilder;
@Qualifier("lmLibRestClientBuilder")
private RestClient.Builder restClientBuilder;
@Test
void jTokkit() throws Exception {
@@ -48,7 +48,7 @@ public class TokenizerTest {
@Disabled("Requires llama.cpp model profile")
@Test
void llamaServer() throws Exception {
final var tokenizer = new LlamaServerTokenizer(this.webClientBuilder);
final var tokenizer = new LlamaServerTokenizer(this.restClientBuilder);
final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
assertEquals(21, tokenizer.count(TEXT, profile));

2
src/test/java/de/extio/lmlib/util/TextUtilsTest.java
View File

@@ -215,8 +215,6 @@ public class TextUtilsTest {
"'This is a test with special characters', 'test{2}', false",
"'This is a test with special characters', 'test[abc]', false",
"'bean:exchangeSenderProducer?method=sendToNamedConnection(${exchange}, \\'HttpsOutbound\\')', 'exchangeSenderProducer', true",
"'Service.process(${param}, [1,2,3])', 'Service', true",
"'Service.process(${param}, [1,2,3])', 'process', true",
"'Service.process(${param}, [1,2,3])', 'param', true",

2
src/test/resources/local-completion.properties
View File

@@ -1,5 +1,5 @@
category=TEXT
prompts=gemma
prompts=gpt-oss
tokenEncoding=cl100k_base
maxTokens=2500
maxContextLength=16000