exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged with tp lmlib

Browse Source
This commit is contained in:
extio-de
2025-06-27 16:36:46 +02:00
parent dcd6eeee41
commit 94894aee85
10 changed files with 279 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/de/extio/lmlib/client/WebClientConfiguration.java
View File

@@ -19,7 +19,7 @@ import reactor.netty.http.client.HttpClient;
@Configuration
public class WebClientConfiguration {
private static final int TIMEOUT = 900000;
private static final int TIMEOUT = 300000;
private static final int CONNECT_TIMEOUT = 10000;

130
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -1,6 +1,8 @@
package de.extio.lmlib.client.azureai;
import java.math.BigDecimal;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
@@ -26,6 +28,8 @@ import com.azure.ai.inference.models.ChatRequestSystemMessage;
import com.azure.ai.inference.models.ChatRequestUserMessage;
import com.azure.ai.inference.models.CompletionsFinishReason;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.core.http.HttpClient;
import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder;
import com.azure.core.http.policy.ExponentialBackoffOptions;
import com.azure.core.http.policy.RetryOptions;
@@ -35,23 +39,49 @@ import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
import okhttp3.Authenticator;
import okhttp3.Challenge;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.Route;
public final class AzureAiClient implements Client {
private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClient.class);
private static final int TIMEOUT = 300000;
private static final int CONNECT_TIMEOUT = 10000;
private final ConcurrentMap<ModelCategory, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
private final ModelProfileService modelProfileService;
private final Tokenizer tokenizer;
AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
private final boolean proxyEnabled;
private final String proxyHost;
private final int proxyPort;
private final String proxyUser;
private final String proxyPassword;
AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer, final boolean proxyEnabled, final String proxyHost, final int proxyPort, final String proxyUser, final String proxyPassword) {
this.modelProfileService = modelProfileService;
this.tokenizer = tokenizer;
this.proxyEnabled = proxyEnabled;
this.proxyHost = proxyHost;
this.proxyPort = proxyPort;
this.proxyUser = proxyUser;
this.proxyPassword = proxyPassword;
}
@Override
@@ -98,7 +128,7 @@ public final class AzureAiClient implements Client {
answers.add(answer);
}
// TODO: Reactivate when multiple (split) prompts have been implemented again
// TODO: Reactivate when multiple (split) prompts have been implemented again in #addChatRequestMessage
// if (answers.size() > 1) {
// this.summarize(answers, modelProfile, promptStrategy, statistics);
// }
@@ -178,7 +208,8 @@ public final class AzureAiClient implements Client {
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
trimmed = true;
}
}
chat.add(messageFactory.apply(prompt));
return trimmed;
@@ -211,21 +242,8 @@ public final class AzureAiClient implements Client {
private ChatCompletions requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
final var client = this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
.setMaxRetries(99)
.setBaseDelay(Duration.ofSeconds(1))
.setMaxDelay(Duration.ofSeconds(10));
final RetryOptions retryOptions = new RetryOptions(exponentialOptions);
final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
.credential(new AzureKeyCredential(modelProfile.apiKey()))
.endpoint(modelProfile.url())
.retryOptions(retryOptions)
.buildClient();
return chatCompletionsClient;
});
final var client = this.createHttpClient(modelCategory, modelProfile);
final var options = new ChatCompletionsOptions(chat);
options.setModel(modelProfile.modelName());
options.setMaxTokens(modelProfile.maxTokens());
@@ -235,6 +253,80 @@ public final class AzureAiClient implements Client {
return client.complete(options);
}
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
return this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
.setMaxRetries(99)
.setBaseDelay(Duration.ofSeconds(1))
.setMaxDelay(Duration.ofSeconds(10));
final RetryOptions retryOptions = new RetryOptions(exponentialOptions)
.setShouldRetryCondition(httpResponse -> {
if (httpResponse != null) {
if (httpResponse.getThrowable() != null) {
return true;
}
if (httpResponse.getResponse() != null) {
return this.shouldRetryError(httpResponse.getResponse().getStatusCode());
}
}
return false;
});
final OkHttpClient.Builder nativeOkHttpClientBuilder = new OkHttpClient.Builder();
if (this.proxyEnabled) {
LOGGER.info("Proxy is enabled. Configuring proxy settings for host: {} port: {} user: {}", this.proxyHost, this.proxyPort, this.proxyUser);
final Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(this.proxyHost, this.proxyPort));
nativeOkHttpClientBuilder.proxy(proxy);
if (this.proxyUser != null && this.proxyPassword != null) {
final Authenticator proxyAuthenticator = new Authenticator() {
@Override
public Request authenticate(final Route route, final Response response) {
for (final Challenge challenge : response.challenges()) {
if (challenge.scheme().equalsIgnoreCase("OkHttp-Preemptive")) {
final String credential = Credentials.basic(proxyUser, proxyPassword);
return response.request().newBuilder()
.header("Proxy-Authorization", credential)
.build();
}
}
return null;
}
};
nativeOkHttpClientBuilder.proxyAuthenticator(proxyAuthenticator);
} else {
LOGGER.warn("Proxy user or password is not set. Skipping proxy authentication.");
}
} else {
LOGGER.info("Proxy is not enabled. Skipping proxy configuration.");
}
final OkHttpClient nativeOkHttpClient = nativeOkHttpClientBuilder.build();
final OkHttpAsyncHttpClientBuilder okHttpClientBuilder = new OkHttpAsyncHttpClientBuilder(nativeOkHttpClient);
final HttpClient okHttpClient = okHttpClientBuilder
.connectionTimeout(Duration.ofMillis(CONNECT_TIMEOUT))
.readTimeout(Duration.ofMillis(TIMEOUT))
.responseTimeout(Duration.ofMillis(TIMEOUT))
.writeTimeout(Duration.ofMillis(TIMEOUT))
.build();
final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
.credential(new AzureKeyCredential(modelProfile.apiKey()))
.endpoint(modelProfile.url())
.retryOptions(retryOptions)
.httpClient(okHttpClient)
.buildClient();
return chatCompletionsClient;
});
}
private boolean shouldRetryError(final int statusCode) {
return statusCode == 408 || statusCode == 429 || (statusCode >= 500 && statusCode < 600);
}
static class CompletionStatistics {
int requests;

20
src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java Normal file → Executable file
View File

@@ -1,5 +1,6 @@
package de.extio.lmlib.client.azureai;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.context.annotation.Bean;
@@ -8,12 +9,27 @@ import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
@AutoConfiguration
@ConditionalOnClass(name = "com.azure.ai.inference.ChatCompletionsClient")
@ConditionalOnClass(name = {"com.azure.ai.inference.ChatCompletionsClient", "okhttp3.OkHttpClient" })
public class AzureAiClientAutoConfiguration {
@Value("${extproxy.enabled:false}")
private boolean proxyEnabled;
@Value("${extproxy.host:}")
private String proxyHost;
@Value("${extproxy.port:}")
private int proxyPort;
@Value("${extproxy.username:}")
private String proxyUser;
@Value("${extproxy.password:}")
private String proxyPassword;
@Bean
AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
return new AzureAiClient(modelProfileService, tokenizer);
return new AzureAiClient(modelProfileService, tokenizer, this.proxyEnabled, this.proxyHost, this.proxyPort, this.proxyUser, this.proxyPassword);
}
}

2
src/test/java/de/extio/lmlib/agent/AgentTest.java
View File

@@ -9,6 +9,7 @@ import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,6 +29,7 @@ import io.netty.util.internal.ThreadLocalRandom;
/**
* These tests have been verified with Llama 3.1 8B and Gemma2 27B models
*/
@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")

4
src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java
View File

@@ -16,7 +16,7 @@ import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.profile.ModelCategory;
@Disabled
@Disabled("This test requires an Azure subscription (setup azure key in model profile)")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@@ -35,7 +35,7 @@ public class AzureAiClientTest {
"You are a clever story teller",
"Continue this story:",
"Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
LOGGER.info(completion.response());
LOGGER.info(completion.toString());
}
@Test

106
src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java Executable file
View File

@@ -0,0 +1,106 @@
package de.extio.lmlib.client.cached;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.profile.ModelCategory;
@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@PropertySource("classpath:/application-test.properties")
public class CachedClientTest {
private static final Logger LOGGER = LoggerFactory.getLogger(CachedClientTest.class);
@Autowired
private TestCachedClientRepository cachedClientRepository;
@Autowired
private ClientService clientService;
@AfterEach
void afterEach() {
this.cachedClientRepository.reset();
}
@Test
void cacheCompletion() throws Exception {
var completion = this.clientService.getClient(ModelCategory.MEDIUM).completion(
ModelCategory.MEDIUM,
"You are a funny person",
"Tell me a cat joke",
"");
LOGGER.info(completion.toString());
assertFalse(completion.cached());
assertEquals(1, completion.requests());
final var firstResponse = completion.response();
assertFalse(firstResponse.isBlank());
completion = this.clientService.getClient(ModelCategory.MEDIUM).completion(
ModelCategory.MEDIUM,
"You are a funny person",
"Tell me a cat joke",
"");
LOGGER.info(completion.toString());
assertTrue(completion.cached());
assertEquals(0, completion.requests());
assertEquals(firstResponse, completion.response());
completion = this.clientService.getClient(ModelCategory.MEDIUM).completion(
ModelCategory.MEDIUM,
"You are a funny person",
"Tell me a dog joke",
"");
LOGGER.info(completion.toString());
assertFalse(completion.cached());
assertEquals(1, completion.requests());
assertNotEquals(firstResponse, completion.response());
}
@Test
void cacheChat() throws Exception {
final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Munich.");
var completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.toString());
assertFalse(completion.cached());
assertEquals(1, completion.requests());
final var firstResponse = completion.response();
assertFalse(firstResponse.isBlank());
completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.toString());
assertTrue(completion.cached());
assertEquals(0, completion.requests());
assertEquals(firstResponse, completion.response());
conversation.addTurn(new Turn(TurnType.ASSISTANT,
"T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius."));
conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?"));
completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.toString());
assertFalse(completion.cached());
assertEquals(1, completion.requests());
assertNotEquals(firstResponse, completion.response());
}
}

27
src/test/java/de/extio/lmlib/client/cached/TestCachedClientRepository.java Executable file
View File

@@ -0,0 +1,27 @@
package de.extio.lmlib.client.cached;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.springframework.stereotype.Component;
@Component
class TestCachedClientRepository implements CachedClientRepository {
private final ConcurrentMap<String, CachedCompletion> cache = new ConcurrentHashMap<>();
@Override
public CachedCompletion get(final String key) {
return this.cache.get(key);
}
@Override
public void put(final String key, final CachedCompletion completion) {
this.cache.put(key, completion);
}
void reset() {
this.cache.clear();
}
}

1
src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java
View File

@@ -23,6 +23,7 @@ import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.profile.ModelCategory;
@Disabled("This test requires a running Llama server")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")

4
src/test/java/de/extio/lmlib/token/TokenizerTest.java
View File

@@ -2,6 +2,7 @@ package de.extio.lmlib.token;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -42,8 +43,9 @@ public class TokenizerTest {
}
/**
* Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model.
* Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model.
*/
@Disabled("Requires llama.cpp model profile")
@Test
void llamaServer() throws Exception {
final var tokenizer = new LlamaServerTokenizer(this.webClientBuilder);