diff --git a/README.md b/README.md index a17e778..077834e 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ The program is a comprehensive conversational AI system that provides natural la The program consists of several packages that work together to achieve this goal. These packages include: -1. **Caching layer (com.transporeon.exchange.lmlib.client.cached)**: Improves performance by reducing requests to the underlying client through caching and retrieving completion responses. -2. **Natural Language Processing (com.transporeon.exchange.lmlib.client)**: Provides conversation management and text completion functionality, generating completions, processing conversations, and managing models. -3. **Azure AI Integration (com.transporeon.exchange.lmlib.client.azureai)**: Integrates with Azure AI services, managing model profiles and handling conversation completions. -4. **Text Completion (com.transporeon.exchange.lmlib.client.textcompletion)**: Provides text completion functionality using AI models, including model selection, prompt generation, and response handling. -5. **Text Tokenization (com.transporeon.exchange.lmlib.token)**: Tokenizes text into numerical tokens and detokenizes numerical tokens back into human-readable text. -6. **Conversation Prompts (com.transporeon.exchange.lmlib.prompt)**: Manages conversation prompts, creating new prompts, continuing conversations, and removing end-of-text markers. -7. **Agent Executor (com.transporeon.exchange.lmlib.agent)**: Executes agent graphs concurrently, handles agent responses, and tracks request statistics. +1. **Caching layer (.lmlib.client.cached)**: Improves performance by reducing requests to the underlying client through caching and retrieving completion responses. +2. **Natural Language Processing (.lmlib.client)**: Provides conversation management and text completion functionality, generating completions, processing conversations, and managing models. +3. **Azure AI Integration (.lmlib.client.azureai)**: Integrates with Azure AI services, managing model profiles and handling conversation completions. +4. **Text Completion (.lmlib.client.textcompletion)**: Provides text completion functionality using AI models, including model selection, prompt generation, and response handling. +5. **Text Tokenization (.lmlib.token)**: Tokenizes text into numerical tokens and detokenizes numerical tokens back into human-readable text. +6. **Conversation Prompts (.lmlib.prompt)**: Manages conversation prompts, creating new prompts, continuing conversations, and removing end-of-text markers. +7. **Agent Executor (.lmlib.agent)**: Executes agent graphs concurrently, handles agent responses, and tracks request statistics. The program's key features include: @@ -88,7 +88,7 @@ If you use jtokkit tokenizer: jtokkit -It is required to select the right prompt template. See package com.transporeon.exchange.lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context). +It is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context). Set the following property in your model profile: diff --git a/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java b/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java index 47796cc..84835d5 100755 --- a/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java +++ b/src/main/java/de/extio/lmlib/client/WebClientConfiguration.java @@ -19,7 +19,7 @@ import reactor.netty.http.client.HttpClient; @Configuration public class WebClientConfiguration { - private static final int TIMEOUT = 900000; + private static final int TIMEOUT = 300000; private static final int CONNECT_TIMEOUT = 10000; diff --git a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java index fa62d07..35ee50e 100755 --- a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java +++ b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java @@ -1,6 +1,8 @@ package de.extio.lmlib.client.azureai; import java.math.BigDecimal; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayDeque; @@ -26,6 +28,8 @@ import com.azure.ai.inference.models.ChatRequestSystemMessage; import com.azure.ai.inference.models.ChatRequestUserMessage; import com.azure.ai.inference.models.CompletionsFinishReason; import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.http.HttpClient; +import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder; import com.azure.core.http.policy.ExponentialBackoffOptions; import com.azure.core.http.policy.RetryOptions; @@ -35,23 +39,49 @@ import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; -import de.extio.lmlib.profile.ModelProfileService; import de.extio.lmlib.profile.ModelProfile.ModelProvider; +import de.extio.lmlib.profile.ModelProfileService; import de.extio.lmlib.token.Tokenizer; +import okhttp3.Authenticator; +import okhttp3.Challenge; +import okhttp3.Credentials; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.Route; + public final class AzureAiClient implements Client { private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClient.class); + private static final int TIMEOUT = 300000; + private static final int CONNECT_TIMEOUT = 10000; + private final ConcurrentMap chatCompletionsClients = new ConcurrentHashMap<>(); private final ModelProfileService modelProfileService; private final Tokenizer tokenizer; - AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) { + private final boolean proxyEnabled; + + private final String proxyHost; + + private final int proxyPort; + + private final String proxyUser; + + private final String proxyPassword; + + AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer, final boolean proxyEnabled, final String proxyHost, final int proxyPort, final String proxyUser, final String proxyPassword) { this.modelProfileService = modelProfileService; this.tokenizer = tokenizer; + this.proxyEnabled = proxyEnabled; + this.proxyHost = proxyHost; + this.proxyPort = proxyPort; + this.proxyUser = proxyUser; + this.proxyPassword = proxyPassword; } @Override @@ -98,7 +128,7 @@ public final class AzureAiClient implements Client { answers.add(answer); } - // TODO: Reactivate when multiple (split) prompts have been implemented again + // TODO: Reactivate when multiple (split) prompts have been implemented again in #addChatRequestMessage // if (answers.size() > 1) { // this.summarize(answers, modelProfile, promptStrategy, statistics); // } @@ -178,7 +208,8 @@ public final class AzureAiClient implements Client { tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength)); prompt = this.tokenizer.detokenize(tokenized, modelProfile); trimmed = true; - } + } + chat.add(messageFactory.apply(prompt)); return trimmed; @@ -211,21 +242,8 @@ public final class AzureAiClient implements Client { private ChatCompletions requestCompletions(final List chat, final ModelCategory modelCategory, final ModelProfile modelProfile) { LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName()); - final var client = this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> { - final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions() - .setMaxRetries(99) - .setBaseDelay(Duration.ofSeconds(1)) - .setMaxDelay(Duration.ofSeconds(10)); - final RetryOptions retryOptions = new RetryOptions(exponentialOptions); - - final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder() - .credential(new AzureKeyCredential(modelProfile.apiKey())) - .endpoint(modelProfile.url()) - .retryOptions(retryOptions) - .buildClient(); - return chatCompletionsClient; - }); - + final var client = this.createHttpClient(modelCategory, modelProfile); + final var options = new ChatCompletionsOptions(chat); options.setModel(modelProfile.modelName()); options.setMaxTokens(modelProfile.maxTokens()); @@ -235,6 +253,80 @@ public final class AzureAiClient implements Client { return client.complete(options); } + private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) { + return this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> { + final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions() + .setMaxRetries(99) + .setBaseDelay(Duration.ofSeconds(1)) + .setMaxDelay(Duration.ofSeconds(10)); + + final RetryOptions retryOptions = new RetryOptions(exponentialOptions) + .setShouldRetryCondition(httpResponse -> { + if (httpResponse != null) { + if (httpResponse.getThrowable() != null) { + return true; + } + if (httpResponse.getResponse() != null) { + return this.shouldRetryError(httpResponse.getResponse().getStatusCode()); + } + } + return false; + }); + + final OkHttpClient.Builder nativeOkHttpClientBuilder = new OkHttpClient.Builder(); + + if (this.proxyEnabled) { + LOGGER.info("Proxy is enabled. Configuring proxy settings for host: {} port: {} user: {}", this.proxyHost, this.proxyPort, this.proxyUser); + + final Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(this.proxyHost, this.proxyPort)); + nativeOkHttpClientBuilder.proxy(proxy); + + if (this.proxyUser != null && this.proxyPassword != null) { + final Authenticator proxyAuthenticator = new Authenticator() { + @Override + public Request authenticate(final Route route, final Response response) { + for (final Challenge challenge : response.challenges()) { + if (challenge.scheme().equalsIgnoreCase("OkHttp-Preemptive")) { + final String credential = Credentials.basic(proxyUser, proxyPassword); + return response.request().newBuilder() + .header("Proxy-Authorization", credential) + .build(); + } + } + return null; + } + }; + nativeOkHttpClientBuilder.proxyAuthenticator(proxyAuthenticator); + } else { + LOGGER.warn("Proxy user or password is not set. Skipping proxy authentication."); + } + } else { + LOGGER.info("Proxy is not enabled. Skipping proxy configuration."); + } + + final OkHttpClient nativeOkHttpClient = nativeOkHttpClientBuilder.build(); + final OkHttpAsyncHttpClientBuilder okHttpClientBuilder = new OkHttpAsyncHttpClientBuilder(nativeOkHttpClient); + final HttpClient okHttpClient = okHttpClientBuilder + .connectionTimeout(Duration.ofMillis(CONNECT_TIMEOUT)) + .readTimeout(Duration.ofMillis(TIMEOUT)) + .responseTimeout(Duration.ofMillis(TIMEOUT)) + .writeTimeout(Duration.ofMillis(TIMEOUT)) + .build(); + + final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder() + .credential(new AzureKeyCredential(modelProfile.apiKey())) + .endpoint(modelProfile.url()) + .retryOptions(retryOptions) + .httpClient(okHttpClient) + .buildClient(); + return chatCompletionsClient; + }); + } + + private boolean shouldRetryError(final int statusCode) { + return statusCode == 408 || statusCode == 429 || (statusCode >= 500 && statusCode < 600); + } + static class CompletionStatistics { int requests; diff --git a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java old mode 100644 new mode 100755 index 737856a..6399c8f --- a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java +++ b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java @@ -1,5 +1,6 @@ package de.extio.lmlib.client.azureai; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.context.annotation.Bean; @@ -8,12 +9,27 @@ import de.extio.lmlib.profile.ModelProfileService; import de.extio.lmlib.token.Tokenizer; @AutoConfiguration -@ConditionalOnClass(name = "com.azure.ai.inference.ChatCompletionsClient") +@ConditionalOnClass(name = {"com.azure.ai.inference.ChatCompletionsClient", "okhttp3.OkHttpClient" }) public class AzureAiClientAutoConfiguration { + @Value("${extproxy.enabled:false}") + private boolean proxyEnabled; + + @Value("${extproxy.host:}") + private String proxyHost; + + @Value("${extproxy.port:}") + private int proxyPort; + + @Value("${extproxy.username:}") + private String proxyUser; + + @Value("${extproxy.password:}") + private String proxyPassword; + @Bean AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) { - return new AzureAiClient(modelProfileService, tokenizer); + return new AzureAiClient(modelProfileService, tokenizer, this.proxyEnabled, this.proxyHost, this.proxyPort, this.proxyUser, this.proxyPassword); } } diff --git a/src/test/java/de/extio/lmlib/agent/AgentTest.java b/src/test/java/de/extio/lmlib/agent/AgentTest.java index d603c65..0afcafa 100644 --- a/src/test/java/de/extio/lmlib/agent/AgentTest.java +++ b/src/test/java/de/extio/lmlib/agent/AgentTest.java @@ -9,6 +9,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +29,7 @@ import io.netty.util.internal.ThreadLocalRandom; /** * These tests have been verified with Llama 3.1 8B and Gemma2 27B models */ +@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)") @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @ComponentScan(basePackages = "de.extio.lmlib") diff --git a/src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java b/src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java index 9eeae03..ccb54c4 100755 --- a/src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java +++ b/src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java @@ -16,7 +16,7 @@ import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.Conversation.TurnType; import de.extio.lmlib.profile.ModelCategory; -@Disabled +@Disabled("This test requires an Azure subscription (setup azure key in model profile)") @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @ComponentScan(basePackages = "de.extio.lmlib") @@ -35,7 +35,7 @@ public class AzureAiClientTest { "You are a clever story teller", "Continue this story:", "Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!"); - LOGGER.info(completion.response()); + LOGGER.info(completion.toString()); } @Test diff --git a/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java new file mode 100755 index 0000000..06d8b3f --- /dev/null +++ b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java @@ -0,0 +1,106 @@ +package de.extio.lmlib.client.cached; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.PropertySource; + +import de.extio.lmlib.client.ClientService; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.Conversation.Turn; +import de.extio.lmlib.client.Conversation.TurnType; +import de.extio.lmlib.profile.ModelCategory; + +@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)") +@SpringBootTest(webEnvironment = WebEnvironment.NONE) +@SpringBootConfiguration +@ComponentScan(basePackages = "de.extio.lmlib") +@PropertySource("classpath:/application-test.properties") +public class CachedClientTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(CachedClientTest.class); + + @Autowired + private TestCachedClientRepository cachedClientRepository; + + @Autowired + private ClientService clientService; + + @AfterEach + void afterEach() { + this.cachedClientRepository.reset(); + } + + @Test + void cacheCompletion() throws Exception { + var completion = this.clientService.getClient(ModelCategory.MEDIUM).completion( + ModelCategory.MEDIUM, + "You are a funny person", + "Tell me a cat joke", + ""); + LOGGER.info(completion.toString()); + assertFalse(completion.cached()); + assertEquals(1, completion.requests()); + final var firstResponse = completion.response(); + assertFalse(firstResponse.isBlank()); + + completion = this.clientService.getClient(ModelCategory.MEDIUM).completion( + ModelCategory.MEDIUM, + "You are a funny person", + "Tell me a cat joke", + ""); + LOGGER.info(completion.toString()); + assertTrue(completion.cached()); + assertEquals(0, completion.requests()); + assertEquals(firstResponse, completion.response()); + + completion = this.clientService.getClient(ModelCategory.MEDIUM).completion( + ModelCategory.MEDIUM, + "You are a funny person", + "Tell me a dog joke", + ""); + LOGGER.info(completion.toString()); + assertFalse(completion.cached()); + assertEquals(1, completion.requests()); + assertNotEquals(firstResponse, completion.response()); + } + + @Test + void cacheChat() throws Exception { + final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Munich."); + + var completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation); + LOGGER.info(completion.toString()); + assertFalse(completion.cached()); + assertEquals(1, completion.requests()); + final var firstResponse = completion.response(); + assertFalse(firstResponse.isBlank()); + + completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation); + LOGGER.info(completion.toString()); + assertTrue(completion.cached()); + assertEquals(0, completion.requests()); + assertEquals(firstResponse, completion.response()); + + conversation.addTurn(new Turn(TurnType.ASSISTANT, + "T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius.")); + conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?")); + completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation); + LOGGER.info(completion.toString()); + assertFalse(completion.cached()); + assertEquals(1, completion.requests()); + assertNotEquals(firstResponse, completion.response()); + } +} diff --git a/src/test/java/de/extio/lmlib/client/cached/TestCachedClientRepository.java b/src/test/java/de/extio/lmlib/client/cached/TestCachedClientRepository.java new file mode 100755 index 0000000..24d1524 --- /dev/null +++ b/src/test/java/de/extio/lmlib/client/cached/TestCachedClientRepository.java @@ -0,0 +1,27 @@ +package de.extio.lmlib.client.cached; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.springframework.stereotype.Component; + +@Component +class TestCachedClientRepository implements CachedClientRepository { + + private final ConcurrentMap cache = new ConcurrentHashMap<>(); + + @Override + public CachedCompletion get(final String key) { + return this.cache.get(key); + } + + @Override + public void put(final String key, final CachedCompletion completion) { + this.cache.put(key, completion); + } + + void reset() { + this.cache.clear(); + } + +} diff --git a/src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java b/src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java index 284e92a..6268301 100755 --- a/src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java +++ b/src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java @@ -23,6 +23,7 @@ import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.Conversation.TurnType; import de.extio.lmlib.profile.ModelCategory; +@Disabled("This test requires a running Llama server") @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @ComponentScan(basePackages = "de.extio.lmlib") diff --git a/src/test/java/de/extio/lmlib/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/token/TokenizerTest.java index 080fadd..9f03191 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerTest.java @@ -2,6 +2,7 @@ package de.extio.lmlib.token; import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -42,8 +43,9 @@ public class TokenizerTest { } /** - * Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model. + * Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model. */ + @Disabled("Requires llama.cpp model profile") @Test void llamaServer() throws Exception { final var tokenizer = new LlamaServerTokenizer(this.webClientBuilder);