Merged with tp lmlib

README.md

@@ -8,13 +8,13 @@ The program is a comprehensive conversational AI system that provides natural la
 
 The program consists of several packages that work together to achieve this goal. These packages include:
 
-1. **Caching layer (com.transporeon.exchange.lmlib.client.cached)**: Improves performance by reducing requests to the underlying client through caching and retrieving completion responses.
+1. **Caching layer (.lmlib.client.cached)**: Improves performance by reducing requests to the underlying client through caching and retrieving completion responses.
-2. **Natural Language Processing (com.transporeon.exchange.lmlib.client)**: Provides conversation management and text completion functionality, generating completions, processing conversations, and managing models.
+2. **Natural Language Processing (.lmlib.client)**: Provides conversation management and text completion functionality, generating completions, processing conversations, and managing models.
-3. **Azure AI Integration (com.transporeon.exchange.lmlib.client.azureai)**: Integrates with Azure AI services, managing model profiles and handling conversation completions.
+3. **Azure AI Integration (.lmlib.client.azureai)**: Integrates with Azure AI services, managing model profiles and handling conversation completions.
-4. **Text Completion (com.transporeon.exchange.lmlib.client.textcompletion)**: Provides text completion functionality using AI models, including model selection, prompt generation, and response handling.
+4. **Text Completion (.lmlib.client.textcompletion)**: Provides text completion functionality using AI models, including model selection, prompt generation, and response handling.
-5. **Text Tokenization (com.transporeon.exchange.lmlib.token)**: Tokenizes text into numerical tokens and detokenizes numerical tokens back into human-readable text.
+5. **Text Tokenization (.lmlib.token)**: Tokenizes text into numerical tokens and detokenizes numerical tokens back into human-readable text.
-6. **Conversation Prompts (com.transporeon.exchange.lmlib.prompt)**: Manages conversation prompts, creating new prompts, continuing conversations, and removing end-of-text markers.
+6. **Conversation Prompts (.lmlib.prompt)**: Manages conversation prompts, creating new prompts, continuing conversations, and removing end-of-text markers.
-7. **Agent Executor (com.transporeon.exchange.lmlib.agent)**: Executes agent graphs concurrently, handles agent responses, and tracks request statistics.
+7. **Agent Executor (.lmlib.agent)**: Executes agent graphs concurrently, handles agent responses, and tracks request statistics.
 
 The program's key features include:
 
@@ -88,7 +88,7 @@ If you use jtokkit tokenizer:
         <artifactId>jtokkit</artifactId>
     </dependency>
 
-It is required to select the right prompt template. See package com.transporeon.exchange.lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context).
+It is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context).
 
 Set the following property in your model profile:
 

src/main/java/de/extio/lmlib/client/WebClientConfiguration.java

@@ -19,7 +19,7 @@ import reactor.netty.http.client.HttpClient;
 @Configuration
 public class WebClientConfiguration {
 	
-	private static final int TIMEOUT = 900000;
+	private static final int TIMEOUT = 300000;
 	
 	private static final int CONNECT_TIMEOUT = 10000;
 	

src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java

@@ -1,6 +1,8 @@
 package de.extio.lmlib.client.azureai;
 
 import java.math.BigDecimal;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
 import java.time.Duration;
 import java.time.LocalDateTime;
 import java.util.ArrayDeque;
@@ -26,6 +28,8 @@ import com.azure.ai.inference.models.ChatRequestSystemMessage;
 import com.azure.ai.inference.models.ChatRequestUserMessage;
 import com.azure.ai.inference.models.CompletionsFinishReason;
 import com.azure.core.credential.AzureKeyCredential;
+import com.azure.core.http.HttpClient;
+import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder;
 import com.azure.core.http.policy.ExponentialBackoffOptions;
 import com.azure.core.http.policy.RetryOptions;
 
@@ -35,23 +39,49 @@ import de.extio.lmlib.client.Conversation;
 import de.extio.lmlib.client.Conversation.Turn;
 import de.extio.lmlib.profile.ModelCategory;
 import de.extio.lmlib.profile.ModelProfile;
-import de.extio.lmlib.profile.ModelProfileService;
 import de.extio.lmlib.profile.ModelProfile.ModelProvider;
+import de.extio.lmlib.profile.ModelProfileService;
 import de.extio.lmlib.token.Tokenizer;
 
+import okhttp3.Authenticator;
+import okhttp3.Challenge;
+import okhttp3.Credentials;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.Route;
+
 public final class AzureAiClient implements Client {
 	
 	private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClient.class);
 	
+	private static final int TIMEOUT = 300000;
+	private static final int CONNECT_TIMEOUT = 10000;
+	
 	private final ConcurrentMap<ModelCategory, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
 	
 	private final ModelProfileService modelProfileService;
 	
 	private final Tokenizer tokenizer;
 	
-	AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
+	private final boolean proxyEnabled;
+	
+	private final String proxyHost;
+	
+	private final int proxyPort;
+	
+	private final String proxyUser;
+	
+	private final String proxyPassword;
+	
+	AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer, final boolean proxyEnabled, final String proxyHost, final int proxyPort, final String proxyUser, final String proxyPassword) {
 		this.modelProfileService = modelProfileService;
 		this.tokenizer = tokenizer;
+		this.proxyEnabled = proxyEnabled;
+		this.proxyHost = proxyHost;
+		this.proxyPort = proxyPort;
+		this.proxyUser = proxyUser;
+		this.proxyPassword = proxyPassword;
 	}
 	
 	@Override
@@ -98,7 +128,7 @@ public final class AzureAiClient implements Client {
 			answers.add(answer);
 		}
 		
-		// TODO: Reactivate when multiple (split) prompts have been implemented again
+		// TODO: Reactivate when multiple (split) prompts have been implemented again in #addChatRequestMessage
 		//		if (answers.size() > 1) {
 		//			this.summarize(answers, modelProfile, promptStrategy, statistics);
 		//		}
@@ -178,7 +208,8 @@ public final class AzureAiClient implements Client {
 			tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
 			prompt = this.tokenizer.detokenize(tokenized, modelProfile);
 			trimmed = true;
-		}		
+		}
+		
 		chat.add(messageFactory.apply(prompt));
 		
 		return trimmed;
@@ -211,21 +242,8 @@ public final class AzureAiClient implements Client {
 	private ChatCompletions requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
 		LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
 		
-		final var client = this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
+		final var client = this.createHttpClient(modelCategory, modelProfile);
-			final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
+
-					.setMaxRetries(99)
-					.setBaseDelay(Duration.ofSeconds(1))
-					.setMaxDelay(Duration.ofSeconds(10));
-			final RetryOptions retryOptions = new RetryOptions(exponentialOptions);
-			
-			final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
-					.credential(new AzureKeyCredential(modelProfile.apiKey()))
-					.endpoint(modelProfile.url())
-					.retryOptions(retryOptions)
-					.buildClient();
-			return chatCompletionsClient;
-		});
-		
 		final var options = new ChatCompletionsOptions(chat);
 		options.setModel(modelProfile.modelName());
 		options.setMaxTokens(modelProfile.maxTokens());
@@ -235,6 +253,80 @@ public final class AzureAiClient implements Client {
 		return client.complete(options);
 	}
 	
+	private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
+		return this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
+			final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
+					.setMaxRetries(99)
+					.setBaseDelay(Duration.ofSeconds(1))
+					.setMaxDelay(Duration.ofSeconds(10));
+			
+			final RetryOptions retryOptions = new RetryOptions(exponentialOptions)
+					.setShouldRetryCondition(httpResponse -> {
+						if (httpResponse != null) {
+							if (httpResponse.getThrowable() != null) {
+								return true;
+							}
+							if (httpResponse.getResponse() != null) {
+								return this.shouldRetryError(httpResponse.getResponse().getStatusCode());
+							}
+						}
+						return false;
+					});
+			
+			final OkHttpClient.Builder nativeOkHttpClientBuilder = new OkHttpClient.Builder();
+
+			if (this.proxyEnabled) {
+				LOGGER.info("Proxy is enabled. Configuring proxy settings for host: {} port: {} user: {}", this.proxyHost, this.proxyPort, this.proxyUser);
+
+				final Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(this.proxyHost, this.proxyPort));
+				nativeOkHttpClientBuilder.proxy(proxy);
+
+				if (this.proxyUser != null && this.proxyPassword != null) {
+					final Authenticator proxyAuthenticator = new Authenticator() {
+						@Override
+                        public Request authenticate(final Route route, final Response response) {
+							for (final Challenge challenge : response.challenges()) {
+								if (challenge.scheme().equalsIgnoreCase("OkHttp-Preemptive")) {
+									final String credential = Credentials.basic(proxyUser, proxyPassword);
+									return response.request().newBuilder()
+											.header("Proxy-Authorization", credential)
+											.build();
+								}
+							}
+							return null;
+						}
+					};
+					nativeOkHttpClientBuilder.proxyAuthenticator(proxyAuthenticator);
+				} else {
+					LOGGER.warn("Proxy user or password is not set. Skipping proxy authentication.");
+				}
+			} else {
+				LOGGER.info("Proxy is not enabled. Skipping proxy configuration.");
+			}
+
+			final OkHttpClient nativeOkHttpClient = nativeOkHttpClientBuilder.build();
+			final OkHttpAsyncHttpClientBuilder okHttpClientBuilder = new OkHttpAsyncHttpClientBuilder(nativeOkHttpClient);
+			final HttpClient okHttpClient = okHttpClientBuilder
+					.connectionTimeout(Duration.ofMillis(CONNECT_TIMEOUT))
+					.readTimeout(Duration.ofMillis(TIMEOUT))
+					.responseTimeout(Duration.ofMillis(TIMEOUT))
+					.writeTimeout(Duration.ofMillis(TIMEOUT))
+					.build();
+			
+			final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
+					.credential(new AzureKeyCredential(modelProfile.apiKey()))
+					.endpoint(modelProfile.url())
+					.retryOptions(retryOptions)
+					.httpClient(okHttpClient)
+					.buildClient();
+			return chatCompletionsClient;
+		});
+	}
+	
+	private boolean shouldRetryError(final int statusCode) {
+		return statusCode == 408 || statusCode == 429 || (statusCode >= 500 && statusCode < 600);
+	}
+	
 	static class CompletionStatistics {
 		
 		int requests;

src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java

@@ -1,5 +1,6 @@
 package de.extio.lmlib.client.azureai;
 
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.context.annotation.Bean;
@@ -8,12 +9,27 @@ import de.extio.lmlib.profile.ModelProfileService;
 import de.extio.lmlib.token.Tokenizer;
 
 @AutoConfiguration
-@ConditionalOnClass(name = "com.azure.ai.inference.ChatCompletionsClient")
+@ConditionalOnClass(name = {"com.azure.ai.inference.ChatCompletionsClient", "okhttp3.OkHttpClient" })
 public class AzureAiClientAutoConfiguration {
 	
+	@Value("${extproxy.enabled:false}")
+	private boolean proxyEnabled;
+	
+	@Value("${extproxy.host:}")
+	private String proxyHost;
+	
+	@Value("${extproxy.port:}")
+	private int proxyPort;
+	
+	@Value("${extproxy.username:}")
+	private String proxyUser;
+	
+	@Value("${extproxy.password:}")
+	private String proxyPassword;
+	
 	@Bean
 	AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
-		return new AzureAiClient(modelProfileService, tokenizer);
+		return new AzureAiClient(modelProfileService, tokenizer, this.proxyEnabled, this.proxyHost, this.proxyPort, this.proxyUser, this.proxyPassword);
 	}
 	
 }

src/test/java/de/extio/lmlib/agent/AgentTest.java

@@ -9,6 +9,7 @@ import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,6 +29,7 @@ import io.netty.util.internal.ThreadLocalRandom;
 /**
  * These tests have been verified with Llama 3.1 8B and Gemma2 27B models
  */
+@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
 @SpringBootTest(webEnvironment = WebEnvironment.NONE)
 @SpringBootConfiguration
 @ComponentScan(basePackages = "de.extio.lmlib")

src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java

@@ -16,7 +16,7 @@ import de.extio.lmlib.client.Conversation.Turn;
 import de.extio.lmlib.client.Conversation.TurnType;
 import de.extio.lmlib.profile.ModelCategory;
 
-@Disabled
+@Disabled("This test requires an Azure subscription (setup azure key in model profile)")
 @SpringBootTest(webEnvironment = WebEnvironment.NONE)
 @SpringBootConfiguration
 @ComponentScan(basePackages = "de.extio.lmlib")
@@ -35,7 +35,7 @@ public class AzureAiClientTest {
 				"You are a clever story teller",
 				"Continue this story:",
 				"Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
-		LOGGER.info(completion.response());
+		LOGGER.info(completion.toString());
 	}
 	
 	@Test

src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java

@@ -0,0 +1,106 @@
+package de.extio.lmlib.client.cached;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.PropertySource;
+
+import de.extio.lmlib.client.ClientService;
+import de.extio.lmlib.client.Conversation;
+import de.extio.lmlib.client.Conversation.Turn;
+import de.extio.lmlib.client.Conversation.TurnType;
+import de.extio.lmlib.profile.ModelCategory;
+
+@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
+@SpringBootTest(webEnvironment = WebEnvironment.NONE)
+@SpringBootConfiguration
+@ComponentScan(basePackages = "de.extio.lmlib")
+@PropertySource("classpath:/application-test.properties")
+public class CachedClientTest {
+	
+	private static final Logger LOGGER = LoggerFactory.getLogger(CachedClientTest.class);
+	
+	@Autowired
+	private TestCachedClientRepository cachedClientRepository;
+	
+	@Autowired
+	private ClientService clientService;
+	
+	@AfterEach
+	void afterEach() {
+		this.cachedClientRepository.reset();
+	}
+	
+	@Test
+	void cacheCompletion() throws Exception {
+		var completion = this.clientService.getClient(ModelCategory.MEDIUM).completion(
+				ModelCategory.MEDIUM,
+				"You are a funny person",
+				"Tell me a cat joke",
+				"");
+		LOGGER.info(completion.toString());
+		assertFalse(completion.cached());
+		assertEquals(1, completion.requests());
+		final var firstResponse = completion.response();
+		assertFalse(firstResponse.isBlank());
+		
+		completion = this.clientService.getClient(ModelCategory.MEDIUM).completion(
+				ModelCategory.MEDIUM,
+				"You are a funny person",
+				"Tell me a cat joke",
+				"");
+		LOGGER.info(completion.toString());
+		assertTrue(completion.cached());
+		assertEquals(0, completion.requests());
+		assertEquals(firstResponse, completion.response());
+		
+		completion = this.clientService.getClient(ModelCategory.MEDIUM).completion(
+				ModelCategory.MEDIUM,
+				"You are a funny person",
+				"Tell me a dog joke",
+				"");
+		LOGGER.info(completion.toString());
+		assertFalse(completion.cached());
+		assertEquals(1, completion.requests());
+		assertNotEquals(firstResponse, completion.response());
+	}
+	
+	@Test
+	void cacheChat() throws Exception {
+		final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Munich.");
+		
+		var completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
+		LOGGER.info(completion.toString());
+		assertFalse(completion.cached());
+		assertEquals(1, completion.requests());
+		final var firstResponse = completion.response();
+		assertFalse(firstResponse.isBlank());
+		
+		completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
+		LOGGER.info(completion.toString());
+		assertTrue(completion.cached());
+		assertEquals(0, completion.requests());
+		assertEquals(firstResponse, completion.response());
+		
+		conversation.addTurn(new Turn(TurnType.ASSISTANT,
+				"T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius."));
+		conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?"));
+		completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
+		LOGGER.info(completion.toString());
+		assertFalse(completion.cached());
+		assertEquals(1, completion.requests());
+		assertNotEquals(firstResponse, completion.response());
+	}
+}

src/test/java/de/extio/lmlib/client/cached/TestCachedClientRepository.java

@@ -0,0 +1,27 @@
+package de.extio.lmlib.client.cached;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.springframework.stereotype.Component;
+
+@Component
+class TestCachedClientRepository implements CachedClientRepository {
+	
+	private final ConcurrentMap<String, CachedCompletion> cache = new ConcurrentHashMap<>();
+	
+	@Override
+	public CachedCompletion get(final String key) {
+		return this.cache.get(key);
+	}
+	
+	@Override
+	public void put(final String key, final CachedCompletion completion) {
+		this.cache.put(key, completion);
+	}
+	
+	void reset() {
+		this.cache.clear();
+	}
+	
+}

src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java

@@ -23,6 +23,7 @@ import de.extio.lmlib.client.Conversation.Turn;
 import de.extio.lmlib.client.Conversation.TurnType;
 import de.extio.lmlib.profile.ModelCategory;
 
+@Disabled("This test requires a running Llama server")
 @SpringBootTest(webEnvironment = WebEnvironment.NONE)
 @SpringBootConfiguration
 @ComponentScan(basePackages = "de.extio.lmlib")

src/test/java/de/extio/lmlib/token/TokenizerTest.java

@@ -2,6 +2,7 @@ package de.extio.lmlib.token;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -42,8 +43,9 @@ public class TokenizerTest {
 	}
 	
 	/**
-	 * Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model. 
+	 * Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model.
 	 */
+	@Disabled("Requires llama.cpp model profile")
 	@Test
 	void llamaServer() throws Exception {
 		final var tokenizer = new LlamaServerTokenizer(this.webClientBuilder);