exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance agent functionality with ModelProfile integration

Browse Source 
- Added ModelProfile to the Agent record to allow for more detailed model configurations.
- Updated AgentExecutorService to utilize ModelProfile when fetching clients.
- Modified BaseAgent interface to include a default modelProfile method.
- Adjusted Client interface and implementations to support ModelProfile in conversation and streaming methods.
- Enhanced ClientService to retrieve clients based on ModelProfile.
- Updated CompletionInterceptor to work with ModelProfile instead of ModelCategory.
- Refactored AzureAiClient and other client implementations to handle ModelProfile correctly.
- Changed CachedClient to cache responses based on ModelProfile.
- Updated ModelCategory to a record type for better validation and immutability.
- Enhanced ModelProfile with validation checks for its fields.
- Updated ModelProfileService to support fetching profiles based on ModelCategory.
- Adjusted tests to accommodate changes in Agent and ModelProfile.
- Updated properties files to include category information for model profiles.
This commit is contained in:
extio-de
2025-10-29 12:07:36 +01:00
parent c5dca3ac07
commit 304924e223
20 changed files with 452 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/de/extio/lmlib/client/Client.java
View File

@@ -3,6 +3,7 @@ package de.extio.lmlib.client;
import java.util.function.Consumer;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
public interface Client {
@@ -14,8 +15,12 @@ public interface Client {
Completion completion(ModelCategory modelCategory, String system, String text);
Completion conversation(ModelCategory modelCategory, Conversation conversation);
Completion conversation(ModelProfile modelProfile, Conversation conversation);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer);
Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer<Chunk> chunkConsumer);
ModelProvider getModelProvider();
}

56
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -10,6 +10,7 @@ import org.springframework.stereotype.Service;
import de.extio.lmlib.client.cached.CachedClient;
import de.extio.lmlib.client.cached.CachedClientRepository;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
@@ -28,25 +29,32 @@ public class ClientService {
@Autowired(required = false)
private List<CompletionInterceptor> completionInterceptors;
@Cacheable("clients")
public Client getClient(final ModelCategory category) {
final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile());
final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile(), category);
return this.getClient(modelProfile);
}
@Cacheable("clients")
public Client getClient(final ModelProfile modelProfile) {
return this.clients.stream()
.filter(client -> client.getModelProvider() == modelProfile.modelProvider())
.findFirst()
.map(client -> this.cachedClientRepository == null ? client : new CachedClient(this.cachedClientRepository, this.modelProfileService, client))
.map(client -> this.completionInterceptors == null || this.completionInterceptors.isEmpty() ? client : new InterceptingClient(client, this.completionInterceptors))
.orElseThrow(() -> new IllegalStateException("No client found for model profile " + category));
.map(client -> this.completionInterceptors == null || this.completionInterceptors.isEmpty() ? client : new InterceptingClient(client, this.modelProfileService, this.completionInterceptors))
.orElseThrow(() -> new IllegalStateException("No client found for model profile " + modelProfile.modelName()));
}
private static class InterceptingClient implements Client {
private final Client client;
private final ModelProfileService modelProfileService;
private final List<CompletionInterceptor> completionInterceptors;
public InterceptingClient(final Client client, final List<CompletionInterceptor> completionInterceptors) {
public InterceptingClient(final Client client, final ModelProfileService modelProfileService, final List<CompletionInterceptor> completionInterceptors) {
this.client = client;
this.modelProfileService = modelProfileService;
this.completionInterceptors = completionInterceptors;
}
@@ -57,26 +65,38 @@ public class ClientService {
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelCategory, conversation);
}
var completion = this.client.conversation(modelCategory, conversation);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelCategory, completion);
}
return completion;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.conversation(modelProfile, conversation_);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation_, chunkConsumer);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelCategory, conversation);
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.streamConversation(modelCategory, conversation, chunkConsumer);
var completion = this.client.conversation(modelProfile, conversation);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelCategory, completion);
completion = interceptor.after(modelProfile, completion);
}
return completion;
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelProfile, completion);
}
return completion;
}

6
src/main/java/de/extio/lmlib/client/CompletionInterceptor.java
View File

@@ -1,11 +1,11 @@
package de.extio.lmlib.client;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
public interface CompletionInterceptor {
Conversation before(ModelCategory modelCategory, Conversation conversation);
Conversation before(ModelProfile modelProfile, Conversation conversation);
Completion after(ModelCategory modelCategory, Completion completion);
Completion after(ModelProfile modelProfile, Completion completion);
}

66
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -55,7 +55,7 @@ public final class AzureAiClient implements Client {
private static final int CONNECT_TIMEOUT = 10000;
private final ConcurrentMap<ModelCategory, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
private final ConcurrentMap<String, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
private final ModelProfileService modelProfileService;
@@ -93,12 +93,22 @@ public final class AzureAiClient implements Client {
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.requestConversation(modelCategory, conversation, null);
return this.requestConversation(modelCategory, null, conversation, null);
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.requestConversation(null, modelProfile, conversation, null);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
return this.requestConversation(modelCategory, conversation, chunkConsumer);
return this.requestConversation(modelCategory, null, conversation, chunkConsumer);
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
return this.requestConversation(null, modelProfile, conversation, chunkConsumer);
}
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
@@ -172,31 +182,41 @@ public final class AzureAiClient implements Client {
return trimmed;
}
private Completion requestConversation(ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
final var chat = this.createChats(conversation, modelProfile);
if (chunkConsumer == null) {
return this.requestCompletions(chat, modelCategory, modelProfile);
private Completion requestConversation(ModelCategory modelCategory, final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final ModelProfile profile;
if (modelProfile != null) {
profile = modelProfile;
if (modelCategory == null) {
modelCategory = new ModelCategory(modelProfile.modelName(), modelProfile.category());
}
}
else {
return this.requestStreamCompletions(chat, modelCategory, modelProfile, chunkConsumer);
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
profile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
}
if (profile == null || profile.modelProvider() != ModelProvider.AZURE_AI) {
throw new IllegalArgumentException("Invalid ModelProfile");
}
final var chat = this.createChats(conversation, profile);
if (chunkConsumer == null) {
return this.requestCompletions(chat, profile);
}
else {
return this.requestStreamCompletions(chat, profile, chunkConsumer);
}
}
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile) {
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
final LocalDateTime start = LocalDateTime.now();
final var options = this.createChatCompletionsOptions(chat, modelProfile);
final var client = this.createHttpClient(modelCategory, modelProfile);
final var client = this.createHttpClient(modelProfile);
final var response = client.complete(options);
final var finishReason = this.mapFinishReason(response.getChoice().getFinishReason());
@@ -205,12 +225,12 @@ public final class AzureAiClient implements Client {
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
}
private Completion requestStreamCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
private Completion requestStreamCompletions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
LOGGER.debug("Requesting Streaming ChatCompletion at {}", modelProfile.modelName());
final LocalDateTime start = LocalDateTime.now();
final var options = this.createChatCompletionsOptions(chat, modelProfile);
final var client = this.createHttpClient(modelCategory, modelProfile);
final var client = this.createHttpClient(modelProfile);
final var completionsStream = client.completeStream(options);
final StringBuilder contentBuilder = new StringBuilder();
@@ -277,8 +297,8 @@ public final class AzureAiClient implements Client {
return statistics;
}
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
return this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
private ChatCompletionsClient createHttpClient(final ModelProfile modelProfile) {
return this.chatCompletionsClients.computeIfAbsent(modelProfile.category(), key -> {
final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
.setMaxRetries(99)
.setBaseDelay(Duration.ofSeconds(1))

43
src/main/java/de/extio/lmlib/client/cached/CachedClient.java
View File

@@ -17,6 +17,7 @@ import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.CompletionStatistics;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
@@ -36,17 +37,31 @@ public class CachedClient implements Client {
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.getCachedResponse(modelCategory, () -> this.client.completion(modelCategory, system, text), system, text);
return this.getCachedResponse(modelCategory, null, () -> this.client.completion(modelCategory, system, text), system, text);
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.getCachedResponse(modelCategory, () -> this.client.conversation(modelCategory, conversation), conversation.getConversation().toString());
return this.getCachedResponse(modelCategory, null, () -> this.client.conversation(modelCategory, conversation), conversation.getConversation().toString());
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.getCachedResponse(null, modelProfile, () -> this.client.conversation(modelProfile, conversation), conversation.getConversation().toString());
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(modelCategory, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
final var response = this.getCachedResponse(modelCategory, null, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
return response;
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(null, modelProfile, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
@@ -58,8 +73,8 @@ public class CachedClient implements Client {
return this.client.getModelProvider();
}
private Completion getCachedResponse(final ModelCategory modelCategory, final Supplier<Completion> supplier, final String... keys) {
final var key = this.createCacheKey(modelCategory, keys);
private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final Supplier<Completion> supplier, final String... keys) {
final var key = this.createCacheKey(modelCategory, modelProfile, keys);
final var cachedCompletion = this.cacheRepository.get(key);
if (cachedCompletion != null) {
@@ -73,23 +88,25 @@ public class CachedClient implements Client {
return completion;
}
private String createCacheKey(ModelCategory modelCategory, final String... keys) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
private String createCacheKey(ModelCategory modelCategory, ModelProfile modelProfile, final String... keys) {
if (modelProfile == null) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
if (modelProfile == null) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
}
MessageDigest digest;
try {
digest = MessageDigest.getInstance("SHA-256");
digest.update(Objects.requireNonNullElse(modelProfile.modelName(), modelCategory.name()).getBytes());
digest.update(modelProfile.modelName().getBytes());
digest.update(modelCategory != null ? modelCategory.getModelProfile().getBytes() : modelProfile.category().getBytes());
digest.update(String.valueOf(modelProfile.modelProvider()).getBytes());
digest.update(String.valueOf(modelProfile.maxTokens()).getBytes());
digest.update(String.valueOf(modelProfile.maxContextLength()).getBytes());
digest.update(String.valueOf("1").getBytes());
digest.update(String.valueOf(modelProfile.temperature()).getBytes());
digest.update(String.valueOf(modelProfile.topP()).getBytes());
for (final var k : keys) {

29
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
View File

@@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
@@ -93,17 +92,27 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
}
@Override
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
return this.requestCompletion(conversation, modelCategory, modelProfile, chunkConsumer);
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.streamConversation(modelProfile, conversation, null);
}
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer);
@Override
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation, chunkConsumer);
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("Invalid ModelProfile");
}
return this.requestCompletion(conversation, modelProfile, chunkConsumer);
}
protected abstract Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer);
protected CompletionFinishReason mapFinishReason(final String finishReason) {
return switch (finishReason) {

2
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java
View File

@@ -40,7 +40,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
final var chat = createChats(conversation, modelProfile);
final var request = new ChatCompletionRequest();

9
src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java
View File

@@ -20,7 +20,6 @@ import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.prompt.PromptStrategy;
@@ -41,13 +40,13 @@ public class TextCompletionClient extends AbstractCompletionClient {
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
if (promptStrategy == null) {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
}
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy);
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
@@ -177,7 +176,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
return new Completion(content, reasoning, finishReason, statistics);
}
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
@@ -188,7 +187,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
}
LOGGER.debug("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
LOGGER.debug("Completion request for {}. Input tokens: {}", modelProfile.category(), tokenized.size());
return prompt.toString();
}