exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance agent functionality with ModelProfile integration

Browse Source 
- Added ModelProfile to the Agent record to allow for more detailed model configurations.
- Updated AgentExecutorService to utilize ModelProfile when fetching clients.
- Modified BaseAgent interface to include a default modelProfile method.
- Adjusted Client interface and implementations to support ModelProfile in conversation and streaming methods.
- Enhanced ClientService to retrieve clients based on ModelProfile.
- Updated CompletionInterceptor to work with ModelProfile instead of ModelCategory.
- Refactored AzureAiClient and other client implementations to handle ModelProfile correctly.
- Changed CachedClient to cache responses based on ModelProfile.
- Updated ModelCategory to a record type for better validation and immutability.
- Enhanced ModelProfile with validation checks for its fields.
- Updated ModelProfileService to support fetching profiles based on ModelCategory.
- Adjusted tests to accommodate changes in Agent and ModelProfile.
- Updated properties files to include category information for model profiles.
This commit is contained in:
extio-de
2025-10-29 12:07:36 +01:00
parent c5dca3ac07
commit 304924e223
20 changed files with 452 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/de/extio/lmlib/agent/Agent.java
View File

@@ -6,10 +6,12 @@ import java.util.function.Function;
import de.extio.lmlib.agent.responsehandler.AgentResponseHandler;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
public record Agent(String name,
AgentType agentType,
ModelCategory modelCategory,
ModelProfile modelProfile,
String systemPrompt,
String textTemplate,
AgentResponseHandler responseHandler,
@@ -44,6 +46,14 @@ public record Agent(String name,
return BaseAgent.super.modelCategory(context);
}
@Override
public ModelProfile modelProfile(final AgentContext context) {
if (modelProfile != null) {
return modelProfile;
}
return BaseAgent.super.modelProfile(context);
}
@Override
public void preProcess(final AgentContext context) {
if (preProcessor != null) {

8
src/main/java/de/extio/lmlib/agent/AgentExecutorService.java
View File

@@ -82,7 +82,13 @@ public class AgentExecutorService implements InitializingBean, DisposableBean {
Client client = null;
if (agent.agentType(context) != AgentType.PROCESSING_ONLY) {
client = this.clientService.getClient(agent.modelCategory(context));
final var modelProfile = agent.modelProfile(context);
if (modelProfile != null) {
client = this.clientService.getClient(modelProfile);
}
else {
client = this.clientService.getClient(agent.modelCategory(context));
}
}
context.setAgentContextUpdateConsumer(agentContextUpdateConsumer);

28
src/main/java/de/extio/lmlib/agent/BaseAgent.java
View File

@@ -6,6 +6,7 @@ import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@@ -17,11 +18,13 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import de.extio.lmlib.agent.responsehandler.AgentResponseHandler;
import de.extio.lmlib.agent.responsehandler.StreamedAgentResponseHandler;
import de.extio.lmlib.agent.responsehandler.TextAgentResponseHandler;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
public interface BaseAgent {
@@ -40,6 +43,10 @@ public interface BaseAgent {
default ModelCategory modelCategory(final AgentContext context) {
return ModelCategory.MEDIUM;
}
default ModelProfile modelProfile(final AgentContext context) {
return null;
}
default String systemPrompt() {
return null;
@@ -90,7 +97,8 @@ public interface BaseAgent {
split.context().getGraph().add("○");
}
else {
split.context().getGraph().add(this.modelCategory(split.context()).getShortName());
final var modelProfile = this.modelProfile(split.context());
split.context().getGraph().add(modelProfile != null ? modelProfile.category() : this.modelCategory(split.context()).getShortName());
}
split.context().getGraph().add(this.name());
@@ -102,21 +110,33 @@ public interface BaseAgent {
LOGGER.debug("Conversation: {}", conversation);
final var responseHandler = this.responseHandler();
final var modelProfile = this.modelProfile(split.context());
Completion completion = null;
if (split.context().isStreaming()) {
if (responseHandler instanceof final StreamedAgentResponseHandler streamedResponseHandler) {
streamedResponseHandler.beforeStream(split.context());
}
completion = client.streamConversation(this.modelCategory(split.context()), conversation, chunk -> {
final Consumer<Chunk> chunkConsumer = chunk -> {
if (responseHandler instanceof final StreamedAgentResponseHandler streamedResponseHandler) {
if (streamedResponseHandler.handleChunk(chunk, split.context()) && split.context().getAgentContextUpdateConsumer() != null) {
split.context().getAgentContextUpdateConsumer().accept(split.context());
}
}
});
};
if (modelProfile != null) {
completion = client.streamConversation(modelProfile, conversation, chunkConsumer);
}
else {
completion = client.streamConversation(this.modelCategory(split.context()), conversation, chunkConsumer);
}
}
else {
completion = client.conversation(this.modelCategory(split.context()), conversation);
if (modelProfile != null) {
completion = client.conversation(modelProfile, conversation);
}
else {
completion = client.conversation(this.modelCategory(split.context()), conversation);
}
}
split.context().setLastCompletion(completion);
split.context().getRequestStatistic().add(completion);

5
src/main/java/de/extio/lmlib/client/Client.java
View File

@@ -3,6 +3,7 @@ package de.extio.lmlib.client;
import java.util.function.Consumer;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
public interface Client {
@@ -14,8 +15,12 @@ public interface Client {
Completion completion(ModelCategory modelCategory, String system, String text);
Completion conversation(ModelCategory modelCategory, Conversation conversation);
Completion conversation(ModelProfile modelProfile, Conversation conversation);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer);
Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer<Chunk> chunkConsumer);
ModelProvider getModelProvider();
}

56
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -10,6 +10,7 @@ import org.springframework.stereotype.Service;
import de.extio.lmlib.client.cached.CachedClient;
import de.extio.lmlib.client.cached.CachedClientRepository;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
@@ -28,25 +29,32 @@ public class ClientService {
@Autowired(required = false)
private List<CompletionInterceptor> completionInterceptors;
@Cacheable("clients")
public Client getClient(final ModelCategory category) {
final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile());
final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile(), category);
return this.getClient(modelProfile);
}
@Cacheable("clients")
public Client getClient(final ModelProfile modelProfile) {
return this.clients.stream()
.filter(client -> client.getModelProvider() == modelProfile.modelProvider())
.findFirst()
.map(client -> this.cachedClientRepository == null ? client : new CachedClient(this.cachedClientRepository, this.modelProfileService, client))
.map(client -> this.completionInterceptors == null || this.completionInterceptors.isEmpty() ? client : new InterceptingClient(client, this.completionInterceptors))
.orElseThrow(() -> new IllegalStateException("No client found for model profile " + category));
.map(client -> this.completionInterceptors == null || this.completionInterceptors.isEmpty() ? client : new InterceptingClient(client, this.modelProfileService, this.completionInterceptors))
.orElseThrow(() -> new IllegalStateException("No client found for model profile " + modelProfile.modelName()));
}
private static class InterceptingClient implements Client {
private final Client client;
private final ModelProfileService modelProfileService;
private final List<CompletionInterceptor> completionInterceptors;
public InterceptingClient(final Client client, final List<CompletionInterceptor> completionInterceptors) {
public InterceptingClient(final Client client, final ModelProfileService modelProfileService, final List<CompletionInterceptor> completionInterceptors) {
this.client = client;
this.modelProfileService = modelProfileService;
this.completionInterceptors = completionInterceptors;
}
@@ -57,26 +65,38 @@ public class ClientService {
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelCategory, conversation);
}
var completion = this.client.conversation(modelCategory, conversation);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelCategory, completion);
}
return completion;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.conversation(modelProfile, conversation_);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation_, chunkConsumer);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelCategory, conversation);
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.streamConversation(modelCategory, conversation, chunkConsumer);
var completion = this.client.conversation(modelProfile, conversation);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelCategory, completion);
completion = interceptor.after(modelProfile, completion);
}
return completion;
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelProfile, completion);
}
return completion;
}

6
src/main/java/de/extio/lmlib/client/CompletionInterceptor.java
View File

@@ -1,11 +1,11 @@
package de.extio.lmlib.client;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
public interface CompletionInterceptor {
Conversation before(ModelCategory modelCategory, Conversation conversation);
Conversation before(ModelProfile modelProfile, Conversation conversation);
Completion after(ModelCategory modelCategory, Completion completion);
Completion after(ModelProfile modelProfile, Completion completion);
}

66
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -55,7 +55,7 @@ public final class AzureAiClient implements Client {
private static final int CONNECT_TIMEOUT = 10000;
private final ConcurrentMap<ModelCategory, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
private final ConcurrentMap<String, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
private final ModelProfileService modelProfileService;
@@ -93,12 +93,22 @@ public final class AzureAiClient implements Client {
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.requestConversation(modelCategory, conversation, null);
return this.requestConversation(modelCategory, null, conversation, null);
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.requestConversation(null, modelProfile, conversation, null);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
return this.requestConversation(modelCategory, conversation, chunkConsumer);
return this.requestConversation(modelCategory, null, conversation, chunkConsumer);
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
return this.requestConversation(null, modelProfile, conversation, chunkConsumer);
}
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
@@ -172,31 +182,41 @@ public final class AzureAiClient implements Client {
return trimmed;
}
private Completion requestConversation(ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
final var chat = this.createChats(conversation, modelProfile);
if (chunkConsumer == null) {
return this.requestCompletions(chat, modelCategory, modelProfile);
private Completion requestConversation(ModelCategory modelCategory, final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final ModelProfile profile;
if (modelProfile != null) {
profile = modelProfile;
if (modelCategory == null) {
modelCategory = new ModelCategory(modelProfile.modelName(), modelProfile.category());
}
}
else {
return this.requestStreamCompletions(chat, modelCategory, modelProfile, chunkConsumer);
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
profile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
}
if (profile == null || profile.modelProvider() != ModelProvider.AZURE_AI) {
throw new IllegalArgumentException("Invalid ModelProfile");
}
final var chat = this.createChats(conversation, profile);
if (chunkConsumer == null) {
return this.requestCompletions(chat, profile);
}
else {
return this.requestStreamCompletions(chat, profile, chunkConsumer);
}
}
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile) {
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
final LocalDateTime start = LocalDateTime.now();
final var options = this.createChatCompletionsOptions(chat, modelProfile);
final var client = this.createHttpClient(modelCategory, modelProfile);
final var client = this.createHttpClient(modelProfile);
final var response = client.complete(options);
final var finishReason = this.mapFinishReason(response.getChoice().getFinishReason());
@@ -205,12 +225,12 @@ public final class AzureAiClient implements Client {
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
}
private Completion requestStreamCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
private Completion requestStreamCompletions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
LOGGER.debug("Requesting Streaming ChatCompletion at {}", modelProfile.modelName());
final LocalDateTime start = LocalDateTime.now();
final var options = this.createChatCompletionsOptions(chat, modelProfile);
final var client = this.createHttpClient(modelCategory, modelProfile);
final var client = this.createHttpClient(modelProfile);
final var completionsStream = client.completeStream(options);
final StringBuilder contentBuilder = new StringBuilder();
@@ -277,8 +297,8 @@ public final class AzureAiClient implements Client {
return statistics;
}
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
return this.chatCompletionsClients.computeIfAbsent(modelCategory, key -> {
private ChatCompletionsClient createHttpClient(final ModelProfile modelProfile) {
return this.chatCompletionsClients.computeIfAbsent(modelProfile.category(), key -> {
final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
.setMaxRetries(99)
.setBaseDelay(Duration.ofSeconds(1))

43
src/main/java/de/extio/lmlib/client/cached/CachedClient.java
View File

@@ -17,6 +17,7 @@ import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.CompletionStatistics;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
@@ -36,17 +37,31 @@ public class CachedClient implements Client {
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.getCachedResponse(modelCategory, () -> this.client.completion(modelCategory, system, text), system, text);
return this.getCachedResponse(modelCategory, null, () -> this.client.completion(modelCategory, system, text), system, text);
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.getCachedResponse(modelCategory, () -> this.client.conversation(modelCategory, conversation), conversation.getConversation().toString());
return this.getCachedResponse(modelCategory, null, () -> this.client.conversation(modelCategory, conversation), conversation.getConversation().toString());
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.getCachedResponse(null, modelProfile, () -> this.client.conversation(modelProfile, conversation), conversation.getConversation().toString());
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(modelCategory, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
final var response = this.getCachedResponse(modelCategory, null, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
return response;
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(null, modelProfile, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
@@ -58,8 +73,8 @@ public class CachedClient implements Client {
return this.client.getModelProvider();
}
private Completion getCachedResponse(final ModelCategory modelCategory, final Supplier<Completion> supplier, final String... keys) {
final var key = this.createCacheKey(modelCategory, keys);
private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final Supplier<Completion> supplier, final String... keys) {
final var key = this.createCacheKey(modelCategory, modelProfile, keys);
final var cachedCompletion = this.cacheRepository.get(key);
if (cachedCompletion != null) {
@@ -73,23 +88,25 @@ public class CachedClient implements Client {
return completion;
}
private String createCacheKey(ModelCategory modelCategory, final String... keys) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
private String createCacheKey(ModelCategory modelCategory, ModelProfile modelProfile, final String... keys) {
if (modelProfile == null) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
if (modelProfile == null) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
}
MessageDigest digest;
try {
digest = MessageDigest.getInstance("SHA-256");
digest.update(Objects.requireNonNullElse(modelProfile.modelName(), modelCategory.name()).getBytes());
digest.update(modelProfile.modelName().getBytes());
digest.update(modelCategory != null ? modelCategory.getModelProfile().getBytes() : modelProfile.category().getBytes());
digest.update(String.valueOf(modelProfile.modelProvider()).getBytes());
digest.update(String.valueOf(modelProfile.maxTokens()).getBytes());
digest.update(String.valueOf(modelProfile.maxContextLength()).getBytes());
digest.update(String.valueOf("1").getBytes());
digest.update(String.valueOf(modelProfile.temperature()).getBytes());
digest.update(String.valueOf(modelProfile.topP()).getBytes());
for (final var k : keys) {

29
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
View File

@@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Chunk;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
@@ -93,17 +92,27 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
}
@Override
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
return this.requestCompletion(conversation, modelCategory, modelProfile, chunkConsumer);
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.streamConversation(modelProfile, conversation, null);
}
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer);
@Override
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation, chunkConsumer);
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("Invalid ModelProfile");
}
return this.requestCompletion(conversation, modelProfile, chunkConsumer);
}
protected abstract Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer);
protected CompletionFinishReason mapFinishReason(final String finishReason) {
return switch (finishReason) {

2
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java
View File

@@ -40,7 +40,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
final var chat = createChats(conversation, modelProfile);
final var request = new ChatCompletionRequest();

9
src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java
View File

@@ -20,7 +20,6 @@ import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.prompt.PromptStrategy;
@@ -41,13 +40,13 @@ public class TextCompletionClient extends AbstractCompletionClient {
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
if (promptStrategy == null) {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
}
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy);
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
@@ -177,7 +176,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
return new Completion(content, reasoning, finishReason, statistics);
}
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
@@ -188,7 +187,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
}
LOGGER.debug("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
LOGGER.debug("Completion request for {}. Input tokens: {}", modelProfile.category(), tokenized.size());
return prompt.toString();
}

1
src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java
View File

@@ -15,6 +15,7 @@ public class AnswerGrader {
"AnswerGrader",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"""
You are a grader agent that evaluates the quality of an answer to a support question.
Please provide a response on a scale between 0 and 100, where: 100 indicates that the answer directly addresses the question, is accurate, and provides a clear and concise solution or explanation; 90-99 indicates that the answer is mostly accurate and provides a good solution or explanation, but may lack some detail or clarity; 80-89 indicates that the answer is generally accurate but may require some additional information or clarification to fully understand the solution or explanation; 75 indicates that the answer describes how the system behaves to get to the subject of the question, but may not directly answer the question; 50-74 indicates that the answer is partially accurate or relevant, but may lack key information or provide an incomplete solution or explanation; 25-49 indicates that the answer does not directly answer the question or is speculative or it contains mainly instructions for changes or mainly refers to the Enterprise Integration team without providing substantial information, but may provide some relevant information or context; 0-24 indicates that the answer is not related to the question, is inaccurate, or provides no useful information;

32
src/main/java/de/extio/lmlib/profile/ModelCategory.java
View File

@@ -1,23 +1,23 @@
package de.extio.lmlib.profile;
public enum ModelCategory {
public record ModelCategory(String modelProfile, String shortName) {
public static final ModelCategory SMALL = new ModelCategory("profile.model.s", "S");
public static final ModelCategory MEDIUM = new ModelCategory("profile.model.m", "M");
public static final ModelCategory LARGE = new ModelCategory("profile.model.l", "L");
public static final ModelCategory XLARGE = new ModelCategory("profile.model.xl", "XL");
public static final ModelCategory HOT = new ModelCategory("profile.model.hot", "HOT");
public static final ModelCategory COLD = new ModelCategory("profile.model.cold", "COLD");
SMALL("profile.model.s", "S"),
MEDIUM("profile.model.m", "M"),
LARGE("profile.model.l", "L"),
XLARGE("profile.model.xl", "XL"),
HOT("profile.model.hot", "HOT"),
COLD("profile.model.cold", "COLD");
private final String modelProfile;
private final String shortName;
ModelCategory(final String modelProfile, final String shortName) {
this.modelProfile = modelProfile;
this.shortName = shortName;
public ModelCategory {
if (modelProfile == null || modelProfile.isEmpty()) {
throw new IllegalArgumentException("modelProfile must not be null or empty");
}
if (shortName == null || shortName.isEmpty()) {
throw new IllegalArgumentException("shortName must not be null or empty");
}
}
public String getModelProfile() {
return this.modelProfile;
}

27
src/main/java/de/extio/lmlib/profile/ModelProfile.java
View File

@@ -16,7 +16,32 @@ public record ModelProfile(
BigDecimal costInToken,
BigDecimal costOutToken,
String reasoningEffort,
String reasoningSummaryDetails) {
String reasoningSummaryDetails,
String category) {
public ModelProfile {
if (maxTokens <= 0) {
throw new IllegalArgumentException("Model profile maxTokens must be greater than zero");
}
if (maxContextLength <= 0) {
throw new IllegalArgumentException("Model profile maxContextLength must be greater than zero");
}
if (temperature <= 0) {
throw new IllegalArgumentException("Model profile temperature must be greater than zero");
}
if (topP <= 0) {
throw new IllegalArgumentException("Model profile topP must be greater than zero");
}
if (modelProvider == null) {
throw new IllegalArgumentException("Model profile modelProvider is mandatory and cannot be null");
}
if (url == null || url.isBlank()) {
throw new IllegalArgumentException("Model profile url is mandatory and cannot be empty");
}
if (category == null || category.isBlank()) {
throw new IllegalArgumentException("Model profile category is mandatory and cannot be empty");
}
}
public static enum ModelProvider {
AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION;

22
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -33,8 +33,12 @@ public class ModelProfileService {
this.environment = environment;
}
@Cacheable("modelProfiles")
public ModelProfile getModelProfile(final String modelProfile) {
return this.getModelProfile(modelProfile, null);
}
@Cacheable("modelProfiles")
public ModelProfile getModelProfile(final String modelProfile, final ModelCategory modelCategory) {
final var modelName = this.environment.getProperty(modelProfile);
if (modelName == null) {
throw new IllegalStateException("No model found for profile: " + modelProfile);
@@ -45,7 +49,7 @@ public class ModelProfileService {
final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model name found for model: " + modelName));
.orElse("");
final var tokenEncoding = Optional.ofNullable(resource.getProperty("tokenEncoding"))
.map(Object::toString)
.orElse("none");
@@ -71,7 +75,7 @@ public class ModelProfileService {
.orElseThrow(() -> new IllegalStateException("No model modelProvider found for model: " + modelName));
final var modelNameCfg = Optional.ofNullable(resource.getProperty("modelName"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model modelName found for model: " + modelName));
.orElse(null);
final var url = Optional.ofNullable(resource.getProperty("url"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model url found for model: " + modelName));
@@ -91,6 +95,10 @@ public class ModelProfileService {
final var reasoningSummaryDetails = Optional.ofNullable(resource.getProperty("reasoningSummaryDetails"))
.map(Object::toString)
.orElse(null);
final var category = Optional
.ofNullable(modelCategory)
.map(ModelCategory::getShortName)
.orElseGet(() -> (String)resource.getProperty("category"));
//To resolve api key placeholder ${apikey} in model-name.properties file
String apiKey = null;
@@ -100,10 +108,10 @@ public class ModelProfileService {
}
apiKey = Optional.ofNullable(this.environment.getProperty("apiKey"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
.orElse(null);
}
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken, reasoningEffort, reasoningSummaryDetails);
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken, reasoningEffort, reasoningSummaryDetails, category);
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile", e);
@@ -111,6 +119,10 @@ public class ModelProfileService {
}
}
public ModelProfile getModelProfile(final ModelCategory modelCategory) {
return this.getModelProfile(modelCategory.getModelProfile(), modelCategory);
}
public List<String> getModelProfileUrls(final ModelProvider... modelProviders) {
final var modelProfiles = Binder.get(this.environment)
.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))

9
src/test/java/de/extio/lmlib/agent/AgentTest.java
View File

@@ -53,6 +53,7 @@ public class AgentTest {
new Agent("CodeSummarizer",
AgentType.START_CONVERSATION,
ModelCategory.MEDIUM,
null,
"Generate a summary of the following Java source code.",
"The source code is:\n{{code}}",
new TextAgentResponseHandler("summary"),
@@ -65,6 +66,7 @@ public class AgentTest {
new Agent("FeatureIdentifier",
AgentType.CONVERSATION,
ModelCategory.MEDIUM,
null,
"",
"Generate a list of 5 distinct functional features. In scope is business logic, not code mechanics. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"features\": [\"Feature 1\", \"Feature 2\", ... ] }",
new JsonAgentResponseHandler(),
@@ -77,6 +79,7 @@ public class AgentTest {
new Agent("FeatureAnalyzer",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"Analyze a feature of the following Java source code and describe the functionality and purpose of this feature. Only describe the implementation, don't suggest any possible improvements.",
"""
The source code is:
@@ -92,6 +95,7 @@ public class AgentTest {
new Agent("AnswerGenerator",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"Generate a clear and concise blog post for an announcement from the following code analysis.",
"""
Write a concise description of the purpose in the blog post first and then concise information for the following topics: {{topics}}
@@ -131,6 +135,7 @@ public class AgentTest {
new Agent("CodeSummarizer",
AgentType.START_CONVERSATION,
ModelCategory.MEDIUM,
null,
"Generate a summary of the following Java source code.",
"The source code is:\n{{code}}",
new TextAgentResponseHandler("summary",
@@ -179,6 +184,7 @@ public class AgentTest {
new Agent("FirstLevel",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"",
"Generate 5 random numbers from 1 to 10. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"numbers\": [\"Number 1\", \"Number 2\", ... ] }",
new JsonAgentResponseHandler(),
@@ -191,6 +197,7 @@ public class AgentTest {
new Agent("SecondLevel",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"",
"Generate 5 random numbers from 1 to 10. The previous number was [[numbers]]. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"random\": [\"Number 1\", \"Number 2\", ... ] }",
new JsonAgentResponseHandler(),
@@ -203,6 +210,7 @@ public class AgentTest {
new Agent("Calculator",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"",
"Add these 2 numbers: {{numbers}} + [[random]] ; Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"result\": \"result\" }",
new JsonAgentResponseHandler(),
@@ -215,6 +223,7 @@ public class AgentTest {
new Agent("Grader",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"You are a helpful assistant.",
"If you add {{numbers}} marbles to {{random}} marbles, do you get {{result}} marbles as the result? Answer with 'yes' or 'no'",
new TextAgentResponseHandler("grade"),

1
src/test/resources/llama3.3-70b-azure.properties
View File

@@ -1,3 +1,4 @@
category=
prompts=
tokenEncoding=cl100k_base
maxTokens=1500

1
src/test/resources/local-chatcompletion.properties
View File

@@ -1,3 +1,4 @@
category=M
prompts=
tokenEncoding=cl100k_base
maxTokens=2500

1
src/test/resources/local-completion.properties
View File

@@ -1,3 +1,4 @@
category=S
prompts=gpt-oss
tokenEncoding=cl100k_base
maxTokens=2500