From 1ce646edb27f1ade2167fd1bd96df6f4fa090840 Mon Sep 17 00:00:00 2001 From: extio-de Date: Fri, 3 Apr 2026 15:40:51 +0200 Subject: [PATCH] Generalize and streamline model name retrieval in clients --- .../extio/lmlib/LmLibAutoConfiguration.java | 155 ------------------ .../java/de/extio/lmlib/client/Client.java | 3 + .../lmlib/client/ClientAutoConfiguration.java | 7 - .../de/extio/lmlib/client/ClientService.java | 14 ++ .../lmlib/client/cached/CachedClient.java | 15 +- .../lmlib/client/oai/ModelNameLoader.java | 43 ----- .../lmlib/client/oai/ModelNameSupplier.java | 38 ----- .../completion/AbstractCompletionClient.java | 66 +++++++- .../completion/chat/ChatCompletionClient.java | 2 +- .../completion/text/TextCompletionClient.java | 2 +- 10 files changed, 95 insertions(+), 250 deletions(-) delete mode 100644 src/main/java/de/extio/lmlib/LmLibAutoConfiguration.java delete mode 100755 src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java delete mode 100755 src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java diff --git a/src/main/java/de/extio/lmlib/LmLibAutoConfiguration.java b/src/main/java/de/extio/lmlib/LmLibAutoConfiguration.java deleted file mode 100644 index 230e129..0000000 --- a/src/main/java/de/extio/lmlib/LmLibAutoConfiguration.java +++ /dev/null @@ -1,155 +0,0 @@ -package de.extio.lmlib; - -import java.util.List; - -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.annotation.Bean; -import org.springframework.core.env.Environment; - -import de.extio.lmlib.agent.AgentExecutorService; -import de.extio.lmlib.client.ClientService; -import de.extio.lmlib.client.oai.ModelNameSupplier; -import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient; -import de.extio.lmlib.client.oai.completion.text.TextCompletionClient; -import de.extio.lmlib.profile.ModelProfileService; -import de.extio.lmlib.prompt.AlpacaShortPromptStrategy; -import de.extio.lmlib.prompt.ChatMLPromptStrategy; -import de.extio.lmlib.prompt.GemmaPromptStrategy; -import de.extio.lmlib.prompt.GptOssPromptStrategy; -import de.extio.lmlib.prompt.Llama3PromptStrategy; -import de.extio.lmlib.prompt.Llama4PromptStrategy; -import de.extio.lmlib.prompt.MistralNemoPromptStrategy; -import de.extio.lmlib.prompt.MistralPromptStrategy; -import de.extio.lmlib.prompt.NoPromptStrategy; -import de.extio.lmlib.prompt.Phi3PromptStrategy; -import de.extio.lmlib.prompt.Phi4PromptStrategy; -import de.extio.lmlib.prompt.PromptStrategy; -import de.extio.lmlib.prompt.PromptStrategyFactory; -import de.extio.lmlib.prompt.Qwen3PromptStrategy; -import de.extio.lmlib.prompt.VicunaPromptStrategy; - -@AutoConfiguration -public class LmLibAutoConfiguration { - - @Bean - @ConditionalOnMissingBean - ModelProfileService modelProfileService(final Environment environment) { - return new ModelProfileService(environment); - } - - @Bean - @ConditionalOnMissingBean - ClientService clientService() { - return new ClientService(); - } - - @Bean - @ConditionalOnMissingBean - AgentExecutorService agentExecutorService() { - return new AgentExecutorService(); - } - - @Bean - @ConditionalOnMissingBean - ModelNameSupplier modelNameSupplier() { - return new ModelNameSupplier(); - } - - @Bean - @ConditionalOnMissingBean - TextCompletionClient textCompletionClient() { - return new TextCompletionClient(); - } - - @Bean - @ConditionalOnMissingBean - ChatCompletionClient chatCompletionClient() { - return new ChatCompletionClient(); - } - - @Bean - @ConditionalOnMissingBean - PromptStrategyFactory promptStrategyFactory(final List strategies) { - return new PromptStrategyFactory(strategies); - } - - @Bean - @ConditionalOnMissingBean - AlpacaShortPromptStrategy alpacaShortPromptStrategy() { - return new AlpacaShortPromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - ChatMLPromptStrategy chatMLPromptStrategy() { - return new ChatMLPromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - GemmaPromptStrategy gemmaPromptStrategy() { - return new GemmaPromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - GptOssPromptStrategy gptOssPromptStrategy() { - return new GptOssPromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - Llama3PromptStrategy llama3PromptStrategy() { - return new Llama3PromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - Llama4PromptStrategy llama4PromptStrategy() { - return new Llama4PromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - MistralNemoPromptStrategy mistralNemoPromptStrategy() { - return new MistralNemoPromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - MistralPromptStrategy mistralPromptStrategy() { - return new MistralPromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - NoPromptStrategy noPromptStrategy() { - return new NoPromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - Phi3PromptStrategy phi3PromptStrategy() { - return new Phi3PromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - Phi4PromptStrategy phi4PromptStrategy() { - return new Phi4PromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - Qwen3PromptStrategy qwen3PromptStrategy() { - return new Qwen3PromptStrategy(); - } - - @Bean - @ConditionalOnMissingBean - VicunaPromptStrategy vicunaPromptStrategy() { - return new VicunaPromptStrategy(); - } - -} diff --git a/src/main/java/de/extio/lmlib/client/Client.java b/src/main/java/de/extio/lmlib/client/Client.java index 7984376..cfc55e0 100755 --- a/src/main/java/de/extio/lmlib/client/Client.java +++ b/src/main/java/de/extio/lmlib/client/Client.java @@ -1,5 +1,6 @@ package de.extio.lmlib.client; +import java.util.List; import java.util.function.Consumer; import de.extio.lmlib.profile.ModelCategory; @@ -15,6 +16,8 @@ public interface Client { Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer chunkConsumer, ToolCallData toolCallData, boolean skipCache); Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer chunkConsumer, ToolCallData toolCallData, boolean skipCache); + + List getModelNames(ModelProfile modelProfile, boolean forceReload); ModelProvider getModelProvider(); diff --git a/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java b/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java index f86373a..04f6185 100644 --- a/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java +++ b/src/main/java/de/extio/lmlib/client/ClientAutoConfiguration.java @@ -6,7 +6,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; -import de.extio.lmlib.client.oai.ModelNameSupplier; import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient; import de.extio.lmlib.client.oai.completion.text.TextCompletionClient; import de.extio.lmlib.profile.ProfileAutoConfiguration; @@ -23,12 +22,6 @@ public class ClientAutoConfiguration { return new ClientService(); } - @Bean - @ConditionalOnMissingBean - ModelNameSupplier modelNameSupplier() { - return new ModelNameSupplier(); - } - @Bean @ConditionalOnMissingBean TextCompletionClient textCompletionClient() { diff --git a/src/main/java/de/extio/lmlib/client/ClientService.java b/src/main/java/de/extio/lmlib/client/ClientService.java index f547277..0be2a17 100755 --- a/src/main/java/de/extio/lmlib/client/ClientService.java +++ b/src/main/java/de/extio/lmlib/client/ClientService.java @@ -32,6 +32,11 @@ public class ClientService { return this.getClient(modelProfile); } + public List getModelNames(final ModelCategory category, final boolean forceReload) { + final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile(), category); + return this.getModelNames(modelProfile, forceReload); + } + public boolean supportsToolCalling(final ModelCategory category) { return this.getClient(category).supportsToolCalling(); } @@ -46,6 +51,10 @@ public class ClientService { .orElseThrow(() -> new IllegalStateException("No client found for model profile " + modelProfile.modelName())); } + public List getModelNames(final ModelProfile modelProfile, final boolean forceReload) { + return this.getClient(modelProfile).getModelNames(modelProfile, forceReload); + } + public boolean supportsToolCalling(final ModelProfile modelProfile) { return this.getClient(modelProfile).supportsToolCalling(); } @@ -101,6 +110,11 @@ public class ClientService { } return completion; } + + @Override + public List getModelNames(final ModelProfile modelProfile, final boolean forceReload) { + return this.client.getModelNames(modelProfile, forceReload); + } @Override public ModelProvider getModelProvider() { diff --git a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java index 088c66f..be653a5 100644 --- a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java +++ b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java @@ -6,6 +6,7 @@ import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.OffsetDateTime; import java.util.HexFormat; +import java.util.List; import java.util.function.Consumer; import java.util.function.Supplier; @@ -65,6 +66,11 @@ public class CachedClient implements Client { } return response; } + + @Override + public List getModelNames(final ModelProfile modelProfile, final boolean forceReload) { + return this.client.getModelNames(modelProfile, forceReload); + } @Override public ModelProvider getModelProvider() { @@ -117,7 +123,7 @@ public class CachedClient implements Client { MessageDigest digest; try { digest = MessageDigest.getInstance("SHA-256"); - digest.update(modelProfile.modelName().getBytes()); + digest.update(this.resolvePrimaryModelName(modelProfile).getBytes()); digest.update(modelCategory != null ? modelCategory.shortName().getBytes() : modelProfile.category().getBytes()); digest.update(String.valueOf(modelProfile.modelProvider()).getBytes()); digest.update(String.valueOf(modelProfile.maxTokens()).getBytes()); @@ -135,5 +141,12 @@ public class CachedClient implements Client { throw new RuntimeException("Error calculating hash", e); } } + + private String resolvePrimaryModelName(final ModelProfile modelProfile) { + if (modelProfile.modelName() != null && !modelProfile.modelName().isBlank()) { + return modelProfile.modelName(); + } + return this.client.getModelNames(modelProfile, false).stream().findFirst().orElse(""); + } } diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java b/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java deleted file mode 100755 index dce4f5b..0000000 --- a/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java +++ /dev/null @@ -1,43 +0,0 @@ -package de.extio.lmlib.client.oai; - -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpMethod; -import org.springframework.web.client.RestClient; - -class ModelNameLoader { - - private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class); - - private final RestClient restClient; - - public ModelNameLoader(final RestClient restClient) { - this.restClient = restClient; - } - - public Optional loadModelName() { - try { - LOGGER.info("Querying model name"); - - final var modelName = this.restClient - .method(HttpMethod.GET) - .uri(uriBuilder -> uriBuilder.path("/v1/models").build()) - .retrieve() - .body(ModelsResponse.class) - .getData() - .stream() - .map(Model::getId) - .findFirst(); - - modelName.ifPresent(name -> LOGGER.info("Model name: {}", name)); - return modelName; - } - catch (final Exception e) { - LOGGER.warn("Failed to load model name: {}", e.getMessage()); - return Optional.empty(); - } - } - -} diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java b/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java deleted file mode 100755 index 59b82ea..0000000 --- a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java +++ /dev/null @@ -1,38 +0,0 @@ -package de.extio.lmlib.client.oai; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.web.client.RestClient; - -import de.extio.lmlib.profile.ModelProfile; -import de.extio.lmlib.profile.ModelProfile.ModelProvider; - -public class ModelNameSupplier { - - @Autowired - @Qualifier("lmLibRestClientBuilder") - private RestClient.Builder restClientBuilder; - - private final Map resolvedModelNames = new ConcurrentHashMap<>(); - - public String getModelName(final ModelProfile modelProfile) { - if (modelProfile.modelName() != null && !modelProfile.modelName().isBlank()) { - return modelProfile.modelName(); - } - - if (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION) { - return ""; - } - - final var cacheKey = modelProfile.category() + "|" + modelProfile.url(); - return this.resolvedModelNames.computeIfAbsent(cacheKey, key -> { - final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); - final var loader = new ModelNameLoader(restClient); - return loader.loadModelName().orElse(""); - }); - } - -} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index c2b8c07..7b63e11 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -5,6 +5,9 @@ import java.text.DecimalFormat; import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import org.slf4j.Logger; @@ -13,7 +16,10 @@ import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpMethod; import org.springframework.web.client.RestClient; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestClientResponseException; import com.fasterxml.jackson.annotation.JsonInclude; import tools.jackson.core.json.JsonReadFeature; @@ -27,7 +33,8 @@ import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.CompletionFinishReason; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.ToolCallData; -import de.extio.lmlib.client.oai.ModelNameSupplier; +import de.extio.lmlib.client.oai.Model; +import de.extio.lmlib.client.oai.ModelsResponse; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; @@ -42,9 +49,6 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean @Qualifier("lmLibRestClientBuilder") protected RestClient.Builder restClientBuilder; - @Autowired - protected ModelNameSupplier modelNameSupplier; - @Autowired protected Tokenizer tokenizer; @@ -54,6 +58,8 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean @Value("${client.collectStatistics:false}") protected boolean collectStatistics; + protected final Map> resolvedModelNames = new ConcurrentHashMap<>(); + protected final CompletionStatistics totalStatistics = new CompletionStatistics(); protected final ObjectMapper objectMapper; @@ -107,8 +113,60 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean return this.requestCompletion(conversation, modelProfile, chunkConsumer, toolCallData); } + + @Override + public List getModelNames(final ModelProfile modelProfile, final boolean forceReload) { + if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) { + return List.of(); + } + if (modelProfile.modelName() != null && !modelProfile.modelName().isBlank()) { + return List.of(modelProfile.modelName()); + } + if (modelProfile.url() == null || modelProfile.url().isBlank()) { + return List.of(); + } + + final var cacheKey = modelProfile.category() + "|" + modelProfile.url() + "|" + modelProfile.apiKey(); + if (forceReload) { + final var modelNames = this.loadModelNames(modelProfile); + this.resolvedModelNames.put(cacheKey, modelNames); + return modelNames; + } + return this.resolvedModelNames.computeIfAbsent(cacheKey, key -> this.loadModelNames(modelProfile)); + } protected abstract Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer chunkConsumer, final ToolCallData toolCallData); + + protected String getPrimaryModelName(final ModelProfile modelProfile) { + return this.getModelNames(modelProfile, false).stream().findFirst().orElse(""); + } + + private List loadModelNames(final ModelProfile modelProfile) { + try { + final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); + final var response = restClient + .method(HttpMethod.GET) + .uri(uriBuilder -> uriBuilder.path("/v1/models").build()) + .retrieve() + .body(ModelsResponse.class); + return response == null || response.getData() == null ? List.of() : response.getData() + .stream() + .map(Model::getId) + .filter(modelName -> modelName != null && !modelName.isBlank()) + .toList(); + } + catch (final RestClientResponseException e) { + LOGGER.warn("Failed to load model names: {} {}", e.getStatusCode(), e.getStatusText()); + return List.of(); + } + catch (final RestClientException e) { + LOGGER.warn("Failed to load model names: {}", e.getMessage()); + return List.of(); + } + catch (final RuntimeException e) { + return List.of(); + } + } protected CompletionFinishReason mapFinishReason(final String finishReason) { return switch (finishReason) { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index 0a0d9f9..eb64dcc 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -50,7 +50,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { final var chat = createChats(conversation, modelProfile); final var request = new ChatCompletionRequest(); - request.setModel(this.modelNameSupplier.getModelName(modelProfile)); + request.setModel(this.getPrimaryModelName(modelProfile)); request.setMessages(chat); request.setMaxTokens(modelProfile.maxTokens()); request.setMaxCompletionTokens(modelProfile.maxTokens()); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index b8b7d7c..0916e15 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -55,7 +55,7 @@ public class TextCompletionClient extends AbstractCompletionClient { final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy); final var request = new CompletionRequest(); - request.setModel(this.modelNameSupplier.getModelName(modelProfile)); + request.setModel(this.getPrimaryModelName(modelProfile)); request.setPrompt(prompt); request.setMaxTokens(modelProfile.maxTokens()); request.setTemperature(modelProfile.temperature());