From 179755df453e58a58faff70832fccae34dd96090 Mon Sep 17 00:00:00 2001 From: extio-de Date: Wed, 29 Oct 2025 13:02:14 +0100 Subject: [PATCH] refactor: Simplify model name loading and update ModelNameSupplier to use ModelProfile --- README.md | 2 +- .../lmlib/client/oai/ModelNameLoader.java | 68 +++++++------------ .../lmlib/client/oai/ModelNameSupplier.java | 53 +++++---------- .../completion/chat/ChatCompletionClient.java | 3 +- .../completion/text/TextCompletionClient.java | 2 +- .../lmlib/profile/ModelProfileService.java | 35 ---------- 6 files changed, 44 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index e4dabf9..e092003 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ Each model profile is a `.properties` file that configures how the library inter | `temperature` | Double | Yes | Controls randomness in responses. Lower values (0.0-0.5) are more deterministic, higher values (0.5-1.0+) more creative. | `0.2`, `0.4`, `0.7` | | `topP` | Double | Yes | Nucleus sampling parameter. Controls diversity via cumulative probability. Typically kept at 1.0. | `1.0`, `0.9` | | `modelProvider` | Enum | Yes | The LLM provider/client type to use. | `OAI_CHAT_COMPLETION`, `OAI_TEXT_COMPLETION`, `AZURE_AI` | -| `modelName` | String | No | The specific model identifier or deployment name. Can be empty for local models where it's auto-detected. | `Llama-3.3-70B-Instruct`, `gpt-4`, `` (empty for auto-detect) | +| `modelName` | String | No | The specific model identifier or deployment name. Can be empty for local models (e.g. lama-server) where it's auto-detected (use first name). | `Llama-3.3-70B-Instruct`, `gpt-4`, `` (empty for auto-detect) | | `url` | String | Yes | Base URL endpoint for the model API. | `https://api.openai.com/v1`, `http://localhost:5001` | | `apiKey` | String | No | Authentication key for the API. Can use Spring property placeholders like `${apikey}`. Can be empty for local models. | `sk-...`, `${apikey}`, `` (empty) | | `cost1MInTokens` | Double | Yes | Cost per 1 million input tokens in your currency. Use `0` for free/local models. | `0.71`, `0`, `5.00` | diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java b/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java index 3a00a49..9093042 100755 --- a/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java +++ b/src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java @@ -1,62 +1,44 @@ package de.extio.lmlib.client.oai; -import java.time.Duration; -import java.util.function.Consumer; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpMethod; import org.springframework.web.reactive.function.client.WebClient; -import reactor.util.retry.Retry; - -class ModelNameLoader implements Runnable { +class ModelNameLoader { private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class); private final WebClient webClient; - private final Consumer consumer; - - public ModelNameLoader(@Qualifier("textCompletionWebClientBuilder") final WebClient webClient, final Consumer consumer) { - this.consumer = consumer; + public ModelNameLoader(final WebClient webClient) { this.webClient = webClient; } - @Override - public void run() { - String modelName = null; - do { - try { - LOGGER.info("Querying model name"); - - modelName = this.webClient - .method(HttpMethod.GET) - .uri(uriBuilder -> uriBuilder.path("/v1/models").build()) - .retrieve() - .bodyToMono(ModelsResponse.class) - .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) - .block() - .getData() - .stream() - .map(Model::getId) - .findFirst() - .orElseThrow(); - } - catch (final Exception e) { - LOGGER.error(e.getMessage(), e); - try { - Thread.sleep(1000l); - } - catch (final InterruptedException e1) { - throw new RuntimeException("Interrupted", e1); - } - } - } while (modelName == null); - - LOGGER.info("Model name: {}", modelName); - this.consumer.accept(modelName); + public Optional loadModelName() { + try { + LOGGER.info("Querying model name"); + + final var modelName = this.webClient + .method(HttpMethod.GET) + .uri(uriBuilder -> uriBuilder.path("/v1/models").build()) + .retrieve() + .bodyToMono(ModelsResponse.class) + .block() + .getData() + .stream() + .map(Model::getId) + .findFirst(); + + modelName.ifPresent(name -> LOGGER.info("Model name: {}", name)); + return modelName; + } + catch (final Exception e) { + LOGGER.warn("Failed to load model name: {}", e.getMessage()); + return Optional.empty(); + } } } diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java b/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java index 876f51b..35db717 100755 --- a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java +++ b/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java @@ -1,61 +1,40 @@ package de.extio.lmlib.client.oai; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; -import de.extio.lmlib.profile.ModelProfileService; +import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; @Component -public class ModelNameSupplier implements InitializingBean { +public class ModelNameSupplier { @Autowired @Qualifier("lmLibWebClientBuilder") private WebClient.Builder webClientBuilder; - private final Map> modelNames = new ConcurrentHashMap<>(); + private final Map resolvedModelNames = new ConcurrentHashMap<>(); - private final Map countDownLatches = new ConcurrentHashMap<>(); - - @Autowired - private ModelProfileService modelProfileService; - - @Override - public void afterPropertiesSet() throws Exception { - final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION, ModelProvider.OAI_CHAT_COMPLETION); - for (final var url : urls) { - this.countDownLatches.put(url, new CountDownLatch(1)); - this.fetchModelName(url); + public String getModelName(final ModelProfile modelProfile) { + if (modelProfile.modelName() != null && !modelProfile.modelName().isBlank()) { + return modelProfile.modelName(); } - } - - private void fetchModelName(final String url) { - final var webClient = this.webClientBuilder.baseUrl(url).build(); - CompletableFuture.runAsync(() -> { - final var modelNameLoader = new ModelNameLoader(webClient, modelName -> { - this.modelNames.put(url, CompletableFuture.completedFuture(modelName)); - this.countDownLatches.get(url).countDown(); - }); - modelNameLoader.run(); + + if (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION) { + return ""; + } + + final var cacheKey = modelProfile.category() + "|" + modelProfile.url(); + return this.resolvedModelNames.computeIfAbsent(cacheKey, key -> { + final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); + final var loader = new ModelNameLoader(webClient); + return loader.loadModelName().orElse(""); }); } - public String getModelName(final String url) { - try { - this.countDownLatches.get(url).await(); - } - catch (final InterruptedException e) { - throw new RuntimeException("Interrupted", e); - } - return this.modelNames.get(url).join(); - } - } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index f28754e..08cb869 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -20,7 +20,6 @@ import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; -import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import reactor.core.publisher.Mono; @@ -44,7 +43,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { final var chat = createChats(conversation, modelProfile); final var request = new ChatCompletionRequest(); - request.setModel(this.modelNameSupplier.getModelName(modelProfile.url())); + request.setModel(this.modelNameSupplier.getModelName(modelProfile)); request.setMessages(chat); request.setMaxTokens(modelProfile.maxTokens()); request.setMaxCompletionTokens(modelProfile.maxTokens()); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 0bc78ee..31d4e32 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -49,7 +49,7 @@ public class TextCompletionClient extends AbstractCompletionClient { final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy); final var request = new CompletionRequest(); - request.setModel(this.modelNameSupplier.getModelName(modelProfile.url())); + request.setModel(this.modelNameSupplier.getModelName(modelProfile)); request.setPrompt(prompt); request.setMaxTokens(modelProfile.maxTokens()); request.setTemperature(modelProfile.temperature()); diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java index 57470c4..ce9527f 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java @@ -2,16 +2,10 @@ package de.extio.lmlib.profile; import java.io.IOException; import java.math.BigDecimal; -import java.util.Collections; -import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.boot.context.properties.bind.Bindable; -import org.springframework.boot.context.properties.bind.Binder; import org.springframework.cache.annotation.Cacheable; import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.Environment; @@ -25,8 +19,6 @@ public class ModelProfileService { private final Environment environment; - private static final String MODEL_PROFILE_PREFIX = "profile"; - private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class); public ModelProfileService(final Environment environment) { @@ -122,31 +114,4 @@ public class ModelProfileService { public ModelProfile getModelProfile(final ModelCategory modelCategory) { return this.getModelProfile(modelCategory.getModelProfile(), modelCategory); } - - public List getModelProfileUrls(final ModelProvider... modelProviders) { - final var modelProfiles = Binder.get(this.environment) - .bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class)) - .orElse(Collections.emptyMap()); - - return modelProfiles.values().stream() - .filter(model -> model != null && !model.isEmpty()) - .map(model -> { - try { - final var resource = new ResourcePropertySource("classpath:" + model + ".properties"); - for (final ModelProvider modelProvider : modelProviders) { - if (modelProvider.name().equals(resource.getProperty("modelProvider"))) { - return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null; - } - } - return null; - } - catch (final IOException e) { - LOGGER.error("Error while reading model profile for: " + model, e); - return null; - } - }) - .filter(Objects::nonNull) - .distinct() - .collect(Collectors.toList()); - } }