exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Simplify model name loading and update ModelNameSupplier to use ModelProfile

Browse Source
This commit is contained in:
extio-de
2025-10-29 13:02:14 +01:00
parent d0b1eeae75
commit 179755df45
6 changed files with 44 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -156,7 +156,7 @@ Each model profile is a `.properties` file that configures how the library inter
| `temperature` | Double | Yes | Controls randomness in responses. Lower values (0.0-0.5) are more deterministic, higher values (0.5-1.0+) more creative. | `0.2`, `0.4`, `0.7` | | `temperature` | Double | Yes | Controls randomness in responses. Lower values (0.0-0.5) are more deterministic, higher values (0.5-1.0+) more creative. | `0.2`, `0.4`, `0.7` |
| `topP` | Double | Yes | Nucleus sampling parameter. Controls diversity via cumulative probability. Typically kept at 1.0. | `1.0`, `0.9` | | `topP` | Double | Yes | Nucleus sampling parameter. Controls diversity via cumulative probability. Typically kept at 1.0. | `1.0`, `0.9` |
| `modelProvider` | Enum | Yes | The LLM provider/client type to use. | `OAI_CHAT_COMPLETION`, `OAI_TEXT_COMPLETION`, `AZURE_AI` | | `modelProvider` | Enum | Yes | The LLM provider/client type to use. | `OAI_CHAT_COMPLETION`, `OAI_TEXT_COMPLETION`, `AZURE_AI` |
| `modelName` | String | No | The specific model identifier or deployment name. Can be empty for local models where it's auto-detected. | `Llama-3.3-70B-Instruct`, `gpt-4`, `` (empty for auto-detect) | | `modelName` | String | No | The specific model identifier or deployment name. Can be empty for local models (e.g. lama-server) where it's auto-detected (use first name). | `Llama-3.3-70B-Instruct`, `gpt-4`, `` (empty for auto-detect) |
| `url` | String | Yes | Base URL endpoint for the model API. | `https://api.openai.com/v1`, `http://localhost:5001` | | `url` | String | Yes | Base URL endpoint for the model API. | `https://api.openai.com/v1`, `http://localhost:5001` |
| `apiKey` | String | No | Authentication key for the API. Can use Spring property placeholders like `${apikey}`. Can be empty for local models. | `sk-...`, `${apikey}`, `` (empty) | | `apiKey` | String | No | Authentication key for the API. Can use Spring property placeholders like `${apikey}`. Can be empty for local models. | `sk-...`, `${apikey}`, `` (empty) |
| `cost1MInTokens` | Double | Yes | Cost per 1 million input tokens in your currency. Use `0` for free/local models. | `0.71`, `0`, `5.00` | | `cost1MInTokens` | Double | Yes | Cost per 1 million input tokens in your currency. Use `0` for free/local models. | `0.71`, `0`, `5.00` |

40
src/main/java/de/extio/lmlib/client/oai/ModelNameLoader.java
View File

@@ -1,62 +1,44 @@
package de.extio.lmlib.client.oai; package de.extio.lmlib.client.oai;
import java.time.Duration; import java.util.Optional;
import java.util.function.Consumer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.HttpMethod; import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClient;
import reactor.util.retry.Retry; class ModelNameLoader {
class ModelNameLoader implements Runnable {
private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class); private static final Logger LOGGER = LoggerFactory.getLogger(ModelNameLoader.class);
private final WebClient webClient; private final WebClient webClient;
private final Consumer<String> consumer; public ModelNameLoader(final WebClient webClient) {
public ModelNameLoader(@Qualifier("textCompletionWebClientBuilder") final WebClient webClient, final Consumer<String> consumer) {
this.consumer = consumer;
this.webClient = webClient; this.webClient = webClient;
} }
@Override public Optional<String> loadModelName() {
public void run() {
String modelName = null;
do {
try { try {
LOGGER.info("Querying model name"); LOGGER.info("Querying model name");
modelName = this.webClient final var modelName = this.webClient
.method(HttpMethod.GET) .method(HttpMethod.GET)
.uri(uriBuilder -> uriBuilder.path("/v1/models").build()) .uri(uriBuilder -> uriBuilder.path("/v1/models").build())
.retrieve() .retrieve()
.bodyToMono(ModelsResponse.class) .bodyToMono(ModelsResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block() .block()
.getData() .getData()
.stream() .stream()
.map(Model::getId) .map(Model::getId)
.findFirst() .findFirst();
.orElseThrow();
modelName.ifPresent(name -> LOGGER.info("Model name: {}", name));
return modelName;
} }
catch (final Exception e) { catch (final Exception e) {
LOGGER.error(e.getMessage(), e); LOGGER.warn("Failed to load model name: {}", e.getMessage());
try { return Optional.empty();
Thread.sleep(1000l);
} }
catch (final InterruptedException e1) {
throw new RuntimeException("Interrupted", e1);
}
}
} while (modelName == null);
LOGGER.info("Model name: {}", modelName);
this.consumer.accept(modelName);
} }
} }

51
src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java
View File

@@ -1,61 +1,40 @@
package de.extio.lmlib.client.oai; package de.extio.lmlib.client.oai;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.profile.ModelProfileService; import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.profile.ModelProfile.ModelProvider;
@Component @Component
public class ModelNameSupplier implements InitializingBean { public class ModelNameSupplier {
@Autowired @Autowired
@Qualifier("lmLibWebClientBuilder") @Qualifier("lmLibWebClientBuilder")
private WebClient.Builder webClientBuilder; private WebClient.Builder webClientBuilder;
private final Map<String, CompletableFuture<String>> modelNames = new ConcurrentHashMap<>(); private final Map<String, String> resolvedModelNames = new ConcurrentHashMap<>();
private final Map<String, CountDownLatch> countDownLatches = new ConcurrentHashMap<>(); public String getModelName(final ModelProfile modelProfile) {
if (modelProfile.modelName() != null && !modelProfile.modelName().isBlank()) {
@Autowired return modelProfile.modelName();
private ModelProfileService modelProfileService;
@Override
public void afterPropertiesSet() throws Exception {
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION, ModelProvider.OAI_CHAT_COMPLETION);
for (final var url : urls) {
this.countDownLatches.put(url, new CountDownLatch(1));
this.fetchModelName(url);
}
} }
private void fetchModelName(final String url) { if (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION) {
final var webClient = this.webClientBuilder.baseUrl(url).build(); return "";
CompletableFuture.runAsync(() -> { }
final var modelNameLoader = new ModelNameLoader(webClient, modelName -> {
this.modelNames.put(url, CompletableFuture.completedFuture(modelName)); final var cacheKey = modelProfile.category() + "|" + modelProfile.url();
this.countDownLatches.get(url).countDown(); return this.resolvedModelNames.computeIfAbsent(cacheKey, key -> {
}); final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
modelNameLoader.run(); final var loader = new ModelNameLoader(webClient);
return loader.loadModelName().orElse("");
}); });
} }
public String getModelName(final String url) {
try {
this.countDownLatches.get(url).await();
}
catch (final InterruptedException e) {
throw new RuntimeException("Interrupted", e);
}
return this.modelNames.get(url).join();
}
} }

3
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java
View File

@@ -20,7 +20,6 @@ import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
@@ -44,7 +43,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
final var chat = createChats(conversation, modelProfile); final var chat = createChats(conversation, modelProfile);
final var request = new ChatCompletionRequest(); final var request = new ChatCompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url())); request.setModel(this.modelNameSupplier.getModelName(modelProfile));
request.setMessages(chat); request.setMessages(chat);
request.setMaxTokens(modelProfile.maxTokens()); request.setMaxTokens(modelProfile.maxTokens());
request.setMaxCompletionTokens(modelProfile.maxTokens()); request.setMaxCompletionTokens(modelProfile.maxTokens());

2
src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java
View File

@@ -49,7 +49,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy); final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy);
final var request = new CompletionRequest(); final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url())); request.setModel(this.modelNameSupplier.getModelName(modelProfile));
request.setPrompt(prompt); request.setPrompt(prompt);
request.setMaxTokens(modelProfile.maxTokens()); request.setMaxTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature()); request.setTemperature(modelProfile.temperature());

35
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -2,16 +2,10 @@ package de.extio.lmlib.profile;
import java.io.IOException; import java.io.IOException;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.stream.Collectors;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.bind.Bindable;
import org.springframework.boot.context.properties.bind.Binder;
import org.springframework.cache.annotation.Cacheable; import org.springframework.cache.annotation.Cacheable;
import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.Environment; import org.springframework.core.env.Environment;
@@ -25,8 +19,6 @@ public class ModelProfileService {
private final Environment environment; private final Environment environment;
private static final String MODEL_PROFILE_PREFIX = "profile";
private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class); private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class);
public ModelProfileService(final Environment environment) { public ModelProfileService(final Environment environment) {
@@ -122,31 +114,4 @@ public class ModelProfileService {
public ModelProfile getModelProfile(final ModelCategory modelCategory) { public ModelProfile getModelProfile(final ModelCategory modelCategory) {
return this.getModelProfile(modelCategory.getModelProfile(), modelCategory); return this.getModelProfile(modelCategory.getModelProfile(), modelCategory);
} }
public List<String> getModelProfileUrls(final ModelProvider... modelProviders) {
final var modelProfiles = Binder.get(this.environment)
.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
.orElse(Collections.emptyMap());
return modelProfiles.values().stream()
.filter(model -> model != null && !model.isEmpty())
.map(model -> {
try {
final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
for (final ModelProvider modelProvider : modelProviders) {
if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
}
}
return null;
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile for: " + model, e);
return null;
}
})
.filter(Objects::nonNull)
.distinct()
.collect(Collectors.toList());
}
} }