exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement tokenizer resolver and fallback tokenizer; update model profiles and README

Browse Source
This commit is contained in:
extio-de
2026-04-03 18:40:53 +02:00
parent dda1efd1e3
commit b4d1a44d93
17 changed files with 419 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/de/extio/lmlib/profile/ModelProfile.java
View File

@@ -4,6 +4,7 @@ import java.math.BigDecimal;
public record ModelProfile(
String prompt,
String tokenizer,
String tokenEncoding,
int maxTokens,
int maxContextLength,

5
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -40,6 +40,9 @@ public class ModelProfileService {
final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts"))
.map(Object::toString)
.orElse("");
final var tokenizer = Optional.ofNullable(resource.getProperty("tokenizer"))
.map(Object::toString)
.orElse(null);
final var tokenEncoding = Optional.ofNullable(resource.getProperty("tokenEncoding"))
.map(Object::toString)
.orElse("none");
@@ -109,7 +112,7 @@ public class ModelProfileService {
.orElse(null);
}
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category);
return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category);
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile", e);