feat: Implement tokenizer resolver and fallback tokenizer; update model profiles and README

2026-04-03 18:40:53 +02:00
parent dda1efd1e3
commit b4d1a44d93
17 changed files with 419 additions and 28 deletions
--- a/src/main/java/de/extio/lmlib/profile/ModelProfile.java
+++ b/src/main/java/de/extio/lmlib/profile/ModelProfile.java
@@ -4,6 +4,7 @@ import java.math.BigDecimal;

 public record ModelProfile(
 		String prompt,
+		String tokenizer,
 		String tokenEncoding,
 		int maxTokens,
 		int maxContextLength,