feat: Implement tokenizer resolver and fallback tokenizer; update model profiles and README

src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java

@@ -39,7 +39,7 @@ import de.extio.lmlib.profile.ModelCategory;
 import de.extio.lmlib.profile.ModelProfile;
 import de.extio.lmlib.profile.ModelProfile.ModelProvider;
 import de.extio.lmlib.profile.ModelProfileService;
-import de.extio.lmlib.token.Tokenizer;
+import de.extio.lmlib.token.TokenizerResolver;
 
 public abstract class AbstractCompletionClient implements Client, DisposableBean {
 	
@@ -50,7 +50,7 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
 	protected RestClient.Builder restClientBuilder;
 	
 	@Autowired
-	protected Tokenizer tokenizer;
+	protected TokenizerResolver tokenizerResolver;
 	
 	@Autowired
 	protected ModelProfileService modelProfileService;
@@ -214,14 +214,14 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
 		}
 		// Fallback to tokenizer estimation
 		if (inTokens <= 0 && prompt != null && !prompt.isBlank()) {
-			inTokens = this.tokenizer.count(prompt, modelProfile);
+			inTokens = this.tokenizerResolver.count(prompt, modelProfile);
 		}
 		if (reasoningOutTokens <= 0 && reasoning != null && !reasoning.isBlank()) {
-			reasoningOutTokens = this.tokenizer.count(reasoning, modelProfile);
+			reasoningOutTokens = this.tokenizerResolver.count(reasoning, modelProfile);
 			outTokens -= reasoningOutTokens;
 		}
 		if (outTokens <= 0 && response != null && !response.isBlank()) {
-			outTokens = this.tokenizer.count(response, modelProfile);
+			outTokens = this.tokenizerResolver.count(response, modelProfile);
 		}
 		
 		// Calculate cost - inTokens and outTokens are already exclusive of cached/reasoning tokens

src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java

@@ -308,13 +308,13 @@ public class ChatCompletionClient extends AbstractCompletionClient {
 		var prompt = turn.text() != null ? turn.text() : "";
 		var trimmed = false;
 		
-		final var textTokens = this.tokenizer.count(prompt, modelProfile);
+		final var textTokens = this.tokenizerResolver.count(prompt, modelProfile);
 		final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
 		if (tokens.addAndGet(textTokens) > maxLength) {
-			var tokenized = this.tokenizer.tokenize(prompt, modelProfile);
+			var tokenized = this.tokenizerResolver.tokenize(prompt, modelProfile);
 			LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
 			tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
-			prompt = this.tokenizer.detokenize(tokenized, modelProfile);
+			prompt = this.tokenizerResolver.detokenize(tokenized, modelProfile);
 			trimmed = true;
 		}
 		

src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java

@@ -200,12 +200,12 @@ public class TextCompletionClient extends AbstractCompletionClient {
 	private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
 		StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
 		
-		List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
+		List<Long> tokenized = this.tokenizerResolver.tokenize(prompt.toString(), modelProfile);
 		final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
 		if (tokenized.size() > maxLength) {
 			LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
 			tokenized = tokenized.subList(0, maxLength);
-			prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
+			prompt = new StringBuilder(this.tokenizerResolver.detokenize(tokenized, modelProfile));
 		}
 		
 		LOGGER.debug("Completion request for {}. Input tokens: {}", modelProfile.category(), tokenized.size());

src/main/java/de/extio/lmlib/profile/ModelProfile.java

@@ -4,6 +4,7 @@ import java.math.BigDecimal;
 
 public record ModelProfile(
 		String prompt,
+		String tokenizer,
 		String tokenEncoding,
 		int maxTokens,
 		int maxContextLength,

src/main/java/de/extio/lmlib/profile/ModelProfileService.java

@@ -40,6 +40,9 @@ public class ModelProfileService {
 			final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts"))
 					.map(Object::toString)
 					.orElse("");
+			final var tokenizer = Optional.ofNullable(resource.getProperty("tokenizer"))
+					.map(Object::toString)
+					.orElse(null);
 			final var tokenEncoding = Optional.ofNullable(resource.getProperty("tokenEncoding"))
 					.map(Object::toString)
 					.orElse("none");
@@ -109,7 +112,7 @@ public class ModelProfileService {
 						.orElse(null);
 			}
 			
-			return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category);
+			return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category);
 		}
 		catch (final IOException e) {
 			LOGGER.error("Error while reading model profile", e);

src/main/java/de/extio/lmlib/token/FallbackTokenizer.java

@@ -0,0 +1,169 @@
+package de.extio.lmlib.token;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import de.extio.lmlib.profile.ModelProfile;
+
+final class FallbackTokenizer implements Tokenizer {
+	
+	static final String NAME = "fallback";
+	
+	@Override
+	public String getName() {
+		return NAME;
+	}
+	
+	@Override
+	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
+		if (txt == null || txt.isEmpty()) {
+			return List.of();
+		}
+		final var tokens = new ArrayList<Long>();
+		var index = 0;
+		while (index < txt.length()) {
+			final var codePoint = txt.codePointAt(index);
+			final var charCount = Character.charCount(codePoint);
+			if (Character.isWhitespace(codePoint)) {
+				final var end = this.consumeWhitespace(txt, index);
+				tokens.addAll(this.encodeChunked(txt.substring(index, end), 6));
+				index = end;
+				continue;
+			}
+			if (Character.isLetterOrDigit(codePoint)) {
+				final var end = this.consumeAlphaNumeric(txt, index);
+				tokens.addAll(this.encodeWordLike(txt.substring(index, end)));
+				index = end;
+				continue;
+			}
+			tokens.add(this.encodeToken(txt.substring(index, index + charCount)));
+			index += charCount;
+		}
+		return List.copyOf(tokens);
+	}
+	
+	@Override
+	public int count(final String txt, final ModelProfile modelProfile) {
+		return this.tokenize(txt, modelProfile).size();
+	}
+	
+	@Override
+	public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
+		final var builder = new StringBuilder();
+		for (final var token : tokens) {
+			builder.append(this.decodeToken(token));
+		}
+		return builder.toString();
+	}
+	
+	private int consumeWhitespace(final String txt, final int start) {
+		var index = start;
+		while (index < txt.length()) {
+			final var codePoint = txt.codePointAt(index);
+			if (!Character.isWhitespace(codePoint)) {
+				break;
+			}
+			index += Character.charCount(codePoint);
+		}
+		return index;
+	}
+	
+	private int consumeAlphaNumeric(final String txt, final int start) {
+		var index = start;
+		while (index < txt.length()) {
+			final var codePoint = txt.codePointAt(index);
+			if (!Character.isLetterOrDigit(codePoint)) {
+				break;
+			}
+			index += Character.charCount(codePoint);
+		}
+		return index;
+	}
+	
+	private List<Long> encodeWordLike(final String text) {
+		if (text.isEmpty()) {
+			return List.of();
+		}
+		if (this.isAsciiDigits(text)) {
+			return this.encodeChunked(text, 3);
+		}
+		if (this.isAsciiLetters(text)) {
+			return this.encodeChunked(text, 4);
+		}
+		return this.encodeChunkedByUtf8Bytes(text, 6);
+	}
+	
+	private boolean isAsciiDigits(final String text) {
+		return text.chars().allMatch(ch -> ch >= '0' && ch <= '9');
+	}
+	
+	private boolean isAsciiLetters(final String text) {
+		return text.chars().allMatch(ch -> ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z');
+	}
+	
+	private List<Long> encodeChunked(final String text, final int maxCharsPerToken) {
+		final var tokens = new ArrayList<Long>();
+		for (var index = 0; index < text.length();) {
+			var end = index;
+			var charCount = 0;
+			while (end < text.length() && charCount < maxCharsPerToken) {
+				final var codePoint = text.codePointAt(end);
+				end += Character.charCount(codePoint);
+				charCount++;
+			}
+			tokens.add(this.encodeToken(text.substring(index, end)));
+			index = end;
+		}
+		return tokens;
+	}
+	
+	private List<Long> encodeChunkedByUtf8Bytes(final String text, final int maxBytesPerToken) {
+		final var tokens = new ArrayList<Long>();
+		for (var index = 0; index < text.length();) {
+			var end = index;
+			var bytes = 0;
+			while (end < text.length()) {
+				final var codePoint = text.codePointAt(end);
+				final var candidateEnd = end + Character.charCount(codePoint);
+				final var candidateBytes = text.substring(end, candidateEnd).getBytes(StandardCharsets.UTF_8).length;
+				if (bytes > 0 && bytes + candidateBytes > maxBytesPerToken) {
+					break;
+				}
+				end = candidateEnd;
+				bytes += candidateBytes;
+				if (bytes >= maxBytesPerToken) {
+					break;
+				}
+			}
+			tokens.add(this.encodeToken(text.substring(index, end)));
+			index = end;
+		}
+		return tokens;
+	}
+	
+	private long encodeToken(final String tokenText) {
+		final var bytes = tokenText.getBytes(StandardCharsets.UTF_8);
+		if (bytes.length == 0 || bytes.length > 7) {
+			throw new IllegalArgumentException("Fallback tokenizer token length must be between 1 and 7 bytes");
+		}
+		long value = bytes.length;
+		for (var i = 0; i < bytes.length; i++) {
+			value |= (bytes[i] & 0xffL) << ((i + 1) * 8);
+		}
+		return value;
+	}
+	
+	private String decodeToken(final long token) {
+		final var length = (int) (token & 0xffL);
+		if (length <= 0 || length > 7) {
+			throw new IllegalArgumentException("Invalid fallback tokenizer token length: " + length);
+		}
+		final var bytes = new byte[length];
+		for (var i = 0; i < length; i++) {
+			bytes[i] = (byte) ((token >> ((i + 1) * 8)) & 0xffL);
+		}
+		return new String(bytes, StandardCharsets.UTF_8);
+	}
+	
+}

src/main/java/de/extio/lmlib/token/JTokkitTokenizer.java

@@ -14,6 +14,8 @@ import de.extio.lmlib.profile.ModelProfile;
 
 final class JTokkitTokenizer implements Tokenizer {
 	
+	static final String NAME = "jTokkit";
+	
 	private static final EncodingRegistry registry;
 	
 	static {
@@ -26,10 +28,15 @@ final class JTokkitTokenizer implements Tokenizer {
 				return registry.getEncoding(EncodingType.fromName(k).orElseThrow(() -> new IllegalArgumentException("Token encoding " + k)));
 			});
 	
+	@Override
+	public String getName() {
+		return NAME;
+	}
+	
 	@Override
 	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
 		return this.encodings
-				.get(modelProfile.tokenEncoding())
+				.get(this.getTokenEncoding(modelProfile))
 				.encodeOrdinary(txt)
 				.boxed()
 				.stream()
@@ -39,7 +46,7 @@ final class JTokkitTokenizer implements Tokenizer {
 	
 	@Override
 	public int count(final String txt, final ModelProfile modelProfile) {
-		return this.encodings.get(modelProfile.tokenEncoding()).countTokensOrdinary(txt);
+		return this.encodings.get(this.getTokenEncoding(modelProfile)).countTokensOrdinary(txt);
 	}
 	
 	@Override
@@ -47,7 +54,14 @@ final class JTokkitTokenizer implements Tokenizer {
 		final var ial = new IntArrayList(tokens.size());
 		tokens.forEach(t -> ial.add(t.intValue()));
 		return this.encodings
-				.get(modelProfile.tokenEncoding())
+				.get(this.getTokenEncoding(modelProfile))
 				.decode(ial);
 	}
+
+	private String getTokenEncoding(final ModelProfile modelProfile) {
+		if (modelProfile.tokenEncoding() == null || modelProfile.tokenEncoding().isBlank() || "none".equalsIgnoreCase(modelProfile.tokenEncoding())) {
+			return "cl100k_base"; // default encoding, used by most models. Will be used if the profile doesn't specify an encoding or specifies an invalid one.
+		}
+		return modelProfile.tokenEncoding();
+	}
 }

src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java

@@ -1,6 +1,5 @@
 package de.extio.lmlib.token;
 
-import java.time.Duration;
 import java.util.List;
 
 import org.slf4j.Logger;
@@ -12,6 +11,8 @@ import de.extio.lmlib.profile.ModelProfile;
 
 final class LlamaServerTokenizer implements Tokenizer {
 	
+	static final String NAME = "llamaServer";
+	
 	private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class);
 	
 	private final RestClient.Builder restClientBuilder;
@@ -20,6 +21,11 @@ final class LlamaServerTokenizer implements Tokenizer {
 		this.restClientBuilder = restClientBuilder;
 	}
 	
+	@Override
+	public String getName() {
+		return NAME;
+	}
+	
 	@Override
 	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
 		LOGGER.debug("Requesting tokenizing");

src/main/java/de/extio/lmlib/token/Tokenizer.java

@@ -6,6 +6,8 @@ import de.extio.lmlib.profile.ModelProfile;
 
 public interface Tokenizer {
 	
+	String getName();
+	
 	List<Long> tokenize(String txt, ModelProfile modelProfile);
 	
 	int count(String txt, ModelProfile modelProfile);

src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java

@@ -2,6 +2,7 @@ package de.extio.lmlib.token;
 
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -13,18 +14,29 @@ import org.springframework.web.client.RestClient;
 public class TokenizerAutoConfiguration {
 	
 	@Bean
-	@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer")
-	@ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer")
+	@ConditionalOnBean(name = "lmLibRestClientBuilder")
+	@ConditionalOnMissingBean(LlamaServerTokenizer.class)
 	Tokenizer llamaServerTokenizer(@Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder) {
 		return new LlamaServerTokenizer(restClientBuilder);
 	}
 	
 	@Bean
 	@ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings")
-	@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "jTokkit", matchIfMissing = true)
-	@ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer")
+	@ConditionalOnMissingBean(JTokkitTokenizer.class)
 	Tokenizer jTokkitTokenizer() {
 		return new JTokkitTokenizer();
 	}
+	
+	@Bean
+	@ConditionalOnMissingBean(FallbackTokenizer.class)
+	Tokenizer fallbackTokenizer() {
+		return new FallbackTokenizer();
+	}
+	
+	@Bean
+	@ConditionalOnMissingBean
+	TokenizerResolver tokenizerResolver(final java.util.List<Tokenizer> tokenizers) {
+		return new TokenizerResolver(tokenizers);
+	}
 
 }

src/main/java/de/extio/lmlib/token/TokenizerResolver.java

@@ -0,0 +1,55 @@
+package de.extio.lmlib.token;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import de.extio.lmlib.profile.ModelProfile;
+
+public class TokenizerResolver {
+	
+	private static final List<String> DEFAULT_TOKENIZER_ORDER = List.of(JTokkitTokenizer.NAME, FallbackTokenizer.NAME);
+	
+	private final Map<String, Tokenizer> tokenizersByName;
+	
+	public TokenizerResolver(final List<Tokenizer> tokenizers) {
+		final var tokenizersByName = new LinkedHashMap<String, Tokenizer>();
+		for (final var tokenizer : tokenizers) {
+			final var previous = tokenizersByName.put(tokenizer.getName(), tokenizer);
+			if (previous != null) {
+				throw new IllegalStateException("Duplicate tokenizer name: " + tokenizer.getName());
+			}
+		}
+		this.tokenizersByName = Map.copyOf(tokenizersByName);
+	}
+	
+	public Tokenizer resolve(final ModelProfile modelProfile) {
+		if (modelProfile != null && modelProfile.tokenizer() != null && !modelProfile.tokenizer().isBlank()) {
+			final var tokenizer = this.tokenizersByName.get(modelProfile.tokenizer());
+			if (tokenizer == null) {
+				throw new IllegalStateException("No tokenizer configured with name: " + modelProfile.tokenizer());
+			}
+			return tokenizer;
+		}
+		for (final var tokenizerName : DEFAULT_TOKENIZER_ORDER) {
+			final var tokenizer = this.tokenizersByName.get(tokenizerName);
+			if (tokenizer != null) {
+				return tokenizer;
+			}
+		}
+		throw new IllegalStateException("No tokenizer available");
+	}
+	
+	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
+		return this.resolve(modelProfile).tokenize(txt, modelProfile);
+	}
+	
+	public int count(final String txt, final ModelProfile modelProfile) {
+		return this.resolve(modelProfile).count(txt, modelProfile);
+	}
+	
+	public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
+		return this.resolve(modelProfile).detokenize(tokens, modelProfile);
+	}
+	
+}