diff --git a/README.md b/README.md index fedd3a0..15df9f6 100644 --- a/README.md +++ b/README.md @@ -105,12 +105,6 @@ If you disable a module that another module depends on, you must provide the mis | `lmlib.client.retry.backoff-interval-min` | Long | `125` | Minimum retry backoff in milliseconds. | | `lmlib.client.retry.backoff-interval-max` | Long | `2500` | Maximum retry backoff in milliseconds. | -### Tokenizer Configuration - -| Property | Type | Default | Description | -|----------|------|---------|-------------| -| `tokenizer.strategy` | String | `jTokkit` | Token counting strategy. Supported values are `jTokkit` and `llamaServer`. | - ## Optional Dependencies ### Caching Support @@ -148,7 +142,19 @@ For the recommended `jTokkit` tokenizer: ``` -For `llamaServer`, no extra dependency is required because tokenization is delegated to the remote server. +For `llamaServer`, no extra dependency is required because tokenization is delegated to the remote server through the configured model endpoint. + +Available built-in tokenizer names: + +- `jTokkit` +- `llamaServer` +- `fallback` + +### Tokenizer Configuration + +lm-lib registers tokenizer implementations as beans when their dependencies are available, then resolves the tokenizer at runtime from the active `ModelProfile`. Applications can implement the `Tokenizer` interface to inject their own Tokenizers. + +Tokenizer selection is configured per model profile with the `tokenizer` property. If a profile does not specify a tokenizer, lm-lib prefers `jTokkit` when the dependency is present and otherwise falls back to the built-in `fallback` tokenizer. The fallback tokenizer is deterministic and reversible, but it only provides approximate token counts. ## Model Profiles @@ -194,6 +200,7 @@ Create a `ModelProfile` directly when you want full runtime control and do not w ```java final var modelProfile = new ModelProfile( "", // prompts + "jTokkit", // tokenizer "cl100k_base", // tokenEncoding 2000, // maxTokens 32768, // maxContextLength @@ -241,6 +248,7 @@ Each profile file is a `.properties` resource on the classpath. |----------|------|----------|-------------|---------| | `category` | String | No | Short profile label used in logging and graph traces. If omitted in the file, it is inferred from the `ModelCategory` used to load it. | `M`, `L`, `CUSTOM` | | `prompts` | String | Yes | Prompt strategy name for text completions. Use an empty value for chat completions. | `llama4`, `gpt-oss`, `no`, `` | +| `tokenizer` | String | No | Tokenizer name used for counting and prompt trimming. Defaults to `jTokkit` when available, otherwise `fallback`. | `jTokkit`, `llamaServer`, `fallback` | | `tokenEncoding` | String | Yes | Token encoding scheme used for token counting. | `cl100k_base`, `none` | | `maxTokens` | Integer | Yes | Maximum response tokens. | `1500`, `2500` | | `maxContextLength` | Integer | Yes | Maximum total context length. | `16000`, `32768`, `128000` | @@ -265,6 +273,7 @@ Notes: - `cost1MCachedInTokens` defaults to the normal input token cost when not provided. - `cost1MReasoningOutTokens` defaults to `0` when not provided. - Reasoning properties only apply to models that expose separate reasoning output. +- `tokenEncoding` is primarily relevant for tokenizer implementations that support multiple encodings, such as `jTokkit`. ### Example Profile Files @@ -273,6 +282,7 @@ Local chat completion: ```properties category=M prompts= +tokenizer=llamaServer tokenEncoding=cl100k_base maxTokens=2500 maxContextLength=16000 @@ -293,6 +303,7 @@ Local text completion: ```properties category=S prompts=gpt-oss +tokenizer=llamaServer tokenEncoding=cl100k_base maxTokens=2500 maxContextLength=16000 diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index 7b63e11..ca999cf 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -39,7 +39,7 @@ import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.profile.ModelProfileService; -import de.extio.lmlib.token.Tokenizer; +import de.extio.lmlib.token.TokenizerResolver; public abstract class AbstractCompletionClient implements Client, DisposableBean { @@ -50,7 +50,7 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean protected RestClient.Builder restClientBuilder; @Autowired - protected Tokenizer tokenizer; + protected TokenizerResolver tokenizerResolver; @Autowired protected ModelProfileService modelProfileService; @@ -214,14 +214,14 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean } // Fallback to tokenizer estimation if (inTokens <= 0 && prompt != null && !prompt.isBlank()) { - inTokens = this.tokenizer.count(prompt, modelProfile); + inTokens = this.tokenizerResolver.count(prompt, modelProfile); } if (reasoningOutTokens <= 0 && reasoning != null && !reasoning.isBlank()) { - reasoningOutTokens = this.tokenizer.count(reasoning, modelProfile); + reasoningOutTokens = this.tokenizerResolver.count(reasoning, modelProfile); outTokens -= reasoningOutTokens; } if (outTokens <= 0 && response != null && !response.isBlank()) { - outTokens = this.tokenizer.count(response, modelProfile); + outTokens = this.tokenizerResolver.count(response, modelProfile); } // Calculate cost - inTokens and outTokens are already exclusive of cached/reasoning tokens diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index eb64dcc..ff9eba5 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -308,13 +308,13 @@ public class ChatCompletionClient extends AbstractCompletionClient { var prompt = turn.text() != null ? turn.text() : ""; var trimmed = false; - final var textTokens = this.tokenizer.count(prompt, modelProfile); + final var textTokens = this.tokenizerResolver.count(prompt, modelProfile); final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens if (tokens.addAndGet(textTokens) > maxLength) { - var tokenized = this.tokenizer.tokenize(prompt, modelProfile); + var tokenized = this.tokenizerResolver.tokenize(prompt, modelProfile); LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength); tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength)); - prompt = this.tokenizer.detokenize(tokenized, modelProfile); + prompt = this.tokenizerResolver.detokenize(tokenized, modelProfile); trimmed = true; } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 0916e15..44ed82a 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -200,12 +200,12 @@ public class TextCompletionClient extends AbstractCompletionClient { private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) { StringBuilder prompt = this.formatConversation(conversation, promptStrategy); - List tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile); + List tokenized = this.tokenizerResolver.tokenize(prompt.toString(), modelProfile); final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens if (tokenized.size() > maxLength) { LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength); tokenized = tokenized.subList(0, maxLength); - prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile)); + prompt = new StringBuilder(this.tokenizerResolver.detokenize(tokenized, modelProfile)); } LOGGER.debug("Completion request for {}. Input tokens: {}", modelProfile.category(), tokenized.size()); diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfile.java b/src/main/java/de/extio/lmlib/profile/ModelProfile.java index b104b64..ea31fc8 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfile.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfile.java @@ -4,6 +4,7 @@ import java.math.BigDecimal; public record ModelProfile( String prompt, + String tokenizer, String tokenEncoding, int maxTokens, int maxContextLength, diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java index decddb5..d44b2ad 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java @@ -40,6 +40,9 @@ public class ModelProfileService { final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts")) .map(Object::toString) .orElse(""); + final var tokenizer = Optional.ofNullable(resource.getProperty("tokenizer")) + .map(Object::toString) + .orElse(null); final var tokenEncoding = Optional.ofNullable(resource.getProperty("tokenEncoding")) .map(Object::toString) .orElse("none"); @@ -109,7 +112,7 @@ public class ModelProfileService { .orElse(null); } - return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category); + return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category); } catch (final IOException e) { LOGGER.error("Error while reading model profile", e); diff --git a/src/main/java/de/extio/lmlib/token/FallbackTokenizer.java b/src/main/java/de/extio/lmlib/token/FallbackTokenizer.java new file mode 100644 index 0000000..8d2df29 --- /dev/null +++ b/src/main/java/de/extio/lmlib/token/FallbackTokenizer.java @@ -0,0 +1,169 @@ +package de.extio.lmlib.token; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import de.extio.lmlib.profile.ModelProfile; + +final class FallbackTokenizer implements Tokenizer { + + static final String NAME = "fallback"; + + @Override + public String getName() { + return NAME; + } + + @Override + public List tokenize(final String txt, final ModelProfile modelProfile) { + if (txt == null || txt.isEmpty()) { + return List.of(); + } + final var tokens = new ArrayList(); + var index = 0; + while (index < txt.length()) { + final var codePoint = txt.codePointAt(index); + final var charCount = Character.charCount(codePoint); + if (Character.isWhitespace(codePoint)) { + final var end = this.consumeWhitespace(txt, index); + tokens.addAll(this.encodeChunked(txt.substring(index, end), 6)); + index = end; + continue; + } + if (Character.isLetterOrDigit(codePoint)) { + final var end = this.consumeAlphaNumeric(txt, index); + tokens.addAll(this.encodeWordLike(txt.substring(index, end))); + index = end; + continue; + } + tokens.add(this.encodeToken(txt.substring(index, index + charCount))); + index += charCount; + } + return List.copyOf(tokens); + } + + @Override + public int count(final String txt, final ModelProfile modelProfile) { + return this.tokenize(txt, modelProfile).size(); + } + + @Override + public String detokenize(final List tokens, final ModelProfile modelProfile) { + final var builder = new StringBuilder(); + for (final var token : tokens) { + builder.append(this.decodeToken(token)); + } + return builder.toString(); + } + + private int consumeWhitespace(final String txt, final int start) { + var index = start; + while (index < txt.length()) { + final var codePoint = txt.codePointAt(index); + if (!Character.isWhitespace(codePoint)) { + break; + } + index += Character.charCount(codePoint); + } + return index; + } + + private int consumeAlphaNumeric(final String txt, final int start) { + var index = start; + while (index < txt.length()) { + final var codePoint = txt.codePointAt(index); + if (!Character.isLetterOrDigit(codePoint)) { + break; + } + index += Character.charCount(codePoint); + } + return index; + } + + private List encodeWordLike(final String text) { + if (text.isEmpty()) { + return List.of(); + } + if (this.isAsciiDigits(text)) { + return this.encodeChunked(text, 3); + } + if (this.isAsciiLetters(text)) { + return this.encodeChunked(text, 4); + } + return this.encodeChunkedByUtf8Bytes(text, 6); + } + + private boolean isAsciiDigits(final String text) { + return text.chars().allMatch(ch -> ch >= '0' && ch <= '9'); + } + + private boolean isAsciiLetters(final String text) { + return text.chars().allMatch(ch -> ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z'); + } + + private List encodeChunked(final String text, final int maxCharsPerToken) { + final var tokens = new ArrayList(); + for (var index = 0; index < text.length();) { + var end = index; + var charCount = 0; + while (end < text.length() && charCount < maxCharsPerToken) { + final var codePoint = text.codePointAt(end); + end += Character.charCount(codePoint); + charCount++; + } + tokens.add(this.encodeToken(text.substring(index, end))); + index = end; + } + return tokens; + } + + private List encodeChunkedByUtf8Bytes(final String text, final int maxBytesPerToken) { + final var tokens = new ArrayList(); + for (var index = 0; index < text.length();) { + var end = index; + var bytes = 0; + while (end < text.length()) { + final var codePoint = text.codePointAt(end); + final var candidateEnd = end + Character.charCount(codePoint); + final var candidateBytes = text.substring(end, candidateEnd).getBytes(StandardCharsets.UTF_8).length; + if (bytes > 0 && bytes + candidateBytes > maxBytesPerToken) { + break; + } + end = candidateEnd; + bytes += candidateBytes; + if (bytes >= maxBytesPerToken) { + break; + } + } + tokens.add(this.encodeToken(text.substring(index, end))); + index = end; + } + return tokens; + } + + private long encodeToken(final String tokenText) { + final var bytes = tokenText.getBytes(StandardCharsets.UTF_8); + if (bytes.length == 0 || bytes.length > 7) { + throw new IllegalArgumentException("Fallback tokenizer token length must be between 1 and 7 bytes"); + } + long value = bytes.length; + for (var i = 0; i < bytes.length; i++) { + value |= (bytes[i] & 0xffL) << ((i + 1) * 8); + } + return value; + } + + private String decodeToken(final long token) { + final var length = (int) (token & 0xffL); + if (length <= 0 || length > 7) { + throw new IllegalArgumentException("Invalid fallback tokenizer token length: " + length); + } + final var bytes = new byte[length]; + for (var i = 0; i < length; i++) { + bytes[i] = (byte) ((token >> ((i + 1) * 8)) & 0xffL); + } + return new String(bytes, StandardCharsets.UTF_8); + } + +} diff --git a/src/main/java/de/extio/lmlib/token/JTokkitTokenizer.java b/src/main/java/de/extio/lmlib/token/JTokkitTokenizer.java index 16a36d7..a3636d4 100755 --- a/src/main/java/de/extio/lmlib/token/JTokkitTokenizer.java +++ b/src/main/java/de/extio/lmlib/token/JTokkitTokenizer.java @@ -14,6 +14,8 @@ import de.extio.lmlib.profile.ModelProfile; final class JTokkitTokenizer implements Tokenizer { + static final String NAME = "jTokkit"; + private static final EncodingRegistry registry; static { @@ -26,10 +28,15 @@ final class JTokkitTokenizer implements Tokenizer { return registry.getEncoding(EncodingType.fromName(k).orElseThrow(() -> new IllegalArgumentException("Token encoding " + k))); }); + @Override + public String getName() { + return NAME; + } + @Override public List tokenize(final String txt, final ModelProfile modelProfile) { return this.encodings - .get(modelProfile.tokenEncoding()) + .get(this.getTokenEncoding(modelProfile)) .encodeOrdinary(txt) .boxed() .stream() @@ -39,7 +46,7 @@ final class JTokkitTokenizer implements Tokenizer { @Override public int count(final String txt, final ModelProfile modelProfile) { - return this.encodings.get(modelProfile.tokenEncoding()).countTokensOrdinary(txt); + return this.encodings.get(this.getTokenEncoding(modelProfile)).countTokensOrdinary(txt); } @Override @@ -47,7 +54,14 @@ final class JTokkitTokenizer implements Tokenizer { final var ial = new IntArrayList(tokens.size()); tokens.forEach(t -> ial.add(t.intValue())); return this.encodings - .get(modelProfile.tokenEncoding()) + .get(this.getTokenEncoding(modelProfile)) .decode(ial); } + + private String getTokenEncoding(final ModelProfile modelProfile) { + if (modelProfile.tokenEncoding() == null || modelProfile.tokenEncoding().isBlank() || "none".equalsIgnoreCase(modelProfile.tokenEncoding())) { + return "cl100k_base"; // default encoding, used by most models. Will be used if the profile doesn't specify an encoding or specifies an invalid one. + } + return modelProfile.tokenEncoding(); + } } diff --git a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java index 803dd00..a44feb9 100755 --- a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java +++ b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java @@ -1,6 +1,5 @@ package de.extio.lmlib.token; -import java.time.Duration; import java.util.List; import org.slf4j.Logger; @@ -12,6 +11,8 @@ import de.extio.lmlib.profile.ModelProfile; final class LlamaServerTokenizer implements Tokenizer { + static final String NAME = "llamaServer"; + private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class); private final RestClient.Builder restClientBuilder; @@ -20,6 +21,11 @@ final class LlamaServerTokenizer implements Tokenizer { this.restClientBuilder = restClientBuilder; } + @Override + public String getName() { + return NAME; + } + @Override public List tokenize(final String txt, final ModelProfile modelProfile) { LOGGER.debug("Requesting tokenizing"); diff --git a/src/main/java/de/extio/lmlib/token/Tokenizer.java b/src/main/java/de/extio/lmlib/token/Tokenizer.java index 9b22b24..c7e747d 100644 --- a/src/main/java/de/extio/lmlib/token/Tokenizer.java +++ b/src/main/java/de/extio/lmlib/token/Tokenizer.java @@ -6,6 +6,8 @@ import de.extio.lmlib.profile.ModelProfile; public interface Tokenizer { + String getName(); + List tokenize(String txt, ModelProfile modelProfile); int count(String txt, ModelProfile modelProfile); diff --git a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java index 094480b..40943ee 100644 --- a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java +++ b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java @@ -2,6 +2,7 @@ package de.extio.lmlib.token; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -13,18 +14,29 @@ import org.springframework.web.client.RestClient; public class TokenizerAutoConfiguration { @Bean - @ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer") - @ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer") + @ConditionalOnBean(name = "lmLibRestClientBuilder") + @ConditionalOnMissingBean(LlamaServerTokenizer.class) Tokenizer llamaServerTokenizer(@Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder) { return new LlamaServerTokenizer(restClientBuilder); } @Bean @ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings") - @ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "jTokkit", matchIfMissing = true) - @ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer") + @ConditionalOnMissingBean(JTokkitTokenizer.class) Tokenizer jTokkitTokenizer() { return new JTokkitTokenizer(); } + + @Bean + @ConditionalOnMissingBean(FallbackTokenizer.class) + Tokenizer fallbackTokenizer() { + return new FallbackTokenizer(); + } + + @Bean + @ConditionalOnMissingBean + TokenizerResolver tokenizerResolver(final java.util.List tokenizers) { + return new TokenizerResolver(tokenizers); + } } diff --git a/src/main/java/de/extio/lmlib/token/TokenizerResolver.java b/src/main/java/de/extio/lmlib/token/TokenizerResolver.java new file mode 100644 index 0000000..134ab2a --- /dev/null +++ b/src/main/java/de/extio/lmlib/token/TokenizerResolver.java @@ -0,0 +1,55 @@ +package de.extio.lmlib.token; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import de.extio.lmlib.profile.ModelProfile; + +public class TokenizerResolver { + + private static final List DEFAULT_TOKENIZER_ORDER = List.of(JTokkitTokenizer.NAME, FallbackTokenizer.NAME); + + private final Map tokenizersByName; + + public TokenizerResolver(final List tokenizers) { + final var tokenizersByName = new LinkedHashMap(); + for (final var tokenizer : tokenizers) { + final var previous = tokenizersByName.put(tokenizer.getName(), tokenizer); + if (previous != null) { + throw new IllegalStateException("Duplicate tokenizer name: " + tokenizer.getName()); + } + } + this.tokenizersByName = Map.copyOf(tokenizersByName); + } + + public Tokenizer resolve(final ModelProfile modelProfile) { + if (modelProfile != null && modelProfile.tokenizer() != null && !modelProfile.tokenizer().isBlank()) { + final var tokenizer = this.tokenizersByName.get(modelProfile.tokenizer()); + if (tokenizer == null) { + throw new IllegalStateException("No tokenizer configured with name: " + modelProfile.tokenizer()); + } + return tokenizer; + } + for (final var tokenizerName : DEFAULT_TOKENIZER_ORDER) { + final var tokenizer = this.tokenizersByName.get(tokenizerName); + if (tokenizer != null) { + return tokenizer; + } + } + throw new IllegalStateException("No tokenizer available"); + } + + public List tokenize(final String txt, final ModelProfile modelProfile) { + return this.resolve(modelProfile).tokenize(txt, modelProfile); + } + + public int count(final String txt, final ModelProfile modelProfile) { + return this.resolve(modelProfile).count(txt, modelProfile); + } + + public String detokenize(final List tokens, final ModelProfile modelProfile) { + return this.resolve(modelProfile).detokenize(tokens, modelProfile); + } + +} \ No newline at end of file diff --git a/src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java b/src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java new file mode 100644 index 0000000..a72c933 --- /dev/null +++ b/src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java @@ -0,0 +1,92 @@ +package de.extio.lmlib.token; + +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.math.BigDecimal; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; + +public class TokenizerResolverTest { + + @Test + void resolvesTokenizerFromProfileName() { + final var fallbackTokenizer = new FallbackTokenizer(); + final var customTokenizer = new NamedTokenizer("custom"); + final var resolver = new TokenizerResolver(List.of(fallbackTokenizer, customTokenizer)); + + assertSame(customTokenizer, resolver.resolve(this.createProfile("custom"))); + } + + @Test + void prefersJTokkitAsDefaultWhenAvailable() { + final var fallbackTokenizer = new FallbackTokenizer(); + final var jTokkitTokenizer = new NamedTokenizer(JTokkitTokenizer.NAME); + final var resolver = new TokenizerResolver(List.of(fallbackTokenizer, jTokkitTokenizer)); + + assertSame(jTokkitTokenizer, resolver.resolve(this.createProfile(null))); + } + + @Test + void fallsBackWhenNoPreferredTokenizerExists() { + final var fallbackTokenizer = new FallbackTokenizer(); + final var resolver = new TokenizerResolver(List.of(fallbackTokenizer)); + + assertSame(fallbackTokenizer, resolver.resolve(this.createProfile(null))); + } + + private ModelProfile createProfile(final String tokenizer) { + return new ModelProfile( + "", + tokenizer, + "cl100k_base", + 1024, + 4096, + 0.4, + 1.0, + ModelProvider.OAI_CHAT_COMPLETION, + "test-model", + "http://localhost:5001", + "", + BigDecimal.ZERO, + BigDecimal.ZERO, + BigDecimal.ZERO, + BigDecimal.ZERO, + null, + null, + "TEST"); + } + + private static final class NamedTokenizer implements Tokenizer { + + private final String name; + + private NamedTokenizer(final String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public List tokenize(final String txt, final ModelProfile modelProfile) { + return List.of(); + } + + @Override + public int count(final String txt, final ModelProfile modelProfile) { + return 0; + } + + @Override + public String detokenize(final List tokens, final ModelProfile modelProfile) { + return ""; + } + } + +} \ No newline at end of file diff --git a/src/test/java/de/extio/lmlib/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/token/TokenizerTest.java index a6ea610..8775dbb 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerTest.java @@ -1,6 +1,9 @@ package de.extio.lmlib.token; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.math.BigDecimal; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -41,6 +44,29 @@ public class TokenizerTest { assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString()); assertEquals(TEXT, tokenizer.detokenize(tokens, profile)); } + + @Test + void fallback() { + final var tokenizer = new FallbackTokenizer(); + final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile()); + + assertEquals(47, tokenizer.count(TEXT, profile)); + final var tokens = tokenizer.tokenize(TEXT, profile); + assertEquals(47, tokens.size()); + assertEquals("[465675044868, 28417, 8193, 495689692164, 8193, 7563522, 8193, 24833, 8193, 500236121092, 8449, 8193, 2003781635, 8193, 1701994755, 8193, 1970239747, 8193, 474215310340, 26369, 16129, 8193, 1701336067, 8193, 499850245892, 1919248387, 8193, 7563522, 8193, 418313824004, 29697, 8193, 418296853508, 30977, 8449, 8193, 435678434052, 29185, 8193, 474299593476, 30977, 8193, 474214855428, 30977, 8449, 8449, 8449]", tokens.toString()); + assertEquals(TEXT, tokenizer.detokenize(tokens, profile)); + } + + @Test + void fallbackTokenizerGroupsWhitespace() { + final var tokenizer = new FallbackTokenizer(); + final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile()); + final var text = " "; + final var tokens = tokenizer.tokenize(text, profile); + + assertEquals(2, tokens.size()); + assertEquals(text, tokenizer.detokenize(tokens, profile)); + } /** * Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model. diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties index dceb105..676d257 100755 --- a/src/test/resources/application-test.properties +++ b/src/test/resources/application-test.properties @@ -11,5 +11,3 @@ profile.model.chatcompletion=local-chatcompletion agent.threads=2 client.collectStatistics=true - -tokenizer.strategy=llamaServer diff --git a/src/test/resources/local-chatcompletion.properties b/src/test/resources/local-chatcompletion.properties index c0285a0..608ecfc 100755 --- a/src/test/resources/local-chatcompletion.properties +++ b/src/test/resources/local-chatcompletion.properties @@ -1,5 +1,6 @@ category=CHAT prompts= +tokenizer=llamaServer tokenEncoding=cl100k_base maxTokens=2500 maxContextLength=16000 diff --git a/src/test/resources/local-completion.properties b/src/test/resources/local-completion.properties index 276f0ac..d8ad0d3 100755 --- a/src/test/resources/local-completion.properties +++ b/src/test/resources/local-completion.properties @@ -1,5 +1,6 @@ category=TEXT prompts=gpt-oss +tokenizer=llamaServer tokenEncoding=cl100k_base maxTokens=2500 maxContextLength=16000