diff --git a/README.md b/README.md index b63d1ed..1be4051 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ public CacheManager cacheManager() { ### Tokenizer Dependencies -For the recommended `jTokkit` tokenizer: +For the `jTokkit` tokenizer: ```xml @@ -130,6 +130,14 @@ For the recommended `jTokkit` tokenizer: ``` +## Tokenizers + +lm-lib registers tokenizer implementations as beans when their dependencies are available, then resolves the tokenizer at runtime from the active `ModelProfile`. Applications can implement the `Tokenizer` interface to inject their own Tokenizers. + +Tokenizer selection is configured per model profile with the `tokenizer` property. If a profile does not specify a tokenizer, lm-lib prefers `jTokkit` when the dependency is present and otherwise falls back to the built-in `fallback` tokenizer. The fallback tokenizer is deterministic and reversible, but it only provides approximate token counts. + +As an alternative, the library supports Hugging Face-style /tokenize and /detokenize endpoints (`llamaServer`). These endpoints must be available below base URL from the modelProfile. +These endpoints are provided, for example, by llama.cpp llama-server and vLLM. For `llamaServer`, no extra dependency is required because tokenization is delegated to the remote server through the configured model endpoint. Available built-in tokenizer names: @@ -138,12 +146,6 @@ Available built-in tokenizer names: - `llamaServer` - `fallback` -### Tokenizer Configuration - -lm-lib registers tokenizer implementations as beans when their dependencies are available, then resolves the tokenizer at runtime from the active `ModelProfile`. Applications can implement the `Tokenizer` interface to inject their own Tokenizers. - -Tokenizer selection is configured per model profile with the `tokenizer` property. If a profile does not specify a tokenizer, lm-lib prefers `jTokkit` when the dependency is present and otherwise falls back to the built-in `fallback` tokenizer. The fallback tokenizer is deterministic and reversible, but it only provides approximate token counts. - ## Model Profiles Model profiles describe how a request should be sent to a model and how its usage should be accounted for. They can be loaded from `.properties` files, selected dynamically at runtime, or created programmatically. diff --git a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java index a44feb9..2fcf7bb 100755 --- a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java +++ b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java @@ -1,5 +1,7 @@ package de.extio.lmlib.token; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.List; import org.slf4j.Logger; @@ -36,10 +38,17 @@ final class LlamaServerTokenizer implements Tokenizer { .uri("/tokenize") .body(request) .retrieve() + .onStatus(status -> { + if (status.getStatusCode().isError()) { + LOGGER.error("Tokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8)); + throw new RuntimeException("Tokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText()); + } + return false; + }) .body(TokenizeResponse.class); return response.tokens(); } - + @Override public int count(final String txt, final ModelProfile modelProfile) { return this.tokenize(txt, modelProfile).size(); @@ -55,6 +64,13 @@ final class LlamaServerTokenizer implements Tokenizer { .uri("/detokenize") .body(request) .retrieve() + .onStatus(status -> { + if (status.getStatusCode().isError()) { + LOGGER.error("Detokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8)); + throw new RuntimeException("Detokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText()); + } + return false; + }) .body(DetokenizeResponse.class); return response.content(); }