diff --git a/README.md b/README.md index 8e52e4e..b299644 100644 --- a/README.md +++ b/README.md @@ -141,10 +141,13 @@ As an alternative, the library supports Hugging Face-style /tokenize and /detoke These endpoints are provided, for example, by llama.cpp llama-server and vLLM. For `llamaServer`, no extra dependency is required because tokenization is delegated to the remote server through the configured model endpoint. +The `anthropic` tokenizer delegates counting to the Anthropic token counting endpoint (`POST /v1/messages/count_tokens`), which must be available below the base URL from the model profile. The endpoint only supports counting, so `tokenize` and `detokenize` operate on a plain codepoint list instead of real model tokens. + Available built-in tokenizer names: - `jTokkit` - `llamaServer` +- `anthropic` - `fallback` ## Model Profiles diff --git a/src/main/java/de/extio/lmlib/token/AnthropicTokenizer.java b/src/main/java/de/extio/lmlib/token/AnthropicTokenizer.java new file mode 100644 index 0000000..6c91897 --- /dev/null +++ b/src/main/java/de/extio/lmlib/token/AnthropicTokenizer.java @@ -0,0 +1,155 @@ +package de.extio.lmlib.token; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpMethod; +import org.springframework.web.client.RestClient; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import de.extio.lmlib.client.ProxyAuthorizationSupport; +import de.extio.lmlib.profile.ModelProfile; + +/** + * {@link Tokenizer} implementation backed by the Anthropic token counting + * endpoint ({@code POST /v1/messages/count_tokens}). + *

+ * The API only supports counting, so {@link #tokenize(String, ModelProfile)} + * and {@link #detokenize(List, ModelProfile)} work with a plain codepoint list + * instead of real model tokens. + */ +final class AnthropicTokenizer implements Tokenizer { + + static final String NAME = "anthropic"; + + private static final String COUNT_TOKENS_PATH = "/v1/messages/count_tokens"; + + private static final String ANTHROPIC_VERSION = "2023-06-01"; + + private static final Logger LOGGER = LoggerFactory.getLogger(AnthropicTokenizer.class); + + private final RestClient.Builder restClientBuilder; + + private final ProxyAuthorizationSupport proxyAuthorizationSupport; + + public AnthropicTokenizer(final RestClient.Builder restClientBuilder, final ProxyAuthorizationSupport proxyAuthorizationSupport) { + this.restClientBuilder = restClientBuilder; + this.proxyAuthorizationSupport = proxyAuthorizationSupport; + } + + @Override + public String getName() { + return NAME; + } + + /** + * Returns the codepoints of the given text, since the Anthropic token + * counting endpoint does not expose token ids. + */ + @Override + public List tokenize(final String txt, final ModelProfile modelProfile) { + if (txt == null || txt.isEmpty()) { + return List.of(); + } + return txt.codePoints().mapToObj(codePoint -> (long) codePoint).toList(); + } + + @Override + public int count(final String txt, final ModelProfile modelProfile) { + if (txt == null || txt.isEmpty()) { + return 0; + } + LOGGER.debug("Requesting token count"); + final var modelName = modelProfile.modelName(); + if (modelName == null || modelName.isBlank()) { + throw new IllegalStateException("Anthropic token counting requires a model name in the model profile"); + } + final var url = modelProfile.url(); + final var baseUrl = stripCountTokensSuffix(url.endsWith("/") ? url.substring(0, url.length() - 1) : url); + final var restClient = this.restClientBuilder.baseUrl(baseUrl).build(); + var requestSpec = restClient + .method(HttpMethod.POST) + .uri(countTokensPath(url)) + .header("anthropic-version", ANTHROPIC_VERSION); + if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { + requestSpec = requestSpec.header("x-api-key", modelProfile.apiKey()); + } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); + final var response = requestSpec + .body(new CountTokensRequest(modelName, List.of(new CountTokensMessage("user", txt)))) + .retrieve() + .onStatus(status -> { + if (status.getStatusCode().isError()) { + LOGGER.error("Token count request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8)); + throw new RuntimeException("Token count request failed with status code " + status.getStatusCode() + " " + status.getStatusText() + ". Your LLM provider may not support the /v1/messages/count_tokens endpoint"); + } + return false; + }) + .body(CountTokensResponse.class); + if (response == null || response.inputTokens() == null) { + throw new RuntimeException("Token count request failed: response is null. Your LLM provider may not support the /v1/messages/count_tokens endpoint"); + } + return response.inputTokens(); + } + + /** + * Rebuilds the text from its codepoints, since the Anthropic token counting + * endpoint does not expose token ids. + */ + @Override + public String detokenize(final List tokens, final ModelProfile modelProfile) { + if (tokens == null || tokens.isEmpty()) { + return ""; + } + final var builder = new StringBuilder(tokens.size()); + for (final var token : tokens) { + if (token != null) { + builder.append(Character.toString(token.intValue())); + } + } + return builder.toString(); + } + + /** + * Resolves the count tokens endpoint path relative to the configured base + * URL, tolerating URLs that already end in {@code /v1} or + * {@code /v1/messages}. + */ + static String countTokensPath(final String url) { + if (url == null) { + return COUNT_TOKENS_PATH; + } + if (url.endsWith("/v1/messages") || url.endsWith("/v1/messages/")) { + return "/count_tokens"; + } + if (url.endsWith("/v1/") || url.endsWith("/v1")) { + return "/messages/count_tokens"; + } + return COUNT_TOKENS_PATH; + } + + /** + * Normalizes the configured base URL by removing a trailing + * {@code /v1/messages} endpoint suffix (the request path is appended + * separately). + */ + static String stripCountTokensSuffix(final String url) { + if (url != null && url.endsWith("/v1/messages")) { + return url.substring(0, url.length() - "/v1/messages".length()); + } + return url; + } + + private record CountTokensRequest(String model, List messages) { + } + + private record CountTokensMessage(String role, String content) { + } + + private record CountTokensResponse(@JsonProperty("input_tokens") Integer inputTokens) { + } + +} diff --git a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java index dc0c9d6..6b8e1ff 100644 --- a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java +++ b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java @@ -24,6 +24,15 @@ public class TokenizerAutoConfiguration { return new LlamaServerTokenizer(restClientBuilder, proxyAuthorizationSupport); } + @Bean + @ConditionalOnBean(name = "lmLibRestClientBuilder") + @ConditionalOnMissingBean(AnthropicTokenizer.class) + Tokenizer anthropicTokenizer( + @Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder, + final ProxyAuthorizationSupport proxyAuthorizationSupport) { + return new AnthropicTokenizer(restClientBuilder, proxyAuthorizationSupport); + } + @Bean @ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings") @ConditionalOnMissingBean(JTokkitTokenizer.class) diff --git a/src/test/java/de/extio/lmlib/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/token/TokenizerTest.java index d56efca..c24618b 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerTest.java @@ -1,6 +1,9 @@ package de.extio.lmlib.token; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -80,4 +83,39 @@ public class TokenizerTest { assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString()); assertEquals(TEXT, tokenizer.detokenize(tokens, profile)); } + + @Test + void anthropicCodepointRoundTrip() { + final var tokenizer = new AnthropicTokenizer(this.restClientBuilder, proxyAuthorizationSupport()); + final var profile = this.modelProfileService.getModelProfile("profile.model.anthropic"); + + final var tokens = tokenizer.tokenize(TEXT, profile); + assertEquals(TEXT.codePoints().count(), tokens.size()); + assertEquals(TEXT, tokenizer.detokenize(tokens, profile)); + assertEquals(0, tokenizer.count("", profile)); + assertEquals(0, tokenizer.tokenize("", profile).size()); + assertEquals("", tokenizer.detokenize(List.of(), profile)); + } + + @Test + void anthropicCountTokensPath() { + assertEquals("/v1/messages/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001")); + assertEquals("/messages/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001/v1")); + assertEquals("/messages/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001/v1/")); + assertEquals("/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001/v1/messages")); + assertEquals("http://127.0.0.1:5001", AnthropicTokenizer.stripCountTokensSuffix("http://127.0.0.1:5001/v1/messages")); + } + + @Disabled("This test requires a running Anthropic server") + @Test + void anthropic() throws Exception { + final var tokenizer = new AnthropicTokenizer(this.restClientBuilder, proxyAuthorizationSupport()); + final var profile = this.modelProfileService.getModelProfile("profile.model.anthropic"); + + assertTrue(tokenizer.count(TEXT, profile) > 0); + } + + private static ProxyAuthorizationSupport proxyAuthorizationSupport() { + return new ProxyAuthorizationSupport(ProxyAuthorizationSupport.MODE_HTTP_CLIENT, "", ""); + } } diff --git a/src/test/resources/local-anthropic.properties b/src/test/resources/local-anthropic.properties index 30340ee..fa67fd3 100644 --- a/src/test/resources/local-anthropic.properties +++ b/src/test/resources/local-anthropic.properties @@ -1,6 +1,6 @@ category=ANTHROPIC prompts= -tokenizer=fallback +tokenizer=anthropic tokenEncoding=cl100k_base maxTokens=2500 maxContextLength=180000