feat: add Anthropic tokenizer backed by count_tokens API

Adds the anthropic tokenizer that delegates token counting to the
Anthropic endpoint POST /v1/messages/count_tokens. Since the endpoint
only supports counting, tokenize and detokenize operate on a plain
codepoint list instead of real model tokens.

The local anthropic test profile now uses the new tokenizer.

README.md

@@ -141,10 +141,13 @@ As an alternative, the library supports Hugging Face-style /tokenize and /detoke
 These endpoints are provided, for example, by llama.cpp llama-server and vLLM.
 For `llamaServer`, no extra dependency is required because tokenization is delegated to the remote server through the configured model endpoint.
 
+The `anthropic` tokenizer delegates counting to the Anthropic token counting endpoint (`POST /v1/messages/count_tokens`), which must be available below the base URL from the model profile. The endpoint only supports counting, so `tokenize` and `detokenize` operate on a plain codepoint list instead of real model tokens.
+
 Available built-in tokenizer names:
 
 - `jTokkit`
 - `llamaServer`
+- `anthropic`
 - `fallback`
 
 ## Model Profiles

src/main/java/de/extio/lmlib/token/AnthropicTokenizer.java

@@ -0,0 +1,155 @@
+package de.extio.lmlib.token;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpMethod;
+import org.springframework.web.client.RestClient;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import de.extio.lmlib.client.ProxyAuthorizationSupport;
+import de.extio.lmlib.profile.ModelProfile;
+
+/**
+ * {@link Tokenizer} implementation backed by the Anthropic token counting
+ * endpoint ({@code POST /v1/messages/count_tokens}).
+ * <p>
+ * The API only supports counting, so {@link #tokenize(String, ModelProfile)}
+ * and {@link #detokenize(List, ModelProfile)} work with a plain codepoint list
+ * instead of real model tokens.
+ */
+final class AnthropicTokenizer implements Tokenizer {
+
+	static final String NAME = "anthropic";
+
+	private static final String COUNT_TOKENS_PATH = "/v1/messages/count_tokens";
+
+	private static final String ANTHROPIC_VERSION = "2023-06-01";
+
+	private static final Logger LOGGER = LoggerFactory.getLogger(AnthropicTokenizer.class);
+
+	private final RestClient.Builder restClientBuilder;
+
+	private final ProxyAuthorizationSupport proxyAuthorizationSupport;
+
+	public AnthropicTokenizer(final RestClient.Builder restClientBuilder, final ProxyAuthorizationSupport proxyAuthorizationSupport) {
+		this.restClientBuilder = restClientBuilder;
+		this.proxyAuthorizationSupport = proxyAuthorizationSupport;
+	}
+
+	@Override
+	public String getName() {
+		return NAME;
+	}
+
+	/**
+	 * Returns the codepoints of the given text, since the Anthropic token
+	 * counting endpoint does not expose token ids.
+	 */
+	@Override
+	public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
+		if (txt == null || txt.isEmpty()) {
+			return List.of();
+		}
+		return txt.codePoints().mapToObj(codePoint -> (long) codePoint).toList();
+	}
+
+	@Override
+	public int count(final String txt, final ModelProfile modelProfile) {
+		if (txt == null || txt.isEmpty()) {
+			return 0;
+		}
+		LOGGER.debug("Requesting token count");
+		final var modelName = modelProfile.modelName();
+		if (modelName == null || modelName.isBlank()) {
+			throw new IllegalStateException("Anthropic token counting requires a model name in the model profile");
+		}
+		final var url = modelProfile.url();
+		final var baseUrl = stripCountTokensSuffix(url.endsWith("/") ? url.substring(0, url.length() - 1) : url);
+		final var restClient = this.restClientBuilder.baseUrl(baseUrl).build();
+		var requestSpec = restClient
+				.method(HttpMethod.POST)
+				.uri(countTokensPath(url))
+				.header("anthropic-version", ANTHROPIC_VERSION);
+		if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) {
+			requestSpec = requestSpec.header("x-api-key", modelProfile.apiKey());
+		}
+		requestSpec = this.proxyAuthorizationSupport.apply(requestSpec);
+		final var response = requestSpec
+				.body(new CountTokensRequest(modelName, List.of(new CountTokensMessage("user", txt))))
+				.retrieve()
+				.onStatus(status -> {
+					if (status.getStatusCode().isError()) {
+						LOGGER.error("Token count request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8));
+						throw new RuntimeException("Token count request failed with status code " + status.getStatusCode() + " " + status.getStatusText() + ". Your LLM provider may not support the /v1/messages/count_tokens endpoint");
+					}
+					return false;
+				})
+				.body(CountTokensResponse.class);
+		if (response == null || response.inputTokens() == null) {
+			throw new RuntimeException("Token count request failed: response is null. Your LLM provider may not support the /v1/messages/count_tokens endpoint");
+		}
+		return response.inputTokens();
+	}
+
+	/**
+	 * Rebuilds the text from its codepoints, since the Anthropic token counting
+	 * endpoint does not expose token ids.
+	 */
+	@Override
+	public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
+		if (tokens == null || tokens.isEmpty()) {
+			return "";
+		}
+		final var builder = new StringBuilder(tokens.size());
+		for (final var token : tokens) {
+			if (token != null) {
+				builder.append(Character.toString(token.intValue()));
+			}
+		}
+		return builder.toString();
+	}
+
+	/**
+	 * Resolves the count tokens endpoint path relative to the configured base
+	 * URL, tolerating URLs that already end in {@code /v1} or
+	 * {@code /v1/messages}.
+	 */
+	static String countTokensPath(final String url) {
+		if (url == null) {
+			return COUNT_TOKENS_PATH;
+		}
+		if (url.endsWith("/v1/messages") || url.endsWith("/v1/messages/")) {
+			return "/count_tokens";
+		}
+		if (url.endsWith("/v1/") || url.endsWith("/v1")) {
+			return "/messages/count_tokens";
+		}
+		return COUNT_TOKENS_PATH;
+	}
+
+	/**
+	 * Normalizes the configured base URL by removing a trailing
+	 * {@code /v1/messages} endpoint suffix (the request path is appended
+	 * separately).
+	 */
+	static String stripCountTokensSuffix(final String url) {
+		if (url != null && url.endsWith("/v1/messages")) {
+			return url.substring(0, url.length() - "/v1/messages".length());
+		}
+		return url;
+	}
+
+	private record CountTokensRequest(String model, List<CountTokensMessage> messages) {
+	}
+
+	private record CountTokensMessage(String role, String content) {
+	}
+
+	private record CountTokensResponse(@JsonProperty("input_tokens") Integer inputTokens) {
+	}
+
+}

src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java

@@ -24,6 +24,15 @@ public class TokenizerAutoConfiguration {
 		return new LlamaServerTokenizer(restClientBuilder, proxyAuthorizationSupport);
 	}
 	
+	@Bean
+	@ConditionalOnBean(name = "lmLibRestClientBuilder")
+	@ConditionalOnMissingBean(AnthropicTokenizer.class)
+	Tokenizer anthropicTokenizer(
+			@Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder,
+			final ProxyAuthorizationSupport proxyAuthorizationSupport) {
+		return new AnthropicTokenizer(restClientBuilder, proxyAuthorizationSupport);
+	}
+	
 	@Bean
 	@ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings")
 	@ConditionalOnMissingBean(JTokkitTokenizer.class)

src/test/java/de/extio/lmlib/token/TokenizerTest.java

@@ -1,6 +1,9 @@
 package de.extio.lmlib.token;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -80,4 +83,39 @@ public class TokenizerTest {
 		assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString());
 		assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
 	}
+	
+	@Test
+	void anthropicCodepointRoundTrip() {
+		final var tokenizer = new AnthropicTokenizer(this.restClientBuilder, proxyAuthorizationSupport());
+		final var profile = this.modelProfileService.getModelProfile("profile.model.anthropic");
+		
+		final var tokens = tokenizer.tokenize(TEXT, profile);
+		assertEquals(TEXT.codePoints().count(), tokens.size());
+		assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
+		assertEquals(0, tokenizer.count("", profile));
+		assertEquals(0, tokenizer.tokenize("", profile).size());
+		assertEquals("", tokenizer.detokenize(List.of(), profile));
+	}
+	
+	@Test
+	void anthropicCountTokensPath() {
+		assertEquals("/v1/messages/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001"));
+		assertEquals("/messages/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001/v1"));
+		assertEquals("/messages/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001/v1/"));
+		assertEquals("/count_tokens", AnthropicTokenizer.countTokensPath("http://127.0.0.1:5001/v1/messages"));
+		assertEquals("http://127.0.0.1:5001", AnthropicTokenizer.stripCountTokensSuffix("http://127.0.0.1:5001/v1/messages"));
+	}
+	
+	@Disabled("This test requires a running Anthropic server")
+	@Test
+	void anthropic() throws Exception {
+		final var tokenizer = new AnthropicTokenizer(this.restClientBuilder, proxyAuthorizationSupport());
+		final var profile = this.modelProfileService.getModelProfile("profile.model.anthropic");
+		
+		assertTrue(tokenizer.count(TEXT, profile) > 0);
+	}
+	
+	private static ProxyAuthorizationSupport proxyAuthorizationSupport() {
+		return new ProxyAuthorizationSupport(ProxyAuthorizationSupport.MODE_HTTP_CLIENT, "", "");
+	}
 }

src/test/resources/local-anthropic.properties

@@ -1,6 +1,6 @@
 category=ANTHROPIC
 prompts=
-tokenizer=fallback
+tokenizer=anthropic
 tokenEncoding=cl100k_base
 maxTokens=2500
 maxContextLength=180000