feat: Implement tokenizer resolver and fallback tokenizer; update model profiles and README

src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java

@@ -0,0 +1,92 @@
+package de.extio.lmlib.token;
+
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import de.extio.lmlib.profile.ModelProfile;
+import de.extio.lmlib.profile.ModelProfile.ModelProvider;
+
+public class TokenizerResolverTest {
+	
+	@Test
+	void resolvesTokenizerFromProfileName() {
+		final var fallbackTokenizer = new FallbackTokenizer();
+		final var customTokenizer = new NamedTokenizer("custom");
+		final var resolver = new TokenizerResolver(List.of(fallbackTokenizer, customTokenizer));
+		
+		assertSame(customTokenizer, resolver.resolve(this.createProfile("custom")));
+	}
+	
+	@Test
+	void prefersJTokkitAsDefaultWhenAvailable() {
+		final var fallbackTokenizer = new FallbackTokenizer();
+		final var jTokkitTokenizer = new NamedTokenizer(JTokkitTokenizer.NAME);
+		final var resolver = new TokenizerResolver(List.of(fallbackTokenizer, jTokkitTokenizer));
+		
+		assertSame(jTokkitTokenizer, resolver.resolve(this.createProfile(null)));
+	}
+	
+	@Test
+	void fallsBackWhenNoPreferredTokenizerExists() {
+		final var fallbackTokenizer = new FallbackTokenizer();
+		final var resolver = new TokenizerResolver(List.of(fallbackTokenizer));
+		
+		assertSame(fallbackTokenizer, resolver.resolve(this.createProfile(null)));
+	}
+	
+	private ModelProfile createProfile(final String tokenizer) {
+		return new ModelProfile(
+				"",
+				tokenizer,
+				"cl100k_base",
+				1024,
+				4096,
+				0.4,
+				1.0,
+				ModelProvider.OAI_CHAT_COMPLETION,
+				"test-model",
+				"http://localhost:5001",
+				"",
+				BigDecimal.ZERO,
+				BigDecimal.ZERO,
+				BigDecimal.ZERO,
+				BigDecimal.ZERO,
+				null,
+				null,
+				"TEST");
+	}
+	
+	private static final class NamedTokenizer implements Tokenizer {
+		
+		private final String name;
+		
+		private NamedTokenizer(final String name) {
+			this.name = name;
+		}
+		
+		@Override
+		public String getName() {
+			return this.name;
+		}
+		
+		@Override
+		public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
+			return List.of();
+		}
+		
+		@Override
+		public int count(final String txt, final ModelProfile modelProfile) {
+			return 0;
+		}
+		
+		@Override
+		public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
+			return "";
+		}
+	}
+	
+}

src/test/java/de/extio/lmlib/token/TokenizerTest.java

@@ -1,6 +1,9 @@
 package de.extio.lmlib.token;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.math.BigDecimal;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -41,6 +44,29 @@ public class TokenizerTest {
 		assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString());
 		assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
 	}
+
+	@Test
+	void fallback() {
+		final var tokenizer = new FallbackTokenizer();
+		final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
+		
+		assertEquals(47, tokenizer.count(TEXT, profile));
+		final var tokens = tokenizer.tokenize(TEXT, profile);
+		assertEquals(47, tokens.size());
+		assertEquals("[465675044868, 28417, 8193, 495689692164, 8193, 7563522, 8193, 24833, 8193, 500236121092, 8449, 8193, 2003781635, 8193, 1701994755, 8193, 1970239747, 8193, 474215310340, 26369, 16129, 8193, 1701336067, 8193, 499850245892, 1919248387, 8193, 7563522, 8193, 418313824004, 29697, 8193, 418296853508, 30977, 8449, 8193, 435678434052, 29185, 8193, 474299593476, 30977, 8193, 474214855428, 30977, 8449, 8449, 8449]", tokens.toString());
+		assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
+	}
+	
+	@Test
+	void fallbackTokenizerGroupsWhitespace() {
+		final var tokenizer = new FallbackTokenizer();
+		final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
+		final var text = "       ";
+		final var tokens = tokenizer.tokenize(text, profile);
+		
+		assertEquals(2, tokens.size());
+		assertEquals(text, tokenizer.detokenize(tokens, profile));
+	}
 	
 	/**
 	 * Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model.

src/test/resources/application-test.properties

@@ -11,5 +11,3 @@ profile.model.chatcompletion=local-chatcompletion
 agent.threads=2
 
 client.collectStatistics=true
-
-tokenizer.strategy=llamaServer

src/test/resources/local-chatcompletion.properties

@@ -1,5 +1,6 @@
 category=CHAT
 prompts=
+tokenizer=llamaServer
 tokenEncoding=cl100k_base
 maxTokens=2500
 maxContextLength=16000

src/test/resources/local-completion.properties

@@ -1,5 +1,6 @@
 category=TEXT
 prompts=gpt-oss
+tokenizer=llamaServer
 tokenEncoding=cl100k_base
 maxTokens=2500
 maxContextLength=16000