exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement tokenizer resolver and fallback tokenizer; update model profiles and README

Browse Source
This commit is contained in:
extio-de
2026-04-03 18:40:53 +02:00
parent dda1efd1e3
commit b4d1a44d93
17 changed files with 419 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
src/test/java/de/extio/lmlib/token/TokenizerResolverTest.java Normal file
View File

@@ -0,0 +1,92 @@
package de.extio.lmlib.token;
import static org.junit.jupiter.api.Assertions.assertSame;
import java.math.BigDecimal;
import java.util.List;
import org.junit.jupiter.api.Test;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
public class TokenizerResolverTest {
@Test
void resolvesTokenizerFromProfileName() {
final var fallbackTokenizer = new FallbackTokenizer();
final var customTokenizer = new NamedTokenizer("custom");
final var resolver = new TokenizerResolver(List.of(fallbackTokenizer, customTokenizer));
assertSame(customTokenizer, resolver.resolve(this.createProfile("custom")));
}
@Test
void prefersJTokkitAsDefaultWhenAvailable() {
final var fallbackTokenizer = new FallbackTokenizer();
final var jTokkitTokenizer = new NamedTokenizer(JTokkitTokenizer.NAME);
final var resolver = new TokenizerResolver(List.of(fallbackTokenizer, jTokkitTokenizer));
assertSame(jTokkitTokenizer, resolver.resolve(this.createProfile(null)));
}
@Test
void fallsBackWhenNoPreferredTokenizerExists() {
final var fallbackTokenizer = new FallbackTokenizer();
final var resolver = new TokenizerResolver(List.of(fallbackTokenizer));
assertSame(fallbackTokenizer, resolver.resolve(this.createProfile(null)));
}
private ModelProfile createProfile(final String tokenizer) {
return new ModelProfile(
"",
tokenizer,
"cl100k_base",
1024,
4096,
0.4,
1.0,
ModelProvider.OAI_CHAT_COMPLETION,
"test-model",
"http://localhost:5001",
"",
BigDecimal.ZERO,
BigDecimal.ZERO,
BigDecimal.ZERO,
BigDecimal.ZERO,
null,
null,
"TEST");
}
private static final class NamedTokenizer implements Tokenizer {
private final String name;
private NamedTokenizer(final String name) {
this.name = name;
}
@Override
public String getName() {
return this.name;
}
@Override
public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
return List.of();
}
@Override
public int count(final String txt, final ModelProfile modelProfile) {
return 0;
}
@Override
public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
return "";
}
}
}

26
src/test/java/de/extio/lmlib/token/TokenizerTest.java
View File

@@ -1,6 +1,9 @@
package de.extio.lmlib.token;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.math.BigDecimal;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -41,6 +44,29 @@ public class TokenizerTest {
assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString());
assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
}
@Test
void fallback() {
final var tokenizer = new FallbackTokenizer();
final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
assertEquals(47, tokenizer.count(TEXT, profile));
final var tokens = tokenizer.tokenize(TEXT, profile);
assertEquals(47, tokens.size());
assertEquals("[465675044868, 28417, 8193, 495689692164, 8193, 7563522, 8193, 24833, 8193, 500236121092, 8449, 8193, 2003781635, 8193, 1701994755, 8193, 1970239747, 8193, 474215310340, 26369, 16129, 8193, 1701336067, 8193, 499850245892, 1919248387, 8193, 7563522, 8193, 418313824004, 29697, 8193, 418296853508, 30977, 8449, 8193, 435678434052, 29185, 8193, 474299593476, 30977, 8193, 474214855428, 30977, 8449, 8449, 8449]", tokens.toString());
assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
}
@Test
void fallbackTokenizerGroupsWhitespace() {
final var tokenizer = new FallbackTokenizer();
final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
final var text = " ";
final var tokens = tokenizer.tokenize(text, profile);
assertEquals(2, tokens.size());
assertEquals(text, tokenizer.detokenize(tokens, profile));
}
/**
* Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model.