exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite FallbackTokenizer to approximate cl100k_base token counts

Browse Source
This commit is contained in:
extio-de
2026-10-17 12:03:46 +02:00
parent 24b0efb0af
commit 46e3e57f4b
2 changed files with 83 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/test/java/de/extio/lmlib/token/TokenizerTest.java
View File

@@ -48,10 +48,10 @@ public class TokenizerTest {
final var tokenizer = new FallbackTokenizer();
final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
assertEquals(47, tokenizer.count(TEXT, profile));
assertEquals(22, tokenizer.count(TEXT, profile));
final var tokens = tokenizer.tokenize(TEXT, profile);
assertEquals(47, tokens.size());
assertEquals("[465675044868, 28417, 8193, 495689692164, 8193, 7563522, 8193, 24833, 8193, 500236121092, 8449, 8193, 2003781635, 8193, 1701994755, 8193, 1970239747, 8193, 474215310340, 26369, 16129, 8193, 1701336067, 8193, 499850245892, 1919248387, 8193, 7563522, 8193, 418313824004, 29697, 8193, 418296853508, 30977, 8449, 8193, 435678434052, 29185, 8193, 474299593476, 30977, 8193, 474214855428, 30977, 8449, 8449, 8449]", tokens.toString());
assertEquals(22, tokens.size());
assertEquals("[122511465728005, 126896561201157, 1936269315, 6365186, 128060447006725, 8449, 512968105988, 435710664708, 504381382660, 29113321720651782, 16129, 435542040580, 7307218357071585287, 29185, 1936269315, 32758185637388294, 34165556176494598, 8449, 32199681024139270, 34179892877926406, 34179871184986118, 555819267]", tokens.toString());
assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
}