From 46e3e57f4b8e2676ebff79eb3d054edb82ffe275 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 17 Oct 2026 12:03:46 +0200 Subject: [PATCH] Rewrite FallbackTokenizer to approximate cl100k_base token counts --- .../extio/lmlib/token/FallbackTokenizer.java | 147 ++++++++++-------- .../de/extio/lmlib/token/TokenizerTest.java | 6 +- 2 files changed, 83 insertions(+), 70 deletions(-) diff --git a/src/main/java/de/extio/lmlib/token/FallbackTokenizer.java b/src/main/java/de/extio/lmlib/token/FallbackTokenizer.java index 8d2df29..ff7b356 100644 --- a/src/main/java/de/extio/lmlib/token/FallbackTokenizer.java +++ b/src/main/java/de/extio/lmlib/token/FallbackTokenizer.java @@ -10,6 +10,16 @@ final class FallbackTokenizer implements Tokenizer { static final String NAME = "fallback"; + private static final int WORD_CLASS = 0; + private static final int WHITESPACE_CLASS = 1; + private static final int PUNCTUATION_CLASS = 2; + private static final int NON_ASCII_CLASS = 3; + + private static final int WORD_BYTES_PER_TOKEN = 7; + private static final int DIGIT_BYTES_PER_TOKEN = 3; + private static final int WHITESPACE_CHARS_PER_TOKEN = 4; + private static final int PUNCTUATION_CHARS_PER_TOKEN = 3; + @Override public String getName() { return NAME; @@ -23,20 +33,33 @@ final class FallbackTokenizer implements Tokenizer { final var tokens = new ArrayList(); var index = 0; while (index < txt.length()) { - final var codePoint = txt.codePointAt(index); - final var charCount = Character.charCount(codePoint); - if (Character.isWhitespace(codePoint)) { - final var end = this.consumeWhitespace(txt, index); - tokens.addAll(this.encodeChunked(txt.substring(index, end), 6)); + final var cls = this.classOf(txt.codePointAt(index)); + if (cls == WHITESPACE_CLASS) { + final var whitespaceEnd = this.consumeRun(txt, index, WHITESPACE_CLASS); + final var next = whitespaceEnd < txt.length() ? this.classOf(txt.codePointAt(whitespaceEnd)) : NON_ASCII_CLASS; + if (next == WORD_CLASS || next == PUNCTUATION_CLASS) { + final var wordEnd = this.consumeRun(txt, whitespaceEnd, next); + tokens.addAll(this.encodeChunked(txt.substring(index, wordEnd), this.bytesPerToken(txt.substring(whitespaceEnd, wordEnd)))); + index = wordEnd; + } else { + tokens.addAll(this.encodeChunkedChars(txt.substring(index, whitespaceEnd), WHITESPACE_CHARS_PER_TOKEN)); + index = whitespaceEnd; + } + continue; + } + if (cls == WORD_CLASS) { + final var end = this.consumeRun(txt, index, WORD_CLASS); + tokens.addAll(this.encodeChunked(txt.substring(index, end), this.bytesPerToken(txt.substring(index, end)))); index = end; continue; } - if (Character.isLetterOrDigit(codePoint)) { - final var end = this.consumeAlphaNumeric(txt, index); - tokens.addAll(this.encodeWordLike(txt.substring(index, end))); + if (cls == PUNCTUATION_CLASS) { + final var end = this.consumeRun(txt, index, PUNCTUATION_CLASS); + tokens.addAll(this.encodeChunkedChars(txt.substring(index, end), PUNCTUATION_CHARS_PER_TOKEN)); index = end; continue; } + final var charCount = Character.charCount(txt.codePointAt(index)); tokens.add(this.encodeToken(txt.substring(index, index + charCount))); index += charCount; } @@ -57,84 +80,51 @@ final class FallbackTokenizer implements Tokenizer { return builder.toString(); } - private int consumeWhitespace(final String txt, final int start) { + private int classOf(final int codePoint) { + if (codePoint >= 0x80) { + return Character.isLetterOrDigit(codePoint) ? WORD_CLASS : NON_ASCII_CLASS; + } + if ((codePoint >= 'a' && codePoint <= 'z') || (codePoint >= 'A' && codePoint <= 'Z') || (codePoint >= '0' && codePoint <= '9')) { + return WORD_CLASS; + } + if (codePoint == ' ' || codePoint == '\t' || codePoint == '\n' || codePoint == '\r' || codePoint == '\f') { + return WHITESPACE_CLASS; + } + return PUNCTUATION_CLASS; + } + + private int consumeRun(final String txt, final int start, final int cls) { var index = start; - while (index < txt.length()) { - final var codePoint = txt.codePointAt(index); - if (!Character.isWhitespace(codePoint)) { - break; - } - index += Character.charCount(codePoint); + while (index < txt.length() && this.classOf(txt.codePointAt(index)) == cls) { + index += Character.charCount(txt.codePointAt(index)); } return index; } - private int consumeAlphaNumeric(final String txt, final int start) { - var index = start; - while (index < txt.length()) { - final var codePoint = txt.codePointAt(index); - if (!Character.isLetterOrDigit(codePoint)) { - break; - } - index += Character.charCount(codePoint); - } - return index; - } - - private List encodeWordLike(final String text) { - if (text.isEmpty()) { - return List.of(); - } - if (this.isAsciiDigits(text)) { - return this.encodeChunked(text, 3); - } - if (this.isAsciiLetters(text)) { - return this.encodeChunked(text, 4); - } - return this.encodeChunkedByUtf8Bytes(text, 6); + private int bytesPerToken(final String text) { + return this.isAsciiDigits(text) ? DIGIT_BYTES_PER_TOKEN : WORD_BYTES_PER_TOKEN; } private boolean isAsciiDigits(final String text) { - return text.chars().allMatch(ch -> ch >= '0' && ch <= '9'); + return text.codePoints().allMatch(cp -> cp >= '0' && cp <= '9'); } - private boolean isAsciiLetters(final String text) { - return text.chars().allMatch(ch -> ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z'); - } - - private List encodeChunked(final String text, final int maxCharsPerToken) { - final var tokens = new ArrayList(); - for (var index = 0; index < text.length();) { - var end = index; - var charCount = 0; - while (end < text.length() && charCount < maxCharsPerToken) { - final var codePoint = text.codePointAt(end); - end += Character.charCount(codePoint); - charCount++; - } - tokens.add(this.encodeToken(text.substring(index, end))); - index = end; - } - return tokens; - } - - private List encodeChunkedByUtf8Bytes(final String text, final int maxBytesPerToken) { + private List encodeChunked(final String text, final int maxBytesPerToken) { final var tokens = new ArrayList(); for (var index = 0; index < text.length();) { var end = index; var bytes = 0; while (end < text.length()) { final var codePoint = text.codePointAt(end); - final var candidateEnd = end + Character.charCount(codePoint); - final var candidateBytes = text.substring(end, candidateEnd).getBytes(StandardCharsets.UTF_8).length; - if (bytes > 0 && bytes + candidateBytes > maxBytesPerToken) { + final var candidateBytes = this.utf8Length(codePoint); + if (bytes + candidateBytes > maxBytesPerToken) { break; } - end = candidateEnd; + end += Character.charCount(codePoint); bytes += candidateBytes; - if (bytes >= maxBytesPerToken) { - break; - } + } + if (end == index) { + end += Character.charCount(text.codePointAt(index)); } tokens.add(this.encodeToken(text.substring(index, end))); index = end; @@ -142,6 +132,29 @@ final class FallbackTokenizer implements Tokenizer { return tokens; } + private List encodeChunkedChars(final String text, final int maxCharsPerToken) { + final var tokens = new ArrayList(); + for (var index = 0; index < text.length();) { + final var end = Math.min(index + maxCharsPerToken, text.length()); + tokens.add(this.encodeToken(text.substring(index, end))); + index = end; + } + return tokens; + } + + private int utf8Length(final int codePoint) { + if (codePoint < 0x80) { + return 1; + } + if (codePoint < 0x800) { + return 2; + } + if (codePoint < 0x10000) { + return 3; + } + return 4; + } + private long encodeToken(final String tokenText) { final var bytes = tokenText.getBytes(StandardCharsets.UTF_8); if (bytes.length == 0 || bytes.length > 7) { diff --git a/src/test/java/de/extio/lmlib/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/token/TokenizerTest.java index 65ea793..d56efca 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerTest.java @@ -48,10 +48,10 @@ public class TokenizerTest { final var tokenizer = new FallbackTokenizer(); final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile()); - assertEquals(47, tokenizer.count(TEXT, profile)); + assertEquals(22, tokenizer.count(TEXT, profile)); final var tokens = tokenizer.tokenize(TEXT, profile); - assertEquals(47, tokens.size()); - assertEquals("[465675044868, 28417, 8193, 495689692164, 8193, 7563522, 8193, 24833, 8193, 500236121092, 8449, 8193, 2003781635, 8193, 1701994755, 8193, 1970239747, 8193, 474215310340, 26369, 16129, 8193, 1701336067, 8193, 499850245892, 1919248387, 8193, 7563522, 8193, 418313824004, 29697, 8193, 418296853508, 30977, 8449, 8193, 435678434052, 29185, 8193, 474299593476, 30977, 8193, 474214855428, 30977, 8449, 8449, 8449]", tokens.toString()); + assertEquals(22, tokens.size()); + assertEquals("[122511465728005, 126896561201157, 1936269315, 6365186, 128060447006725, 8449, 512968105988, 435710664708, 504381382660, 29113321720651782, 16129, 435542040580, 7307218357071585287, 29185, 1936269315, 32758185637388294, 34165556176494598, 8449, 32199681024139270, 34179892877926406, 34179871184986118, 555819267]", tokens.toString()); assertEquals(TEXT, tokenizer.detokenize(tokens, profile)); }