exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite FallbackTokenizer to approximate cl100k_base token counts

Browse Source
This commit is contained in:
extio-de
2026-10-17 12:03:46 +02:00
parent 24b0efb0af
commit 46e3e57f4b
2 changed files with 83 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
src/main/java/de/extio/lmlib/token/FallbackTokenizer.java
View File

@@ -10,6 +10,16 @@ final class FallbackTokenizer implements Tokenizer {
static final String NAME = "fallback"; static final String NAME = "fallback";
private static final int WORD_CLASS = 0;
private static final int WHITESPACE_CLASS = 1;
private static final int PUNCTUATION_CLASS = 2;
private static final int NON_ASCII_CLASS = 3;
private static final int WORD_BYTES_PER_TOKEN = 7;
private static final int DIGIT_BYTES_PER_TOKEN = 3;
private static final int WHITESPACE_CHARS_PER_TOKEN = 4;
private static final int PUNCTUATION_CHARS_PER_TOKEN = 3;
@Override @Override
public String getName() { public String getName() {
return NAME; return NAME;
@@ -23,20 +33,33 @@ final class FallbackTokenizer implements Tokenizer {
final var tokens = new ArrayList<Long>(); final var tokens = new ArrayList<Long>();
var index = 0; var index = 0;
while (index < txt.length()) { while (index < txt.length()) {
final var codePoint = txt.codePointAt(index); final var cls = this.classOf(txt.codePointAt(index));
final var charCount = Character.charCount(codePoint); if (cls == WHITESPACE_CLASS) {
if (Character.isWhitespace(codePoint)) { final var whitespaceEnd = this.consumeRun(txt, index, WHITESPACE_CLASS);
final var end = this.consumeWhitespace(txt, index); final var next = whitespaceEnd < txt.length() ? this.classOf(txt.codePointAt(whitespaceEnd)) : NON_ASCII_CLASS;
tokens.addAll(this.encodeChunked(txt.substring(index, end), 6)); if (next == WORD_CLASS || next == PUNCTUATION_CLASS) {
final var wordEnd = this.consumeRun(txt, whitespaceEnd, next);
tokens.addAll(this.encodeChunked(txt.substring(index, wordEnd), this.bytesPerToken(txt.substring(whitespaceEnd, wordEnd))));
index = wordEnd;
} else {
tokens.addAll(this.encodeChunkedChars(txt.substring(index, whitespaceEnd), WHITESPACE_CHARS_PER_TOKEN));
index = whitespaceEnd;
}
continue;
}
if (cls == WORD_CLASS) {
final var end = this.consumeRun(txt, index, WORD_CLASS);
tokens.addAll(this.encodeChunked(txt.substring(index, end), this.bytesPerToken(txt.substring(index, end))));
index = end; index = end;
continue; continue;
} }
if (Character.isLetterOrDigit(codePoint)) { if (cls == PUNCTUATION_CLASS) {
final var end = this.consumeAlphaNumeric(txt, index); final var end = this.consumeRun(txt, index, PUNCTUATION_CLASS);
tokens.addAll(this.encodeWordLike(txt.substring(index, end))); tokens.addAll(this.encodeChunkedChars(txt.substring(index, end), PUNCTUATION_CHARS_PER_TOKEN));
index = end; index = end;
continue; continue;
} }
final var charCount = Character.charCount(txt.codePointAt(index));
tokens.add(this.encodeToken(txt.substring(index, index + charCount))); tokens.add(this.encodeToken(txt.substring(index, index + charCount)));
index += charCount; index += charCount;
} }
@@ -57,84 +80,51 @@ final class FallbackTokenizer implements Tokenizer {
return builder.toString(); return builder.toString();
} }
private int consumeWhitespace(final String txt, final int start) { private int classOf(final int codePoint) {
var index = start; if (codePoint >= 0x80) {
while (index < txt.length()) { return Character.isLetterOrDigit(codePoint) ? WORD_CLASS : NON_ASCII_CLASS;
final var codePoint = txt.codePointAt(index);
if (!Character.isWhitespace(codePoint)) {
break;
} }
index += Character.charCount(codePoint); if ((codePoint >= 'a' && codePoint <= 'z') || (codePoint >= 'A' && codePoint <= 'Z') || (codePoint >= '0' && codePoint <= '9')) {
return WORD_CLASS;
}
if (codePoint == ' ' || codePoint == '\t' || codePoint == '\n' || codePoint == '\r' || codePoint == '\f') {
return WHITESPACE_CLASS;
}
return PUNCTUATION_CLASS;
}
private int consumeRun(final String txt, final int start, final int cls) {
var index = start;
while (index < txt.length() && this.classOf(txt.codePointAt(index)) == cls) {
index += Character.charCount(txt.codePointAt(index));
} }
return index; return index;
} }
private int consumeAlphaNumeric(final String txt, final int start) { private int bytesPerToken(final String text) {
var index = start; return this.isAsciiDigits(text) ? DIGIT_BYTES_PER_TOKEN : WORD_BYTES_PER_TOKEN;
while (index < txt.length()) {
final var codePoint = txt.codePointAt(index);
if (!Character.isLetterOrDigit(codePoint)) {
break;
}
index += Character.charCount(codePoint);
}
return index;
}
private List<Long> encodeWordLike(final String text) {
if (text.isEmpty()) {
return List.of();
}
if (this.isAsciiDigits(text)) {
return this.encodeChunked(text, 3);
}
if (this.isAsciiLetters(text)) {
return this.encodeChunked(text, 4);
}
return this.encodeChunkedByUtf8Bytes(text, 6);
} }
private boolean isAsciiDigits(final String text) { private boolean isAsciiDigits(final String text) {
return text.chars().allMatch(ch -> ch >= '0' && ch <= '9'); return text.codePoints().allMatch(cp -> cp >= '0' && cp <= '9');
} }
private boolean isAsciiLetters(final String text) { private List<Long> encodeChunked(final String text, final int maxBytesPerToken) {
return text.chars().allMatch(ch -> ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z');
}
private List<Long> encodeChunked(final String text, final int maxCharsPerToken) {
final var tokens = new ArrayList<Long>();
for (var index = 0; index < text.length();) {
var end = index;
var charCount = 0;
while (end < text.length() && charCount < maxCharsPerToken) {
final var codePoint = text.codePointAt(end);
end += Character.charCount(codePoint);
charCount++;
}
tokens.add(this.encodeToken(text.substring(index, end)));
index = end;
}
return tokens;
}
private List<Long> encodeChunkedByUtf8Bytes(final String text, final int maxBytesPerToken) {
final var tokens = new ArrayList<Long>(); final var tokens = new ArrayList<Long>();
for (var index = 0; index < text.length();) { for (var index = 0; index < text.length();) {
var end = index; var end = index;
var bytes = 0; var bytes = 0;
while (end < text.length()) { while (end < text.length()) {
final var codePoint = text.codePointAt(end); final var codePoint = text.codePointAt(end);
final var candidateEnd = end + Character.charCount(codePoint); final var candidateBytes = this.utf8Length(codePoint);
final var candidateBytes = text.substring(end, candidateEnd).getBytes(StandardCharsets.UTF_8).length; if (bytes + candidateBytes > maxBytesPerToken) {
if (bytes > 0 && bytes + candidateBytes > maxBytesPerToken) {
break; break;
} }
end = candidateEnd; end += Character.charCount(codePoint);
bytes += candidateBytes; bytes += candidateBytes;
if (bytes >= maxBytesPerToken) {
break;
} }
if (end == index) {
end += Character.charCount(text.codePointAt(index));
} }
tokens.add(this.encodeToken(text.substring(index, end))); tokens.add(this.encodeToken(text.substring(index, end)));
index = end; index = end;
@@ -142,6 +132,29 @@ final class FallbackTokenizer implements Tokenizer {
return tokens; return tokens;
} }
private List<Long> encodeChunkedChars(final String text, final int maxCharsPerToken) {
final var tokens = new ArrayList<Long>();
for (var index = 0; index < text.length();) {
final var end = Math.min(index + maxCharsPerToken, text.length());
tokens.add(this.encodeToken(text.substring(index, end)));
index = end;
}
return tokens;
}
private int utf8Length(final int codePoint) {
if (codePoint < 0x80) {
return 1;
}
if (codePoint < 0x800) {
return 2;
}
if (codePoint < 0x10000) {
return 3;
}
return 4;
}
private long encodeToken(final String tokenText) { private long encodeToken(final String tokenText) {
final var bytes = tokenText.getBytes(StandardCharsets.UTF_8); final var bytes = tokenText.getBytes(StandardCharsets.UTF_8);
if (bytes.length == 0 || bytes.length > 7) { if (bytes.length == 0 || bytes.length > 7) {

6
src/test/java/de/extio/lmlib/token/TokenizerTest.java
View File

@@ -48,10 +48,10 @@ public class TokenizerTest {
final var tokenizer = new FallbackTokenizer(); final var tokenizer = new FallbackTokenizer();
final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile()); final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
assertEquals(47, tokenizer.count(TEXT, profile)); assertEquals(22, tokenizer.count(TEXT, profile));
final var tokens = tokenizer.tokenize(TEXT, profile); final var tokens = tokenizer.tokenize(TEXT, profile);
assertEquals(47, tokens.size()); assertEquals(22, tokens.size());
assertEquals("[465675044868, 28417, 8193, 495689692164, 8193, 7563522, 8193, 24833, 8193, 500236121092, 8449, 8193, 2003781635, 8193, 1701994755, 8193, 1970239747, 8193, 474215310340, 26369, 16129, 8193, 1701336067, 8193, 499850245892, 1919248387, 8193, 7563522, 8193, 418313824004, 29697, 8193, 418296853508, 30977, 8449, 8193, 435678434052, 29185, 8193, 474299593476, 30977, 8193, 474214855428, 30977, 8449, 8449, 8449]", tokens.toString()); assertEquals("[122511465728005, 126896561201157, 1936269315, 6365186, 128060447006725, 8449, 512968105988, 435710664708, 504381382660, 29113321720651782, 16129, 435542040580, 7307218357071585287, 29185, 1936269315, 32758185637388294, 34165556176494598, 8449, 32199681024139270, 34179892877926406, 34179871184986118, 555819267]", tokens.toString());
assertEquals(TEXT, tokenizer.detokenize(tokens, profile)); assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
} }