exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: enhance error handling in tokenization and detokenization processes with clearer messages

Browse Source
This commit is contained in:
extio-de
2026-07-05 13:48:42 +02:00
parent e18880354e
commit e811c9b2fe
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java
View File

@@ -1,6 +1,5 @@
package de.extio.lmlib.token;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -41,11 +40,14 @@ final class LlamaServerTokenizer implements Tokenizer {
.onStatus(status -> {
if (status.getStatusCode().isError()) {
LOGGER.error("Tokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8));
throw new RuntimeException("Tokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText());
throw new RuntimeException("Tokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText() + ". Your LLM provider may not support the /tokenize endpoint");
}
return false;
})
.body(TokenizeResponse.class);
if (response == null || response.tokens() == null) {
throw new RuntimeException("Tokenization request failed: response is null. Your LLM provider may not support the /tokenize endpoint");
}
return response.tokens();
}
@@ -67,11 +69,14 @@ final class LlamaServerTokenizer implements Tokenizer {
.onStatus(status -> {
if (status.getStatusCode().isError()) {
LOGGER.error("Detokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8));
throw new RuntimeException("Detokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText());
throw new RuntimeException("Detokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText() + ". Your LLM provider may not support the /detokenize endpoint");
}
return false;
})
.body(DetokenizeResponse.class);
if (response == null || response.content() == null) {
throw new RuntimeException("Detokenization request failed: response is null. Your LLM provider may not support the /detokenize endpoint");
}
return response.content();
}