exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: enhance error handling in tokenization and detokenization processes with clearer messages

Browse Source
This commit is contained in:
extio-de
2026-07-05 13:48:42 +02:00
parent e18880354e
commit e811c9b2fe
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java
View File

@@ -1,6 +1,5 @@
package de.extio.lmlib.token; package de.extio.lmlib.token;
import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -41,11 +40,14 @@ final class LlamaServerTokenizer implements Tokenizer {
.onStatus(status -> { .onStatus(status -> {
if (status.getStatusCode().isError()) { if (status.getStatusCode().isError()) {
LOGGER.error("Tokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8)); LOGGER.error("Tokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8));
throw new RuntimeException("Tokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText()); throw new RuntimeException("Tokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText() + ". Your LLM provider may not support the /tokenize endpoint");
} }
return false; return false;
}) })
.body(TokenizeResponse.class); .body(TokenizeResponse.class);
if (response == null || response.tokens() == null) {
throw new RuntimeException("Tokenization request failed: response is null. Your LLM provider may not support the /tokenize endpoint");
}
return response.tokens(); return response.tokens();
} }
@@ -67,11 +69,14 @@ final class LlamaServerTokenizer implements Tokenizer {
.onStatus(status -> { .onStatus(status -> {
if (status.getStatusCode().isError()) { if (status.getStatusCode().isError()) {
LOGGER.error("Detokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8)); LOGGER.error("Detokenization request failed with status code {} {}: {}", status.getStatusCode(), status.getStatusText(), new String(status.getBody().readAllBytes(), StandardCharsets.UTF_8));
throw new RuntimeException("Detokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText()); throw new RuntimeException("Detokenization request failed with status code " + status.getStatusCode() + " " + status.getStatusText() + ". Your LLM provider may not support the /detokenize endpoint");
} }
return false; return false;
}) })
.body(DetokenizeResponse.class); .body(DetokenizeResponse.class);
if (response == null || response.content() == null) {
throw new RuntimeException("Detokenization request failed: response is null. Your LLM provider may not support the /detokenize endpoint");
}
return response.content(); return response.content();
} }