exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lm-lib refactoring;

Browse Source 
git-svn-id: svn://home0/lm-lib@3610 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2024-12-18 18:58:27 +00:00
parent 35b59043e1
commit f01309526a
28 changed files with 63 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmlib/client/Client.java
View File

@@ -1,7 +1,7 @@
package de.extio.lmlib.client;
import de.extio.lmlib.client.profile.ModelCategory;
import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
public interface Client {

4
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -5,8 +5,8 @@ import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import de.extio.lmlib.client.profile.ModelCategory;
import de.extio.lmlib.client.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfileService;
@Service
public class ClientService {

10
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -32,11 +32,11 @@ import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.profile.ModelCategory;
import de.extio.lmlib.client.profile.ModelProfile;
import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.client.profile.ModelProfileService;
import de.extio.lmlib.client.token.Tokenizer;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.token.Tokenizer;
public final class AzureAiClient implements Client {

4
src/main/java/de/extio/lmlib/client/azureai/AzureAiClientAutoConfiguration.java
View File

@@ -4,8 +4,8 @@ import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.context.annotation.Bean;
import de.extio.lmlib.client.profile.ModelProfileService;
import de.extio.lmlib.client.token.Tokenizer;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
@AutoConfiguration
@ConditionalOnClass(name = "com.azure.ai.inference.ChatCompletionsClient")

27
src/main/java/de/extio/lmlib/client/profile/ModelCategory.java
View File

@@ -1,27 +0,0 @@
package de.extio.lmlib.client.profile;
public enum ModelCategory {
SMALL("profile.model.s", "S"),
MEDIUM("profile.model.m", "M"),
LARGE("profile.model.l", "L"),
HOT("profile.model.hot", "HOT"),
COLD("profile.model.cold", "COLD");
private final String modelProfile;
private final String shortName;
ModelCategory(final String modelProfile, final String shortName) {
this.modelProfile = modelProfile;
this.shortName = shortName;
}
public String getModelProfile() {
return this.modelProfile;
}
public String getShortName() {
return this.shortName;
}
}

19
src/main/java/de/extio/lmlib/client/profile/ModelProfile.java
View File

@@ -1,19 +0,0 @@
package de.extio.lmlib.client.profile;
public record ModelProfile(
String prompt,
String tokenEncoding,
int maxTokens,
int maxContextLength,
double temperature,
double topP,
int maxContinuations,
ModelProvider modelProvider,
String modelName,
String url,
String apiKey) {
public static enum ModelProvider {
AZURE_AI, OAI_TEXT_COMPLETION;
}
}

113
src/main/java/de/extio/lmlib/client/profile/ModelProfileService.java
View File

@@ -1,113 +0,0 @@
package de.extio.lmlib.client.profile;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.bind.Bindable;
import org.springframework.boot.context.properties.bind.Binder;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.core.env.Environment;
import org.springframework.core.io.support.ResourcePropertySource;
import org.springframework.stereotype.Service;
import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
@Service
public class ModelProfileService {
private final Environment environment;
private static final String MODEL_PROFILE_PREFIX = "profile";
private static final Logger LOGGER = LoggerFactory.getLogger(ModelProfileService.class);
public ModelProfileService(final Environment environment) {
this.environment = environment;
}
@Cacheable("modelProfiles")
public ModelProfile getModelProfile(final String modelProfile) {
final var modelName = this.environment.getProperty(modelProfile);
if (modelName == null) {
throw new IllegalStateException("No model found for profile: " + modelProfile);
}
try {
final var resource = new ResourcePropertySource("classpath:" + modelName + ".properties");
final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model name found for model: " + modelName));
final var tokenEncoding = Optional.ofNullable(resource.getProperty("tokenEncoding"))
.map(Object::toString)
.orElse("none");
final var maxTokens = Optional.ofNullable(resource.getProperty("maxTokens"))
.map(Object::toString)
.map(Integer::parseInt)
.orElseThrow(() -> new IllegalStateException("No model maxTokens found for model: " + modelName));
final var maxContextLength = Optional.ofNullable(resource.getProperty("maxContextLength"))
.map(Object::toString)
.map(Integer::parseInt)
.orElseThrow(() -> new IllegalStateException("No model maxContextLength found for model: " + modelName));
final var temperature = Optional.ofNullable(resource.getProperty("temperature"))
.map(Object::toString)
.map(Double::parseDouble)
.orElseThrow(() -> new IllegalStateException("No model temperature found for model: " + modelName));
final var topP = Optional.ofNullable(resource.getProperty("topP"))
.map(Object::toString)
.map(Double::parseDouble)
.orElseThrow(() -> new IllegalStateException("No model topP found for model: " + modelName));
final var maxContinuations = Optional.ofNullable(resource.getProperty("maxContinuations"))
.map(Object::toString)
.map(Integer::parseInt)
.orElseThrow(() -> new IllegalStateException("No model maxContinuations found for model: " + modelName));
final var modelProvider = Optional.ofNullable(resource.getProperty("modelProvider"))
.map(Object::toString)
.map(ModelProvider::valueOf)
.orElseThrow(() -> new IllegalStateException("No model modelProvider found for model: " + modelName));
final var modelNameCfg = Optional.ofNullable(resource.getProperty("modelName"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model modelName found for model: " + modelName));
final var url = Optional.ofNullable(resource.getProperty("url"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model url found for model: " + modelName));
final var apiKey = Optional.ofNullable(resource.getProperty("apiKey"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, maxContinuations, modelProvider, modelNameCfg, url, apiKey);
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile", e);
return null;
}
}
public List<String> getModelProfileUrls(final ModelProvider modelProvider) {
final var modelProfiles = Binder.get(this.environment)
.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
.orElse(Collections.emptyMap());
return modelProfiles.values().stream()
.filter(model -> model != null && !model.isEmpty())
.map(model -> {
try {
final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
}
return null;
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile for: " + model, e);
return null;
}
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
}

43
src/main/java/de/extio/lmlib/client/prompt/AlpacaShortPromptStrategy.java
View File

@@ -1,43 +0,0 @@
package de.extio.lmlib.client.prompt;
public class AlpacaShortPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("### Instruction:\n");
prompt.append(instruction);
prompt.append("\n\n### Input:\n");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append("\n\n### Response:\n");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);
prompt.append("\n\n### Input:\n");
prompt.append(user);
prompt.append("\n\n### Response:\n");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("</s>", "");
}
@Override
public String getPromptName() {
return "alpaca";
}
}

49
src/main/java/de/extio/lmlib/client/prompt/ChatMLPromptStrategy.java
View File

@@ -1,49 +0,0 @@
package de.extio.lmlib.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class ChatMLPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String system, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<|im_start|>");
if (!system.isEmpty()) {
prompt.append("system\n");
prompt.append(system);
prompt.append("<|im_end|>\n");
}
prompt.append("<|im_start|>user\n");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append("<|im_end|>\n<|im_start|>assistant\n");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);
prompt.append("<|im_end|>\n<|im_start|>user\n");
prompt.append(user);
prompt.append("<|im_end|>\n<|im_start|>assistant\n");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<|im_end|>", "");
}
@Override
public String getPromptName() {
return "chatml";
}
}

48
src/main/java/de/extio/lmlib/client/prompt/Gemma2PromptStrategy.java
View File

@@ -1,48 +0,0 @@
package de.extio.lmlib.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class Gemma2PromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String system, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<start_of_turn>user\n");
if (!system.isEmpty()) {
prompt.append(system);
prompt.append("\n");
}
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append("<end_of_turn>\n<start_of_turn>model\n");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);
prompt.append("<end_of_turn>\n<start_of_turn>user\n");
prompt.append(user);
prompt.append("<end_of_turn>\n<start_of_turn>model\n");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<end_of_turn>", "");
}
@Override
public String getPromptName() {
return "gemma2";
}
}

51
src/main/java/de/extio/lmlib/client/prompt/Llama3PromptStrategy.java
View File

@@ -1,51 +0,0 @@
package de.extio.lmlib.client.prompt;
import org.springframework.stereotype.Component;
/**
* @see https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
*/
@Component
public class Llama3PromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String system, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<|begin_of_text|>");
if (!system.isEmpty()) {
prompt.append("<|start_header_id|>system<|end_header_id|>\n\n");
prompt.append(system);
prompt.append("<|eot_id|>");
}
prompt.append("<|start_header_id|>user<|end_header_id|>\n\n");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n\n");
}
prompt.append(text);
prompt.append("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);
prompt.append("<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n" + user + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<|eot_id|>", "");
}
@Override
public String getPromptName() {
return "llama3";
}
}

49
src/main/java/de/extio/lmlib/client/prompt/MixtralPromptStrategy.java
View File

@@ -1,49 +0,0 @@
package de.extio.lmlib.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class MixtralPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String instruction, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<s>[INST] ");
prompt.append(instruction);
if (!instruction.isEmpty() && !question.isEmpty()) {
prompt.append("\n");
}
prompt.append(question);
if ((!instruction.isEmpty() || !question.isEmpty()) && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append(" [/INST]");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append("\n\"");
prompt.append(assistant);
prompt.append("\"</s> [INST] ");
prompt.append(user);
prompt.append(" [/INST]");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("</s>", "");
}
@Override
public String getPromptName() {
return "mixtral";
}
}

48
src/main/java/de/extio/lmlib/client/prompt/Phi3PromptStrategy.java
View File

@@ -1,48 +0,0 @@
package de.extio.lmlib.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class Phi3PromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String system, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<|user|>\n");
prompt.append(system);
if (!system.isEmpty()) {
prompt.append("\n");
}
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append(" <|end|>\n<|assistant|>");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append("\n");
prompt.append(assistant);
prompt.append("<|user|>\n");
prompt.append(user);
prompt.append("<|end|>\n<|assistant|>");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("<|end|>", "");
}
@Override
public String getPromptName() {
return "phi3";
}
}

17
src/main/java/de/extio/lmlib/client/prompt/PromptStrategy.java
View File

@@ -1,17 +0,0 @@
package de.extio.lmlib.client.prompt;
public interface PromptStrategy {
String getPromptName();
StringBuilder start(String system, String question, String text);
void continue_(StringBuilder prompt, String assistant);
void next(StringBuilder prompt, String assistant, String user);
default String removeEOT(final String prompt) {
return prompt.strip();
}
}

22
src/main/java/de/extio/lmlib/client/prompt/PromptStrategyFactory.java
View File

@@ -1,22 +0,0 @@
package de.extio.lmlib.client.prompt;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.stereotype.Component;
@Component
public class PromptStrategyFactory {
private final Map<String, PromptStrategy> strategies = new HashMap<>();
public PromptStrategyFactory(final List<PromptStrategy> strategies) {
strategies.forEach(strategy -> this.strategies.put(strategy.getPromptName(), strategy));
}
public PromptStrategy getStrategy(final String modelPrompt) {
return this.strategies.get(modelPrompt);
}
}

46
src/main/java/de/extio/lmlib/client/prompt/VicunaPromptStrategy.java
View File

@@ -1,46 +0,0 @@
package de.extio.lmlib.client.prompt;
import org.springframework.stereotype.Component;
@Component
public class VicunaPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String system, final String question, final String text) {
final StringBuilder prompt = new StringBuilder();
prompt.append(system);
prompt.append(" USER: ");
prompt.append(question);
if (!question.isEmpty() && !text.isEmpty()) {
prompt.append("\n");
}
prompt.append(text);
prompt.append(" ASSISTANT:");
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(" ");
prompt.append(assistant);
prompt.append("</s>USER: ");
prompt.append(user);
prompt.append(" ASSISTANT:");
}
@Override
public String removeEOT(final String prompt) {
return prompt.strip().replace("</s>", "");
}
@Override
public String getPromptName() {
return "vicuna";
}
}

4
src/main/java/de/extio/lmlib/client/textcompletion/ModelNameSupplier.java
View File

@@ -11,8 +11,8 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.client.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
@Component
class ModelNameSupplier implements InitializingBean {

14
src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java
View File

@@ -23,16 +23,16 @@ import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.profile.ModelCategory;
import de.extio.lmlib.client.profile.ModelProfile;
import de.extio.lmlib.client.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.client.profile.ModelProfileService;
import de.extio.lmlib.client.prompt.PromptStrategy;
import de.extio.lmlib.client.prompt.PromptStrategyFactory;
import de.extio.lmlib.client.textcompletion.dto.Choice;
import de.extio.lmlib.client.textcompletion.dto.CompletionRequest;
import de.extio.lmlib.client.textcompletion.dto.CompletionResponse;
import de.extio.lmlib.client.token.Tokenizer;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import de.extio.lmlib.token.Tokenizer;
import reactor.util.retry.Retry;

53
src/main/java/de/extio/lmlib/client/token/JTokkitTokenizer.java
View File

@@ -1,53 +0,0 @@
package de.extio.lmlib.client.token;
import java.util.List;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.knuddels.jtokkit.Encodings;
import com.knuddels.jtokkit.api.Encoding;
import com.knuddels.jtokkit.api.EncodingRegistry;
import com.knuddels.jtokkit.api.EncodingType;
import com.knuddels.jtokkit.api.IntArrayList;
import de.extio.lmlib.client.profile.ModelProfile;
final class JTokkitTokenizer implements Tokenizer {
private static final EncodingRegistry registry;
static {
registry = Encodings.newDefaultEncodingRegistry();
}
private final LoadingCache<String, Encoding> encodings = Caffeine
.newBuilder()
.build(k -> {
return registry.getEncoding(EncodingType.fromName(k).orElseThrow(() -> new IllegalArgumentException("Token encoding " + k)));
});
@Override
public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
return this.encodings
.get(modelProfile.tokenEncoding())
.encodeOrdinary(txt)
.boxed()
.stream()
.map(Long::valueOf)
.toList();
}
@Override
public int count(final String txt, final ModelProfile modelProfile) {
return this.encodings.get(modelProfile.tokenEncoding()).countTokensOrdinary(txt);
}
@Override
public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
final var ial = new IntArrayList(tokens.size());
tokens.forEach(t -> ial.add(t.intValue()));
return this.encodings
.get(modelProfile.tokenEncoding())
.decode(ial);
}
}

75
src/main/java/de/extio/lmlib/client/token/LlamaServerTokenizer.java
View File

@@ -1,75 +0,0 @@
package de.extio.lmlib.client.token;
import java.time.Duration;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClient.Builder;
import de.extio.lmlib.client.profile.ModelProfile;
import reactor.util.retry.Retry;
final class LlamaServerTokenizer implements Tokenizer {
private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class);
private final WebClient.Builder webClientBuilder;
public LlamaServerTokenizer(final Builder webClientBuilder) {
this.webClientBuilder = webClientBuilder;
}
@Override
public List<Long> tokenize(final String txt, final ModelProfile modelProfile) {
LOGGER.debug("Requesting tokenizing");
final var request = new TokenizeRequest(txt);
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/tokenize").build())
.bodyValue(request)
.retrieve()
.bodyToMono(TokenizeResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
return response.tokens();
}
@Override
public int count(final String txt, final ModelProfile modelProfile) {
return this.tokenize(txt, modelProfile).size();
}
@Override
public String detokenize(final List<Long> tokens, final ModelProfile modelProfile) {
LOGGER.debug("Requesting detokenizing");
final var request = new DetokenizeRequest(tokens);
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/detokenize").build())
.bodyValue(request)
.retrieve()
.bodyToMono(DetokenizeResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
return response.content();
}
private record TokenizeRequest(String content) {
}
private record TokenizeResponse(List<Long> tokens) {
}
private record DetokenizeRequest(List<Long> tokens) {
}
private record DetokenizeResponse(String content) {
}
}

15
src/main/java/de/extio/lmlib/client/token/Tokenizer.java
View File

@@ -1,15 +0,0 @@
package de.extio.lmlib.client.token;
import java.util.List;
import de.extio.lmlib.client.profile.ModelProfile;
public interface Tokenizer {
List<Long> tokenize(String txt, ModelProfile modelProfile);
int count(String txt, ModelProfile modelProfile);
String detokenize(List<Long> tokens, ModelProfile modelProfile);
}

35
src/main/java/de/extio/lmlib/client/token/TokenizerAutoConfiguration.java
View File

@@ -1,35 +0,0 @@
package de.extio.lmlib.client.token;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.reactive.function.client.WebClient;
@AutoConfiguration
public class TokenizerAutoConfiguration {
@Bean
@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer")
@ConditionalOnMissingBean
Tokenizer llamaServerTokenizer(@Qualifier("lmLibWebClientBuilder") final WebClient.Builder webClientBuilder) {
return new LlamaServerTokenizer(webClientBuilder);
}
@Configuration(proxyBeanMethods = false)
@ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings")
@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "jTokkit")
@ConditionalOnMissingBean
public static class SomeServiceConfiguration {
@Bean
Tokenizer jTokkitTokenizer() {
return new JTokkitTokenizer();
}
}
}