exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored llm clients, removed maxContinuations feature, exposed finishReason instead;

Browse Source
This commit is contained in:
extio-de
2025-06-27 20:43:04 +02:00
parent f3706a408a
commit 47ef9ba436
26 changed files with 178 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java
View File

@@ -62,17 +62,17 @@ public final class AgentRequestStatistic {
}
public void add(final Completion completion) {
if (completion.cached()) {
if (completion.statistics().cached()) {
this.cachedPrompts.incrementAndGet();
}
else {
this.getRequests().addAndGet(completion.requests());
this.getRequests().addAndGet(completion.statistics().requests());
}
this.getInTokens().addAndGet(completion.inTokens());
this.getOutTokens().addAndGet(completion.outTokens());
this.getInTokens().addAndGet(completion.statistics().inTokens());
this.getOutTokens().addAndGet(completion.statistics().outTokens());
synchronized (this) {
this.requestDuration = this.requestDuration.plus(completion.duration());
this.cost = this.cost.add(completion.cost());
this.requestDuration = this.requestDuration.plus(completion.statistics().duration());
this.cost = this.cost.add(completion.statistics().cost());
}
}

5
src/main/java/de/extio/lmlib/client/Completion.java
View File

@@ -1,8 +1,5 @@
package de.extio.lmlib.client;
import java.math.BigDecimal;
import java.time.Duration;
public final record Completion(String response, CompletionFinishReason finishReason, CompletionStatistics statistics) {
public final record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, BigDecimal cost, boolean cached) {
}

20
src/main/java/de/extio/lmlib/client/CompletionFinishReason.java Normal file
View File

@@ -0,0 +1,20 @@
package de.extio.lmlib.client;
public enum CompletionFinishReason {
/**
* Regular completion
*/
DONE,
/**
* Completion was stopped because the token limit was reached. You may continue the conversation.
*/
TOKEN_LIMIT_REACHED,
/**
* Completion was stopped because the content was filtered.
*/
CONTENT_FILTERED
}

8
src/main/java/de/extio/lmlib/client/CompletionStatistics.java Normal file
View File

@@ -0,0 +1,8 @@
package de.extio.lmlib.client;
import java.math.BigDecimal;
import java.time.Duration;
public record CompletionStatistics(int requests, Duration duration, long inTokens, long outTokens, BigDecimal cost, boolean cached) {
}

146
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -13,14 +13,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.azure.ai.inference.ChatCompletionsClient;
import com.azure.ai.inference.ChatCompletionsClientBuilder;
import com.azure.ai.inference.models.ChatCompletions;
import com.azure.ai.inference.models.ChatCompletionsOptions;
import com.azure.ai.inference.models.ChatRequestAssistantMessage;
import com.azure.ai.inference.models.ChatRequestMessage;
@@ -35,6 +33,8 @@ import com.azure.core.http.policy.RetryOptions;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.CompletionStatistics;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.profile.ModelCategory;
@@ -42,7 +42,6 @@ import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
import okhttp3.Authenticator;
import okhttp3.Challenge;
import okhttp3.Credentials;
@@ -56,6 +55,7 @@ public final class AzureAiClient implements Client {
private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClient.class);
private static final int TIMEOUT = 300000;
private static final int CONNECT_TIMEOUT = 10000;
private final ConcurrentMap<ModelCategory, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
@@ -104,32 +104,12 @@ public final class AzureAiClient implements Client {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
final List<List<ChatRequestMessage>> chats = this.createChats(conversation, modelProfile);
final var chat = this.createChats(conversation, modelProfile);
final List<StringBuilder> answers = new ArrayList<>();
final CompletionStatistics statistics = new CompletionStatistics();
for (final List<ChatRequestMessage> chat : chats) {
final StringBuilder answer = new StringBuilder();
this.requestCompletionsContinuations(chat, answer, statistics, modelCategory, modelProfile);
answers.add(answer);
}
// TODO: Reactivate when multiple (split) prompts have been implemented again in #addChatRequestMessage
// if (answers.size() > 1) {
// this.summarize(answers, modelProfile, promptStrategy, statistics);
// }
return new Completion(
answers.stream().collect(Collectors.joining()),
statistics.requests,
statistics.duration,
statistics.inTokens,
statistics.outTokens,
new BigDecimal(statistics.inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(statistics.outTokens).multiply(modelProfile.costOutToken())),
false);
return this.requestCompletionsContinuations(chat, modelCategory, modelProfile);
}
private List<List<ChatRequestMessage>> createChats(final Conversation conversation, final ModelProfile modelProfile) {
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
final List<ChatRequestMessage> chat = new ArrayList<>();
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
@@ -138,7 +118,6 @@ public final class AzureAiClient implements Client {
Turn assistantTurn = null;
boolean first = true;
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
final AtomicInteger tokens = new AtomicInteger();
while (!turns.isEmpty()) {
@@ -161,17 +140,17 @@ public final class AzureAiClient implements Client {
if (systemTurn != null) {
chat.add(new ChatRequestSystemMessage(systemTurn.text()));
}
if (this.addChatRequestMessage(userTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
if (this.addChatRequestMessage(userTurn.text(), tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
break;
}
userTurn = null;
first = false;
}
else if (!first && assistantTurn != null && userTurn != null) {
if (this.addChatRequestMessage(assistantTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestAssistantMessage::new)) {
if (this.addChatRequestMessage(assistantTurn.text(), tokens, chat, modelProfile, ChatRequestAssistantMessage::new)) {
break;
}
if (this.addChatRequestMessage(userTurn.text(), maxLength, tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
if (this.addChatRequestMessage(userTurn.text(), tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
break;
}
userTurn = null;
@@ -179,17 +158,17 @@ public final class AzureAiClient implements Client {
}
}
return List.of(chat);
return chat;
}
private boolean addChatRequestMessage(final String text, final int maxLength, final AtomicInteger tokens, final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Function<String, ChatRequestMessage> messageFactory) {
String prompt = text;
boolean trimmed = false;
private boolean addChatRequestMessage(final String text, final AtomicInteger tokens, final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Function<String, ChatRequestMessage> messageFactory) {
var prompt = text;
var trimmed = false;
final int textTokens = this.tokenizer.count(prompt, modelProfile);
final var textTokens = this.tokenizer.count(prompt, modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokens.addAndGet(textTokens) > maxLength) {
// TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
List<Long> tokenized = this.tokenizer.tokenize(prompt, modelProfile);
var tokenized = this.tokenizer.tokenize(prompt, modelProfile);
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
@@ -201,42 +180,40 @@ public final class AzureAiClient implements Client {
return trimmed;
}
private void requestCompletionsContinuations(final List<ChatRequestMessage> chat, final StringBuilder answer, final CompletionStatistics statistics, final ModelCategory modelCategory, final ModelProfile modelProfile) {
for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
final LocalDateTime start = LocalDateTime.now();
final var response = this.requestCompletions(chat, modelCategory, modelProfile);
final var choice = response.getChoice();
statistics.add(Duration.between(start, LocalDateTime.now()), response.getUsage().getPromptTokens(), response.getUsage().getCompletionTokens());
answer.append(choice.getMessage().getContent());
final var complete = !(choice.getFinishReason() == CompletionsFinishReason.TOKEN_LIMIT_REACHED);
if (complete) {
LOGGER.debug("Prompt response complete");
break;
}
answer.append('\n');
chat.add(new ChatRequestAssistantMessage(choice.getMessage().getContent()));
chat.add(new ChatRequestUserMessage("continue"));
}
}
private ChatCompletions requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
private Completion requestCompletionsContinuations(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
final LocalDateTime start = LocalDateTime.now();
final var client = this.createHttpClient(modelCategory, modelProfile);
final var options = new ChatCompletionsOptions(chat);
options.setModel(modelProfile.modelName());
options.setMaxTokens(modelProfile.maxTokens());
options.setTemperature(modelProfile.temperature());
options.setTopP(modelProfile.topP());
return client.complete(options);
final var client = this.createHttpClient(modelCategory, modelProfile);
final var response = client.complete(options);
final CompletionFinishReason finishReason;
if (CompletionsFinishReason.CONTENT_FILTERED.equals(response.getChoice().getFinishReason())) {
finishReason = CompletionFinishReason.CONTENT_FILTERED;
}
else if (CompletionsFinishReason.TOKEN_LIMIT_REACHED.equals(response.getChoice().getFinishReason())) {
finishReason = CompletionFinishReason.TOKEN_LIMIT_REACHED;
}
else {
finishReason = CompletionFinishReason.DONE;
}
final var statistics = new CompletionStatistics(
1,
Duration.between(start, LocalDateTime.now()),
response.getUsage().getPromptTokens(),
response.getUsage().getCompletionTokens(),
new BigDecimal(response.getUsage().getPromptTokens()).multiply(modelProfile.costInToken()).add(new BigDecimal(response.getUsage().getCompletionTokens()).multiply(modelProfile.costOutToken())),
false);
LOGGER.debug("{}", statistics);
return new Completion(response.getChoice().getMessage().getContent(), finishReason, statistics);
}
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
@@ -258,19 +235,20 @@ public final class AzureAiClient implements Client {
}
return false;
});
final OkHttpClient.Builder nativeOkHttpClientBuilder = new OkHttpClient.Builder();
if (this.proxyEnabled) {
LOGGER.info("Proxy is enabled. Configuring proxy settings for host: {} port: {} user: {}", this.proxyHost, this.proxyPort, this.proxyUser);
final Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(this.proxyHost, this.proxyPort));
nativeOkHttpClientBuilder.proxy(proxy);
if (this.proxyUser != null && this.proxyPassword != null) {
final Authenticator proxyAuthenticator = new Authenticator() {
@Override
public Request authenticate(final Route route, final Response response) {
public Request authenticate(final Route route, final Response response) {
for (final Challenge challenge : response.challenges()) {
if (challenge.scheme().equalsIgnoreCase("OkHttp-Preemptive")) {
final String credential = Credentials.basic(proxyUser, proxyPassword);
@@ -283,13 +261,15 @@ public final class AzureAiClient implements Client {
}
};
nativeOkHttpClientBuilder.proxyAuthenticator(proxyAuthenticator);
} else {
}
else {
LOGGER.warn("Proxy user or password is not set. Skipping proxy authentication.");
}
} else {
}
else {
LOGGER.info("Proxy is not enabled. Skipping proxy configuration.");
}
final OkHttpClient nativeOkHttpClient = nativeOkHttpClientBuilder.build();
final OkHttpAsyncHttpClientBuilder okHttpClientBuilder = new OkHttpAsyncHttpClientBuilder(nativeOkHttpClient);
final HttpClient okHttpClient = okHttpClientBuilder
@@ -313,24 +293,4 @@ public final class AzureAiClient implements Client {
return statusCode == 408 || statusCode == 429 || (statusCode >= 500 && statusCode < 600);
}
static class CompletionStatistics {
int requests;
Duration duration = Duration.ofMillis(0l);
long inTokens;
long outTokens;
void add(final Duration duration, final int inTokens, final int outTokens) {
LOGGER.debug("Request duration: {}; in: {}; out: {}; tps: {}", duration, inTokens, outTokens, String.format("%.2f", (inTokens + outTokens) / (double) duration.toMillis() * 1000.0));
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
}
}
}

8
src/main/java/de/extio/lmlib/client/cached/CachedClient.java
View File

@@ -11,6 +11,8 @@ import java.util.function.Supplier;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.CompletionStatistics;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
@@ -50,12 +52,12 @@ public class CachedClient implements Client {
final var cachedCompletion = this.cacheRepository.get(key);
if (cachedCompletion != null) {
return new Completion(cachedCompletion.response(), 0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true);
return new Completion(cachedCompletion.response(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true));
}
final var completion = supplier.get();
if (completion != null) {
this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.inTokens(), completion.outTokens(), OffsetDateTime.now()));
this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now()));
}
return completion;
}
@@ -76,7 +78,7 @@ public class CachedClient implements Client {
digest.update(String.valueOf(modelProfile.modelProvider()).getBytes());
digest.update(String.valueOf(modelProfile.maxTokens()).getBytes());
digest.update(String.valueOf(modelProfile.maxContextLength()).getBytes());
digest.update(String.valueOf(modelProfile.maxContinuations()).getBytes());
digest.update(String.valueOf("1").getBytes());
digest.update(String.valueOf(modelProfile.temperature()).getBytes());
digest.update(String.valueOf(modelProfile.topP()).getBytes());
for (final var k : keys) {

4
src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java
View File

@@ -2,6 +2,8 @@ package de.extio.lmlib.client.cached;
import java.time.OffsetDateTime;
public record CachedCompletion(String response, long inTokens, long outTokens, OffsetDateTime created) {
import de.extio.lmlib.client.CompletionFinishReason;
public record CachedCompletion(String response, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) {
}

128
src/main/java/de/extio/lmlib/client/oai/textcompletion/TextCompletionClient.java
View File

@@ -6,10 +6,8 @@ import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -23,6 +21,7 @@ import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.ModelNameSupplier;
@@ -33,7 +32,6 @@ import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import de.extio.lmlib.token.Tokenizer;
import reactor.util.retry.Retry;
@Component
@@ -87,37 +85,17 @@ public class TextCompletionClient implements Client, DisposableBean {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
}
final List<StringBuilder> prompts = this.createPrompts(conversation, modelCategory, modelProfile, promptStrategy);
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
final List<StringBuilder> answers = new ArrayList<>();
final CompletionStatistics statistics = new CompletionStatistics();
for (final StringBuilder prompt : prompts) {
final StringBuilder answer = new StringBuilder();
this.requestCompletionsContinuations(prompt, answer, statistics, modelProfile, promptStrategy);
answers.add(answer);
}
if (answers.size() > 1) {
this.summarize(answers, modelProfile, promptStrategy, statistics);
}
return new Completion(
answers.stream().collect(Collectors.joining()),
statistics.requests,
statistics.duration,
statistics.inTokens,
statistics.outTokens,
new BigDecimal(statistics.inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(statistics.outTokens).multiply(modelProfile.costOutToken())),
false);
return this.requestCompletion(prompt, modelProfile, promptStrategy);
}
private List<StringBuilder> createPrompts(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokenized.size() > maxLength) {
// TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, maxLength);
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
@@ -125,7 +103,7 @@ public class TextCompletionClient implements Client, DisposableBean {
LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
return List.of(prompt);
return prompt.toString();
}
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
@@ -168,51 +146,9 @@ public class TextCompletionClient implements Client, DisposableBean {
return prompt;
}
private void requestCompletionsContinuations(final StringBuilder prompt, final StringBuilder answer, final CompletionStatistics statistics,
final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
for (int continuation = 0; continuation < modelProfile.maxContinuations(); continuation++) {
final LocalDateTime start = LocalDateTime.now();
final String promptStr = prompt.toString();
final var response = this.requestCompletions(promptStr, modelProfile);
String content = null;
var complete = false;
if (response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
complete = response.isStoppedEos();
}
else {
final var choice = response.getChoices().getFirst();
content = promptStrategy.removeEOT(choice.getText());
complete = !Choice.FINISH_REASON_LENGTH.equals(choice.getFinishReason());
}
if (this.collectStatistics) {
final var dur = Duration.between(start, LocalDateTime.now());
final var in = this.tokenizer.count(promptStr, modelProfile);
final var out = this.tokenizer.count(content, modelProfile);
final var cost = new BigDecimal(in).multiply(modelProfile.costInToken()).add(new BigDecimal(out).multiply(modelProfile.costOutToken()));
LOGGER.debug("Request duration: " + dur + "; in tokens: " + in + "; out tokens: " + out + "; cost: " + new DecimalFormat("0.####").format(cost));
statistics.add(dur, in, out, cost);
this.totalStatistics.add(dur, in, out, cost);
}
answer.append(content);
if (complete) {
LOGGER.debug("Prompt response complete");
break;
}
answer.append('\n');
promptStrategy.continue_(prompt, content);
}
}
private CompletionResponse requestCompletions(final String prompt, final ModelProfile modelProfile) {
private Completion requestCompletion(final String prompt, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
LOGGER.debug("Requesting completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
@@ -223,7 +159,7 @@ public class TextCompletionClient implements Client, DisposableBean {
request.setStream(false);
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
return webClient
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.bodyValue(request)
@@ -231,22 +167,38 @@ public class TextCompletionClient implements Client, DisposableBean {
.bodyToMono(CompletionResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
}
private void summarize(final List<StringBuilder> answers, final ModelProfile modelProfile, final PromptStrategy promptStrategy, final CompletionStatistics statistics) {
// TODO: Reactivate when multiple (split) prompts have been implemented again in #createPrompts
// LOGGER.debug("Summarizing answers");
//
// final StringBuilder summary = new StringBuilder();
//
// final StringBuilder prompt = promptStrategy.start(instruction, question, "");
// answers.forEach(answer -> promptStrategy.next(prompt, answer.toString(), "Continue"));
// promptStrategy.next(prompt, "", "Generate now a full summary");
//
// this.requestCompletionsContinuations(prompt, summary, statistics, modelProfile, promptStrategy);
//
// answers.clear();
// answers.add(summary);
String content;
CompletionFinishReason finishReason;
if (response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
}
else {
final var choice = response.getChoices().getFirst();
content = promptStrategy.removeEOT(choice.getText());
finishReason = switch (choice.getFinishReason()) {
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
}
final var in = this.tokenizer.count(prompt, modelProfile);
final var out = this.tokenizer.count(content, modelProfile);
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
1,
Duration.between(start, LocalDateTime.now()),
in,
out,
new BigDecimal(in).multiply(modelProfile.costInToken()).add(new BigDecimal(out).multiply(modelProfile.costOutToken())),
false);
LOGGER.debug("{}", statistics);
if (this.collectStatistics) {
this.totalStatistics.add(statistics.duration(), in, out, statistics.cost());
}
return new Completion(content, finishReason, statistics);
}
@Override

1
src/main/java/de/extio/lmlib/profile/ModelProfile.java
View File

@@ -9,7 +9,6 @@ public record ModelProfile(
int maxContextLength,
double temperature,
double topP,
int maxContinuations,
ModelProvider modelProvider,
String modelName,
String url,

8
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -65,10 +65,6 @@ public class ModelProfileService {
.map(Object::toString)
.map(Double::parseDouble)
.orElseThrow(() -> new IllegalStateException("No model topP found for model: " + modelName));
final var maxContinuations = Optional.ofNullable(resource.getProperty("maxContinuations"))
.map(Object::toString)
.map(Integer::parseInt)
.orElseThrow(() -> new IllegalStateException("No model maxContinuations found for model: " + modelName));
final var modelProvider = Optional.ofNullable(resource.getProperty("modelProvider"))
.map(Object::toString)
.map(ModelProvider::valueOf)
@@ -93,7 +89,7 @@ public class ModelProfileService {
//To resolve api key placeholder ${exchangeautodocumentation.azure.apikey} in model-name.properties file
String apiKey = null;
synchronized (this.environment) {
if (this.environment instanceof ConfigurableEnvironment configurableEnvironment) {
if (this.environment instanceof final ConfigurableEnvironment configurableEnvironment) {
configurableEnvironment.getPropertySources().addFirst(resource);
}
apiKey = Optional.ofNullable(this.environment.getProperty("apiKey"))
@@ -101,7 +97,7 @@ public class ModelProfileService {
.orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
}
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, maxContinuations, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken);
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken);
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile", e);

5
src/main/java/de/extio/lmlib/prompt/AlpacaShortPromptStrategy.java
View File

@@ -16,11 +16,6 @@ public class AlpacaShortPromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);

8
src/main/java/de/extio/lmlib/prompt/ChatMLPromptStrategy.java
View File

@@ -8,9 +8,8 @@ public class ChatMLPromptStrategy implements PromptStrategy {
@Override
public StringBuilder start(final String system, final String user) {
final StringBuilder prompt = new StringBuilder();
prompt.append("<|im_start|>");
if (!system.isEmpty()) {
prompt.append("system\n");
prompt.append("<|im_start|>system\n");
prompt.append(system);
prompt.append("<|im_end|>\n");
}
@@ -20,11 +19,6 @@ public class ChatMLPromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);

5
src/main/java/de/extio/lmlib/prompt/Gemma2PromptStrategy.java
View File

@@ -22,11 +22,6 @@ public class Gemma2PromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);

5
src/main/java/de/extio/lmlib/prompt/Llama3PromptStrategy.java
View File

@@ -24,11 +24,6 @@ public class Llama3PromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);

5
src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java
View File

@@ -24,11 +24,6 @@ public class Llama4PromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);

5
src/main/java/de/extio/lmlib/prompt/MistralNemoPromptStrategy.java
View File

@@ -18,11 +18,6 @@ public class MistralNemoPromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(assistant);

5
src/main/java/de/extio/lmlib/prompt/MistralPromptStrategy.java
View File

@@ -18,11 +18,6 @@ public class MistralPromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(" ");

5
src/main/java/de/extio/lmlib/prompt/Phi3PromptStrategy.java
View File

@@ -18,11 +18,6 @@ public class Phi3PromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append("\n");

5
src/main/java/de/extio/lmlib/prompt/Phi4PromptStrategy.java
View File

@@ -19,11 +19,6 @@ public class Phi4PromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append("\n");

2
src/main/java/de/extio/lmlib/prompt/PromptStrategy.java
View File

@@ -6,8 +6,6 @@ public interface PromptStrategy {
StringBuilder start(String system, String user);
void continue_(StringBuilder prompt, String assistant);
void next(StringBuilder prompt, String assistant, String user);
default String removeEOT(final String prompt) {

5
src/main/java/de/extio/lmlib/prompt/VicunaPromptStrategy.java
View File

@@ -18,11 +18,6 @@ public class VicunaPromptStrategy implements PromptStrategy {
return prompt;
}
@Override
public void continue_(final StringBuilder prompt, final String assistant) {
this.next(prompt, assistant, "Continue");
}
@Override
public void next(final StringBuilder prompt, final String assistant, final String user) {
prompt.append(" ");