exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lm-lib: Merged with tp fork;

Browse Source 
git-svn-id: svn://home0/lm-lib@3723 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2025-02-23 19:08:49 +00:00
parent c5ada1049d
commit 3c412077bb
13 changed files with 284 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -3,8 +3,11 @@ package de.extio.lmlib.client;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import de.extio.lmlib.client.cached.CachedClient;
import de.extio.lmlib.client.cached.CachedClientRepository;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfileService;
@@ -17,11 +20,16 @@ public class ClientService {
@Autowired
private ModelProfileService modelProfileService;
@Autowired(required = false)
private CachedClientRepository cachedClientRepository;
@Cacheable("clients")
public Client getClient(final ModelCategory category) {
final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile());
return this.clients.stream()
.filter(client -> client.getModelProvider() == modelProfile.modelProvider())
.findFirst()
.map(client -> this.cachedClientRepository == null ? client : new CachedClient(this.cachedClientRepository, this.modelProfileService, client))
.orElseThrow(() -> new IllegalStateException("No client found for model profile " + category));
}

3
src/main/java/de/extio/lmlib/client/Completion.java
View File

@@ -1,7 +1,8 @@
package de.extio.lmlib.client;
import java.math.BigDecimal;
import java.time.Duration;
public final record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, boolean cached) {
public final record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, BigDecimal cost, boolean cached) {
}

26
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -1,5 +1,6 @@
package de.extio.lmlib.client.azureai;
import java.math.BigDecimal;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
@@ -108,6 +109,7 @@ public final class AzureAiClient implements Client {
statistics.duration,
statistics.inTokens,
statistics.outTokens,
new BigDecimal(statistics.inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(statistics.outTokens).multiply(modelProfile.costOutToken())),
false);
}
@@ -120,7 +122,7 @@ public final class AzureAiClient implements Client {
Turn assistantTurn = null;
boolean first = true;
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 50; // 50 is a margin for EOT and other special tokens
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
final AtomicInteger tokens = new AtomicInteger();
while (!turns.isEmpty()) {
@@ -165,21 +167,19 @@ public final class AzureAiClient implements Client {
}
private boolean addChatRequestMessage(final String text, final int maxLength, final AtomicInteger tokens, final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Function<String, ChatRequestMessage> messageFactory) {
final int textTokens = this.tokenizer.count(text, modelProfile);
String trimmedText;
String prompt = text;
boolean trimmed = false;
final int textTokens = this.tokenizer.count(prompt, modelProfile);
if (tokens.addAndGet(textTokens) > maxLength) {
// TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokens.get(), maxLength);
trimmedText = text.substring(0, text.length() - (tokens.get() - maxLength) / 3);
List<Long> tokenized = this.tokenizer.tokenize(prompt, modelProfile);
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
trimmed = true;
}
else {
trimmedText = text;
}
chat.add(messageFactory.apply(trimmedText));
}
chat.add(messageFactory.apply(prompt));
return trimmed;
}
@@ -246,7 +246,7 @@ public final class AzureAiClient implements Client {
long outTokens;
void add(final Duration duration, final int inTokens, final int outTokens) {
LOGGER.info("Request duration: {}; in: {}; out: {}; tps: {}", duration, inTokens, outTokens, String.format("%.2f", (inTokens + outTokens) / (double) duration.toMillis() * 1000.0));
LOGGER.debug("Request duration: {}; in: {}; out: {}; tps: {}", duration, inTokens, outTokens, String.format("%.2f", (inTokens + outTokens) / (double) duration.toMillis() * 1000.0));
this.requests++;
this.duration = this.duration.plus(duration);

36
src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java
View File

@@ -1,5 +1,6 @@
package de.extio.lmlib.client.textcompletion;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
@@ -122,6 +123,7 @@ public class TextCompletionClient implements Client, DisposableBean {
statistics.duration,
statistics.inTokens,
statistics.outTokens,
new BigDecimal(statistics.inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(statistics.outTokens).multiply(modelProfile.costOutToken())),
false);
}
@@ -207,9 +209,10 @@ public class TextCompletionClient implements Client, DisposableBean {
final var dur = Duration.between(start, LocalDateTime.now());
final var in = this.tokenizer.count(promptStr, modelProfile);
final var out = this.tokenizer.count(content, modelProfile);
LOGGER.debug("Request duration: " + dur + "; in tokens: " + in + "; out tokens: " + out);
statistics.add(dur, in, out);
this.totalStatistics.add(dur, in, out);
final var cost = new BigDecimal(in).multiply(modelProfile.costInToken()).add(new BigDecimal(out).multiply(modelProfile.costOutToken()));
LOGGER.debug("Request duration: " + dur + "; in tokens: " + in + "; out tokens: " + out + "; cost: " + new DecimalFormat("0.####").format(cost));
statistics.add(dur, in, out, cost);
this.totalStatistics.add(dur, in, out, cost);
}
answer.append(content);
@@ -247,7 +250,7 @@ public class TextCompletionClient implements Client, DisposableBean {
}
private void summarize(final List<StringBuilder> answers, final ModelProfile modelProfile, final PromptStrategy promptStrategy, final CompletionStatistics statistics) {
// TODO: Reactivate when multiple (split) prompts have been implemented again
// TODO: Reactivate when multiple (split) prompts have been implemented again in #createPrompts
// LOGGER.debug("Summarizing answers");
//
// final StringBuilder summary = new StringBuilder();
@@ -264,7 +267,7 @@ public class TextCompletionClient implements Client, DisposableBean {
@Override
public void destroy() throws Exception {
if (this.collectStatistics) {
if (this.collectStatistics && this.totalStatistics.requests > 0) {
LOGGER.info("Client Statistics: {}", this.totalStatistics);
}
}
@@ -273,19 +276,22 @@ public class TextCompletionClient implements Client, DisposableBean {
private final Instant start = Instant.now();
volatile int requests;
private int requests;
volatile Duration duration = Duration.ofMillis(0l);
private Duration duration = Duration.ofMillis(0l);
volatile long inTokens;
private long inTokens;
volatile long outTokens;
private long outTokens;
synchronized void add(final Duration duration, final int inTokens, final int outTokens) {
private BigDecimal cost = BigDecimal.ZERO;
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
this.cost = this.cost.add(cost);
}
@Override
@@ -300,16 +306,18 @@ public class TextCompletionClient implements Client, DisposableBean {
builder.append(", outTokens=");
builder.append(this.outTokens);
builder.append(", averageTps=");
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(", cost=");
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
builder.append("]");
return builder.toString();
}