exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lm-lib: Merged with tp fork;

Browse Source 
git-svn-id: svn://home0/lm-lib@3723 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2025-02-23 19:08:49 +00:00
parent c5ada1049d
commit 3c412077bb
13 changed files with 284 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/de/extio/lmlib/agent/AgentExecutorService.java
View File

@@ -15,6 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.ClientService;
@Service
@@ -47,10 +48,14 @@ public class AgentExecutorService implements InitializingBean, DisposableBean {
public List<AgentContext> walkGraph(final Agent agent, final AgentContext context) {
LOGGER.info("Agent: {}", agent.name());
final var client = this.clientService.getClient(agent.modelCategory());
final var responses = Collections.synchronizedList(new ArrayList<AgentContext>());
Client client = null;
if (agent.agentType() != AgentType.PROCESSING_ONLY) {
client = this.clientService.getClient(agent.modelCategory());
}
final var branches = agent.execute(client, this.agentExecutorService, context);
final var responses = Collections.synchronizedList(new ArrayList<AgentContext>());
final var branchTasks = new ArrayList<CompletableFuture<?>>(branches.size());
for (final var branchContext : branches) {
if (!branchContext.isError() && branchContext.getNextAgent() != null) {

49
src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java
View File

@@ -1,5 +1,6 @@
package de.extio.lmlib.agent;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
@@ -12,18 +13,26 @@ public final class AgentRequestStatistic {
private final AtomicInteger requests = new AtomicInteger();
private final AtomicInteger cachedPrompts = new AtomicInteger();
private final AtomicLong inTokens = new AtomicLong();
private final AtomicLong outTokens = new AtomicLong();
private final Instant start = Instant.now();
private volatile Instant start = Instant.now();
private volatile Duration requestDuration = Duration.ofMillis(0L);
private volatile BigDecimal cost = BigDecimal.ZERO;
public AtomicInteger getRequests() {
return this.requests;
}
public AtomicInteger getCachedPrompts() {
return this.cachedPrompts;
}
public AtomicLong getInTokens() {
return this.inTokens;
}
@@ -41,22 +50,44 @@ public final class AgentRequestStatistic {
}
public String getTps() {
return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get()) / (double)this.getEffectiveDuration().toMillis() * 1000.0);
return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get()) / (double) this.getEffectiveDuration().toMillis() * 1000.0);
}
public String getOutTps() {
return new DecimalFormat("#.##").format((double) this.outTokens.get() / (double)this.getEffectiveDuration().toMillis() * 1000.0);
return new DecimalFormat("#.##").format((double) this.outTokens.get() / (double) this.getEffectiveDuration().toMillis() * 1000.0);
}
public BigDecimal getCost() {
return this.cost;
}
public void add(final Completion completion) {
this.getRequests().addAndGet(completion.requests());
if (completion.cached()) {
this.cachedPrompts.incrementAndGet();
}
else {
this.getRequests().addAndGet(completion.requests());
}
this.getInTokens().addAndGet(completion.inTokens());
this.getOutTokens().addAndGet(completion.outTokens());
this.addRequestDuration(completion.duration());
synchronized (this) {
this.requestDuration = this.requestDuration.plus(completion.duration());
this.cost = this.cost.add(completion.cost());
}
}
public synchronized void addRequestDuration(final Duration duration) {
this.requestDuration = this.requestDuration.plus(duration);
public void add(final AgentRequestStatistic other) {
this.getRequests().addAndGet(other.getRequests().get());
this.getCachedPrompts().addAndGet(other.getCachedPrompts().get());
this.getInTokens().addAndGet(other.getInTokens().get());
this.getOutTokens().addAndGet(other.getOutTokens().get());
synchronized (this) {
if (other.start.isBefore(this.start)) {
this.start = other.start;
}
this.requestDuration = this.requestDuration.plus(other.getRequestDuration());
this.cost = this.cost.add(other.getCost());
}
}
@Override
@@ -64,6 +95,8 @@ public final class AgentRequestStatistic {
final StringBuilder builder = new StringBuilder();
builder.append("AgentRequestStatistic [requests=");
builder.append(this.requests);
builder.append(", cachedPrompts=");
builder.append(this.cachedPrompts);
builder.append(", inTokens=");
builder.append(this.inTokens);
builder.append(", outTokens=");
@@ -76,6 +109,8 @@ public final class AgentRequestStatistic {
builder.append(this.getTps());
builder.append(", getOutTps()=");
builder.append(this.getOutTps());
builder.append(", getCost()=");
builder.append(this.getCost());
builder.append("]");
return builder.toString();
}

8
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -3,8 +3,11 @@ package de.extio.lmlib.client;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import de.extio.lmlib.client.cached.CachedClient;
import de.extio.lmlib.client.cached.CachedClientRepository;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfileService;
@@ -17,11 +20,16 @@ public class ClientService {
@Autowired
private ModelProfileService modelProfileService;
@Autowired(required = false)
private CachedClientRepository cachedClientRepository;
@Cacheable("clients")
public Client getClient(final ModelCategory category) {
final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile());
return this.clients.stream()
.filter(client -> client.getModelProvider() == modelProfile.modelProvider())
.findFirst()
.map(client -> this.cachedClientRepository == null ? client : new CachedClient(this.cachedClientRepository, this.modelProfileService, client))
.orElseThrow(() -> new IllegalStateException("No client found for model profile " + category));
}

3
src/main/java/de/extio/lmlib/client/Completion.java
View File

@@ -1,7 +1,8 @@
package de.extio.lmlib.client;
import java.math.BigDecimal;
import java.time.Duration;
public final record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, boolean cached) {
public final record Completion(String response, int requests, Duration duration, long inTokens, long outTokens, BigDecimal cost, boolean cached) {
}

26
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -1,5 +1,6 @@
package de.extio.lmlib.client.azureai;
import java.math.BigDecimal;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
@@ -108,6 +109,7 @@ public final class AzureAiClient implements Client {
statistics.duration,
statistics.inTokens,
statistics.outTokens,
new BigDecimal(statistics.inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(statistics.outTokens).multiply(modelProfile.costOutToken())),
false);
}
@@ -120,7 +122,7 @@ public final class AzureAiClient implements Client {
Turn assistantTurn = null;
boolean first = true;
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 50; // 50 is a margin for EOT and other special tokens
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
final AtomicInteger tokens = new AtomicInteger();
while (!turns.isEmpty()) {
@@ -165,21 +167,19 @@ public final class AzureAiClient implements Client {
}
private boolean addChatRequestMessage(final String text, final int maxLength, final AtomicInteger tokens, final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Function<String, ChatRequestMessage> messageFactory) {
final int textTokens = this.tokenizer.count(text, modelProfile);
String trimmedText;
String prompt = text;
boolean trimmed = false;
final int textTokens = this.tokenizer.count(prompt, modelProfile);
if (tokens.addAndGet(textTokens) > maxLength) {
// TODO: Cut smarter, e.g. split conversation into parts while keeping system prompt
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokens.get(), maxLength);
trimmedText = text.substring(0, text.length() - (tokens.get() - maxLength) / 3);
List<Long> tokenized = this.tokenizer.tokenize(prompt, modelProfile);
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
trimmed = true;
}
else {
trimmedText = text;
}
chat.add(messageFactory.apply(trimmedText));
}
chat.add(messageFactory.apply(prompt));
return trimmed;
}
@@ -246,7 +246,7 @@ public final class AzureAiClient implements Client {
long outTokens;
void add(final Duration duration, final int inTokens, final int outTokens) {
LOGGER.info("Request duration: {}; in: {}; out: {}; tps: {}", duration, inTokens, outTokens, String.format("%.2f", (inTokens + outTokens) / (double) duration.toMillis() * 1000.0));
LOGGER.debug("Request duration: {}; in: {}; out: {}; tps: {}", duration, inTokens, outTokens, String.format("%.2f", (inTokens + outTokens) / (double) duration.toMillis() * 1000.0));
this.requests++;
this.duration = this.duration.plus(duration);

36
src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java
View File

@@ -1,5 +1,6 @@
package de.extio.lmlib.client.textcompletion;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
@@ -122,6 +123,7 @@ public class TextCompletionClient implements Client, DisposableBean {
statistics.duration,
statistics.inTokens,
statistics.outTokens,
new BigDecimal(statistics.inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(statistics.outTokens).multiply(modelProfile.costOutToken())),
false);
}
@@ -207,9 +209,10 @@ public class TextCompletionClient implements Client, DisposableBean {
final var dur = Duration.between(start, LocalDateTime.now());
final var in = this.tokenizer.count(promptStr, modelProfile);
final var out = this.tokenizer.count(content, modelProfile);
LOGGER.debug("Request duration: " + dur + "; in tokens: " + in + "; out tokens: " + out);
statistics.add(dur, in, out);
this.totalStatistics.add(dur, in, out);
final var cost = new BigDecimal(in).multiply(modelProfile.costInToken()).add(new BigDecimal(out).multiply(modelProfile.costOutToken()));
LOGGER.debug("Request duration: " + dur + "; in tokens: " + in + "; out tokens: " + out + "; cost: " + new DecimalFormat("0.####").format(cost));
statistics.add(dur, in, out, cost);
this.totalStatistics.add(dur, in, out, cost);
}
answer.append(content);
@@ -247,7 +250,7 @@ public class TextCompletionClient implements Client, DisposableBean {
}
private void summarize(final List<StringBuilder> answers, final ModelProfile modelProfile, final PromptStrategy promptStrategy, final CompletionStatistics statistics) {
// TODO: Reactivate when multiple (split) prompts have been implemented again
// TODO: Reactivate when multiple (split) prompts have been implemented again in #createPrompts
// LOGGER.debug("Summarizing answers");
//
// final StringBuilder summary = new StringBuilder();
@@ -264,7 +267,7 @@ public class TextCompletionClient implements Client, DisposableBean {
@Override
public void destroy() throws Exception {
if (this.collectStatistics) {
if (this.collectStatistics && this.totalStatistics.requests > 0) {
LOGGER.info("Client Statistics: {}", this.totalStatistics);
}
}
@@ -273,19 +276,22 @@ public class TextCompletionClient implements Client, DisposableBean {
private final Instant start = Instant.now();
volatile int requests;
private int requests;
volatile Duration duration = Duration.ofMillis(0l);
private Duration duration = Duration.ofMillis(0l);
volatile long inTokens;
private long inTokens;
volatile long outTokens;
private long outTokens;
synchronized void add(final Duration duration, final int inTokens, final int outTokens) {
private BigDecimal cost = BigDecimal.ZERO;
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
this.cost = this.cost.add(cost);
}
@Override
@@ -300,16 +306,18 @@ public class TextCompletionClient implements Client, DisposableBean {
builder.append(", outTokens=");
builder.append(this.outTokens);
builder.append(", averageTps=");
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(", cost=");
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
builder.append("]");
return builder.toString();
}

6
src/main/java/de/extio/lmlib/profile/ModelProfile.java
View File

@@ -1,5 +1,7 @@
package de.extio.lmlib.profile;
import java.math.BigDecimal;
public record ModelProfile(
String prompt,
String tokenEncoding,
@@ -11,7 +13,9 @@ public record ModelProfile(
ModelProvider modelProvider,
String modelName,
String url,
String apiKey) {
String apiKey,
BigDecimal costInToken,
BigDecimal costOutToken) {
public static enum ModelProvider {
AZURE_AI, OAI_TEXT_COMPLETION;

28
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -1,6 +1,7 @@
package de.extio.lmlib.profile;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@@ -12,6 +13,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.bind.Bindable;
import org.springframework.boot.context.properties.bind.Binder;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.Environment;
import org.springframework.core.io.support.ResourcePropertySource;
import org.springframework.stereotype.Service;
@@ -38,7 +40,9 @@ public class ModelProfileService {
throw new IllegalStateException("No model found for profile: " + modelProfile);
}
try {
final var resource = new ResourcePropertySource("classpath:" + modelName + ".properties");
final var promptTemplate = Optional.ofNullable(resource.getProperty("prompts"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model name found for model: " + modelName));
@@ -75,11 +79,29 @@ public class ModelProfileService {
final var url = Optional.ofNullable(resource.getProperty("url"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model url found for model: " + modelName));
final var apiKey = Optional.ofNullable(resource.getProperty("apiKey"))
final var costInToken = Optional.ofNullable(resource.getProperty("cost1MInTokens"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
.map(value -> new BigDecimal(value))
.orElse(BigDecimal.ZERO)
.divide(new BigDecimal(1000000.0));
final var costOutToken = Optional.ofNullable(resource.getProperty("cost1MOutTokens"))
.map(Object::toString)
.map(value -> new BigDecimal(value))
.orElse(BigDecimal.ZERO)
.divide(new BigDecimal(1000000.0));
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, maxContinuations, modelProvider, modelNameCfg, url, apiKey);
//To resolve api key placeholder ${exchangeautodocumentation.azure.apikey} in model-name.properties file
String apiKey = null;
synchronized (this.environment) {
if (this.environment instanceof ConfigurableEnvironment configurableEnvironment) {
configurableEnvironment.getPropertySources().addLast(resource);
}
apiKey = Optional.ofNullable(this.environment.getProperty("apiKey"))
.map(Object::toString)
.orElseThrow(() -> new IllegalStateException("No model apiKey found for model: " + modelName));
}
return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, maxContinuations, modelProvider, modelNameCfg, url, apiKey, costInToken, costOutToken);
}
catch (final IOException e) {
LOGGER.error("Error while reading model profile", e);

4
src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java
View File

@@ -14,7 +14,7 @@ public class TokenizerAutoConfiguration {
@Bean
@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "llamaServer")
@ConditionalOnMissingBean
@ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer")
Tokenizer llamaServerTokenizer(@Qualifier("lmLibWebClientBuilder") final WebClient.Builder webClientBuilder) {
return new LlamaServerTokenizer(webClientBuilder);
}
@@ -22,7 +22,7 @@ public class TokenizerAutoConfiguration {
@Configuration(proxyBeanMethods = false)
@ConditionalOnClass(name = "com.knuddels.jtokkit.Encodings")
@ConditionalOnProperty(name = "tokenizer.strategy", havingValue = "jTokkit")
@ConditionalOnMissingBean
@ConditionalOnMissingBean(type = "de.extio.lmlib.token.Tokenizer")
public static class SomeServiceConfiguration {
@Bean

2
src/test/resources/llama3-local.properties
View File

@@ -9,3 +9,5 @@ modelProvider=OAI_TEXT_COMPLETION
modelName=
url=http://localhost:5001
apiKey=
cost1MInTokens=0
cost1MOutTokens=0

13
src/test/resources/llama3.3-70b-azure.properties Normal file
View File

@@ -0,0 +1,13 @@
prompts=
tokenEncoding=cl100k_base
maxTokens=1500
maxContextLength=32767
temperature=0.2
topP=1.0
maxContinuations=1
modelProvider=AZURE_AI
modelName=Llama-3.3-70B-Instruct
url=XXX
apiKey=XXX
cost1MInTokens=0.71
cost1MOutTokens=0.71