exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Update model profile and statistics tracking

Browse Source 
- Updated model profile to support new cost parameters: costPerCachedInToken and costPerReasoningOutToken.
- Enhanced AgentRequestStatistic to track cached input tokens and reasoning output tokens.
- Modified CompletionStatistics to include cached input tokens and reasoning output tokens.
- Updated AzureAiClient, CachedClient, and other clients to handle new statistics fields.
- Introduced Timings class to capture detailed timing metrics for requests.
- Adjusted Usage class to include input and output token details.
- Updated properties files to include new cost parameters for cached and reasoning tokens.
This commit is contained in:
extio-de
2025-11-30 18:19:01 +01:00
parent a6087a0613
commit b7c1e90dc7
19 changed files with 362 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java
View File

@@ -31,8 +31,12 @@ public final class AgentRequestStatistic {
private final AtomicLong inTokens = new AtomicLong();
private final AtomicLong cachedInTokens = new AtomicLong();
private final AtomicLong outTokens = new AtomicLong();
private final AtomicLong reasoningOutTokens = new AtomicLong();
private volatile Instant start;
private volatile Duration requestDuration = Duration.ofMillis(0L);
@@ -51,10 +55,18 @@ public final class AgentRequestStatistic {
return this.inTokens;
}
public AtomicLong getCachedInTokens() {
return this.cachedInTokens;
}
public AtomicLong getOutTokens() {
return this.outTokens;
}
public AtomicLong getReasoningOutTokens() {
return this.reasoningOutTokens;
}
public Duration getRequestDuration() {
return this.requestDuration;
}
@@ -68,12 +80,12 @@ public final class AgentRequestStatistic {
public String getTps() {
final var duration = this.start == null ? this.getRequestDuration() : this.getEffectiveDuration();
return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get()) / (double) duration.toMillis() * 1000.0);
return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get() + this.reasoningOutTokens.get()) / (double) duration.toMillis() * 1000.0);
}
public String getOutTps() {
final var duration = this.start == null ? this.getRequestDuration() : this.getEffectiveDuration();
return new DecimalFormat("#.##").format((double) this.outTokens.get() / (double) duration.toMillis() * 1000.0);
return new DecimalFormat("#.##").format((double) (this.outTokens.get() + this.reasoningOutTokens.get()) / (double) duration.toMillis() * 1000.0);
}
public BigDecimal getCost() {
@@ -86,9 +98,11 @@ public final class AgentRequestStatistic {
}
else {
this.getRequests().addAndGet(completion.statistics().requests());
this.getInTokens().addAndGet(completion.statistics().inTokens());
this.getCachedInTokens().addAndGet(completion.statistics().cachedInTokens());
this.getOutTokens().addAndGet(completion.statistics().outTokens());
this.getReasoningOutTokens().addAndGet(completion.statistics().reasoningOutTokens());
}
this.getInTokens().addAndGet(completion.statistics().inTokens());
this.getOutTokens().addAndGet(completion.statistics().outTokens());
synchronized (this) {
this.requestDuration = this.requestDuration.plus(completion.statistics().duration());
this.cost = this.cost.add(completion.statistics().cost());
@@ -99,7 +113,9 @@ public final class AgentRequestStatistic {
this.getRequests().addAndGet(other.getRequests().get());
this.getCachedPrompts().addAndGet(other.getCachedPrompts().get());
this.getInTokens().addAndGet(other.getInTokens().get());
this.getCachedInTokens().addAndGet(other.getCachedInTokens().get());
this.getOutTokens().addAndGet(other.getOutTokens().get());
this.getReasoningOutTokens().addAndGet(other.getReasoningOutTokens().get());
synchronized (this) {
if (this.start != null && other.start.isBefore(this.start)) {
this.start = other.start;
@@ -118,8 +134,12 @@ public final class AgentRequestStatistic {
builder.append(this.cachedPrompts);
builder.append(", inTokens=");
builder.append(this.inTokens);
builder.append(", cachedInTokens=");
builder.append(this.cachedInTokens);
builder.append(", outTokens=");
builder.append(this.outTokens);
builder.append(", reasoningOutTokens=");
builder.append(this.reasoningOutTokens);
builder.append(", requestDuration=");
builder.append(this.requestDuration);
builder.append(", getEffectiveDuration()=");