exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Calculate effective duration and tps;

Browse Source 
git-svn-id: svn://home0/lm-lib@3652 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2024-12-22 12:34:07 +00:00
parent 63087a1cd6
commit cb2c1c02da
2 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java
View File

@@ -36,16 +36,16 @@ public final class AgentRequestStatistic {
return this.requestDuration;
}
public Duration getAbsoluteDuration() {
public Duration getEffectiveDuration() {
return Duration.between(this.start, Instant.now());
}
public String getTps() {
return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get()) / (double)this.getAbsoluteDuration().toMillis() * 1000.0);
return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get()) / (double)this.getEffectiveDuration().toMillis() * 1000.0);
}
public String getOutTps() {
return new DecimalFormat("#.##").format((double) this.outTokens.get() / (double)this.getAbsoluteDuration().toMillis() * 1000.0);
return new DecimalFormat("#.##").format((double) this.outTokens.get() / (double)this.getEffectiveDuration().toMillis() * 1000.0);
}
public void add(final Completion completion) {
@@ -70,8 +70,8 @@ public final class AgentRequestStatistic {
builder.append(this.outTokens);
builder.append(", requestDuration=");
builder.append(this.requestDuration);
builder.append(", getAbsoluteDuration()=");
builder.append(this.getAbsoluteDuration());
builder.append(", getEffectiveDuration()=");
builder.append(this.getEffectiveDuration());
builder.append(", getTps()=");
builder.append(this.getTps());
builder.append(", getOutTps()=");

12
src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java
View File

@@ -2,6 +2,7 @@ package de.extio.lmlib.client.textcompletion;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -28,8 +29,8 @@ import de.extio.lmlib.client.textcompletion.dto.CompletionRequest;
import de.extio.lmlib.client.textcompletion.dto.CompletionResponse;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import de.extio.lmlib.token.Tokenizer;
@@ -270,6 +271,8 @@ public class TextCompletionClient implements Client, DisposableBean {
static class CompletionStatistics {
private final Instant start = Instant.now();
volatile int requests;
volatile Duration duration = Duration.ofMillis(0l);
@@ -300,6 +303,13 @@ public class TextCompletionClient implements Client, DisposableBean {
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append("]");
return builder.toString();
}