exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Calculate effective duration and tps;

Browse Source 
git-svn-id: svn://home0/lm-lib@3652 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2024-12-22 12:34:07 +00:00
parent 63087a1cd6
commit cb2c1c02da
2 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/de/extio/lmlib/client/textcompletion/TextCompletionClient.java
View File

@@ -2,6 +2,7 @@ package de.extio.lmlib.client.textcompletion;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -28,8 +29,8 @@ import de.extio.lmlib.client.textcompletion.dto.CompletionRequest;
import de.extio.lmlib.client.textcompletion.dto.CompletionResponse;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import de.extio.lmlib.token.Tokenizer;
@@ -270,6 +271,8 @@ public class TextCompletionClient implements Client, DisposableBean {
static class CompletionStatistics {
private final Instant start = Instant.now();
volatile int requests;
volatile Duration duration = Duration.ofMillis(0l);
@@ -300,6 +303,13 @@ public class TextCompletionClient implements Client, DisposableBean {
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("#.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("#.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append("]");
return builder.toString();
}