From b7c1e90dc7d1c68a288f25984dcb15793a653897 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sun, 30 Nov 2025 18:19:01 +0100 Subject: [PATCH] feat: Update model profile and statistics tracking - Updated model profile to support new cost parameters: costPerCachedInToken and costPerReasoningOutToken. - Enhanced AgentRequestStatistic to track cached input tokens and reasoning output tokens. - Modified CompletionStatistics to include cached input tokens and reasoning output tokens. - Updated AzureAiClient, CachedClient, and other clients to handle new statistics fields. - Introduced Timings class to capture detailed timing metrics for requests. - Adjusted Usage class to include input and output token details. - Updated properties files to include new cost parameters for cached and reasoning tokens. --- README.md | 24 ++-- .../lmlib/agent/AgentRequestStatistic.java | 28 ++++- .../lmlib/client/CompletionStatistics.java | 2 +- .../lmlib/client/azureai/AzureAiClient.java | 2 + .../lmlib/client/cached/CachedClient.java | 4 +- .../lmlib/client/cached/CachedCompletion.java | 2 +- .../completion/AbstractCompletionClient.java | 73 +++++++++--- .../completion/AbstractCompletionRequest.java | 11 ++ .../AbstractCompletionResponse.java | 13 +- .../lmlib/client/oai/completion/Timings.java | 111 ++++++++++++++++++ .../lmlib/client/oai/completion/Usage.java | 86 +++++++++++++- .../completion/chat/ChatCompletionClient.java | 15 ++- .../completion/text/TextCompletionClient.java | 15 ++- .../de/extio/lmlib/profile/ModelProfile.java | 2 + .../lmlib/profile/ModelProfileService.java | 10 +- .../client/oai/CompletionClientTest.java | 2 +- .../resources/llama3.3-70b-azure.properties | 2 + .../resources/local-chatcompletion.properties | 2 + .../resources/local-completion.properties | 2 + 19 files changed, 362 insertions(+), 44 deletions(-) create mode 100644 src/main/java/de/extio/lmlib/client/oai/completion/Timings.java diff --git a/README.md b/README.md index 3fd5e08..0637184 100644 --- a/README.md +++ b/README.md @@ -108,11 +108,13 @@ final var modelProfile = new ModelProfile( 0.7, // temperature 1.0, // topP ModelProfile.ModelProvider.OAI_CHAT_COMPLETION, - "gpt-4", // modelName + "gpt-5.1", // modelName "https://api.openai.com/v1", // url System.getenv("OPENAI_API_KEY"), // apiKey - new BigDecimal("0.3").divide(1000000), // costPerInToken (per token) - new BigDecimal("0.6").divide(1000000), // costPerOutToken (per token) + new BigDecimal("1.25").divide(new BigDecimal(1000000)), // costPerInToken ($1.25/1M) + new BigDecimal("0.125").divide(new BigDecimal(1000000)), // costPerCachedInToken ($0.125/1M) + new BigDecimal("10.0").divide(new BigDecimal(1000000)), // costPerOutToken ($10.00/1M) + BigDecimal.ZERO, // costPerReasoningOutToken null, // reasoningEffort null, // reasoningSummaryDetails "DYNAMIC" // category (short name for logging) @@ -160,7 +162,9 @@ Each model profile is a `.properties` file that configures how the library inter | `url` | String | Yes | Base URL endpoint for the model API. | `https://api.openai.com/v1`, `http://localhost:5001` | | `apiKey` | String | No | Authentication key for the API. Can use Spring property placeholders like `${apikey}`. Can be empty for local models. | `sk-...`, `${apikey}`, `` (empty) | | `cost1MInTokens` | Double | Yes | Cost per 1 million input tokens in your currency. Use `0` for free/local models. | `0.71`, `0`, `5.00` | +| `cost1MCachedInTokens` | Double | No | Cost per 1 million cached input tokens. Defaults to `cost1MInTokens` if not specified. | `0.35`, `0` | | `cost1MOutTokens` | Double | Yes | Cost per 1 million output tokens in your currency. Use `0` for free/local models. | `0.71`, `0`, `15.00` | +| `cost1MReasoningOutTokens` | Double | No | Cost per 1 million reasoning output tokens. Defaults to `0` if not specified. | `0`, `60.00` | | `reasoningEffort` | String | No | For reasoning models (e.g., OpenAI o1): controls depth of reasoning process. | `low`, `medium`, `high` | | `reasoningSummaryDetails` | String | No | For reasoning models: controls how reasoning content is summarized in the response. | `auto`, `concise`, `detailed` | @@ -168,8 +172,10 @@ Each model profile is a `.properties` file that configures how the library inter - Properties marked "Yes" in the Required column must be present in every profile; missing required properties will cause runtime errors. - The `category` field is optional in profile files. When omitted, the category short name from the `ModelCategory` used to load the profile will be used instead. - The `prompts` property determines the prompt formatting strategy. For text completions, specify a strategy name (e.g., `llama3`, `gpt-oss`) or `no` for no formatting. For chat completions, use an empty string. -- The `modelName` and `apiKey` properties are optional and can be empty for local model deployments. +- The `modelName` and `apiKey` properties are optional and can be empty for local model deployments. When `apiKey` is provided, it is sent as a Bearer token in the Authorization header. - Cost properties are used for statistics tracking and have no impact on API calls. They help you monitor expenses. +- The `cost1MCachedInTokens` property defaults to `cost1MInTokens` if not specified, assuming the provider does not distinguish between cached and non-cached tokens. When the provider does support KV-cache pricing, cached tokens are typically charged at significantly lower rates (e.g., 10-20% of the regular input token cost). +- The `cost1MReasoningOutTokens` property defaults to `0` if not specified, as most models don't charge separately for reasoning tokens. - Reasoning properties are only applicable to models that support separate reasoning output (like OpenAI's o1 series). ### Reasoning Models @@ -672,14 +678,16 @@ final var customProfile = new ModelProfile( 0.3, // temperature 1.0, // topP ModelProfile.ModelProvider.OAI_CHAT_COMPLETION, // modelProvider - "gpt-4-turbo", // modelName + "gpt-5.1", // modelName "https://api.openai.com/v1", // url System.getenv("OPENAI_API_KEY"), // apiKey - new BigDecimal("0.3").divide(1000000), // costPerInToken (per token) - new BigDecimal("0.6").divide(1000000), // costPerOutToken (per token) + new BigDecimal("1.25").divide(new BigDecimal(1000000)), // costPerInToken ($1.25/1M) + new BigDecimal("0.125").divide(new BigDecimal(1000000)), // costPerCachedInToken ($0.125/1M) + new BigDecimal("10.0").divide(new BigDecimal(1000000)), // costPerOutToken ($10.00/1M) + BigDecimal.ZERO, // costPerReasoningOutToken null, // reasoningEffort null, // reasoningSummaryDetails - "GPT4T" // category + "GPT51" // category ); final var customModelAgent = new Agent( diff --git a/src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java b/src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java index 672c56d..12447fa 100755 --- a/src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java +++ b/src/main/java/de/extio/lmlib/agent/AgentRequestStatistic.java @@ -31,8 +31,12 @@ public final class AgentRequestStatistic { private final AtomicLong inTokens = new AtomicLong(); + private final AtomicLong cachedInTokens = new AtomicLong(); + private final AtomicLong outTokens = new AtomicLong(); + private final AtomicLong reasoningOutTokens = new AtomicLong(); + private volatile Instant start; private volatile Duration requestDuration = Duration.ofMillis(0L); @@ -51,10 +55,18 @@ public final class AgentRequestStatistic { return this.inTokens; } + public AtomicLong getCachedInTokens() { + return this.cachedInTokens; + } + public AtomicLong getOutTokens() { return this.outTokens; } + public AtomicLong getReasoningOutTokens() { + return this.reasoningOutTokens; + } + public Duration getRequestDuration() { return this.requestDuration; } @@ -68,12 +80,12 @@ public final class AgentRequestStatistic { public String getTps() { final var duration = this.start == null ? this.getRequestDuration() : this.getEffectiveDuration(); - return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get()) / (double) duration.toMillis() * 1000.0); + return new DecimalFormat("#.##").format((double) (this.inTokens.get() + this.outTokens.get() + this.reasoningOutTokens.get()) / (double) duration.toMillis() * 1000.0); } public String getOutTps() { final var duration = this.start == null ? this.getRequestDuration() : this.getEffectiveDuration(); - return new DecimalFormat("#.##").format((double) this.outTokens.get() / (double) duration.toMillis() * 1000.0); + return new DecimalFormat("#.##").format((double) (this.outTokens.get() + this.reasoningOutTokens.get()) / (double) duration.toMillis() * 1000.0); } public BigDecimal getCost() { @@ -86,9 +98,11 @@ public final class AgentRequestStatistic { } else { this.getRequests().addAndGet(completion.statistics().requests()); + this.getInTokens().addAndGet(completion.statistics().inTokens()); + this.getCachedInTokens().addAndGet(completion.statistics().cachedInTokens()); + this.getOutTokens().addAndGet(completion.statistics().outTokens()); + this.getReasoningOutTokens().addAndGet(completion.statistics().reasoningOutTokens()); } - this.getInTokens().addAndGet(completion.statistics().inTokens()); - this.getOutTokens().addAndGet(completion.statistics().outTokens()); synchronized (this) { this.requestDuration = this.requestDuration.plus(completion.statistics().duration()); this.cost = this.cost.add(completion.statistics().cost()); @@ -99,7 +113,9 @@ public final class AgentRequestStatistic { this.getRequests().addAndGet(other.getRequests().get()); this.getCachedPrompts().addAndGet(other.getCachedPrompts().get()); this.getInTokens().addAndGet(other.getInTokens().get()); + this.getCachedInTokens().addAndGet(other.getCachedInTokens().get()); this.getOutTokens().addAndGet(other.getOutTokens().get()); + this.getReasoningOutTokens().addAndGet(other.getReasoningOutTokens().get()); synchronized (this) { if (this.start != null && other.start.isBefore(this.start)) { this.start = other.start; @@ -118,8 +134,12 @@ public final class AgentRequestStatistic { builder.append(this.cachedPrompts); builder.append(", inTokens="); builder.append(this.inTokens); + builder.append(", cachedInTokens="); + builder.append(this.cachedInTokens); builder.append(", outTokens="); builder.append(this.outTokens); + builder.append(", reasoningOutTokens="); + builder.append(this.reasoningOutTokens); builder.append(", requestDuration="); builder.append(this.requestDuration); builder.append(", getEffectiveDuration()="); diff --git a/src/main/java/de/extio/lmlib/client/CompletionStatistics.java b/src/main/java/de/extio/lmlib/client/CompletionStatistics.java index 14b7ae7..88e2dcb 100644 --- a/src/main/java/de/extio/lmlib/client/CompletionStatistics.java +++ b/src/main/java/de/extio/lmlib/client/CompletionStatistics.java @@ -3,6 +3,6 @@ package de.extio.lmlib.client; import java.math.BigDecimal; import java.time.Duration; -public record CompletionStatistics(int requests, Duration duration, long inTokens, long outTokens, BigDecimal cost, boolean cached) { +public record CompletionStatistics(int requests, Duration duration, long inTokens, long cachedInTokens, long outTokens, long reasoningOutTokens, BigDecimal cost, boolean cached) { } diff --git a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java index 83e016b..52cdac8 100755 --- a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java +++ b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java @@ -290,7 +290,9 @@ public final class AzureAiClient implements Client { 1, Duration.between(start, LocalDateTime.now()), promptTokens, + 0, completionTokens, + 0, new BigDecimal(promptTokens).multiply(modelProfile.costPerInToken()).add(new BigDecimal(completionTokens).multiply(modelProfile.costPerOutToken())), false); LOGGER.debug("{}", statistics); diff --git a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java index f110d10..446ce16 100644 --- a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java +++ b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java @@ -78,12 +78,12 @@ public class CachedClient implements Client { final var cachedCompletion = this.cacheRepository.get(key); if (cachedCompletion != null) { - return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.outTokens(), BigDecimal.ZERO, true)); + return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.cachedInTokens(), cachedCompletion.outTokens(), cachedCompletion.reasoningOutTokens(), BigDecimal.ZERO, true)); } final var completion = supplier.get(); if (completion != null) { - this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.reasoning(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().outTokens(), OffsetDateTime.now())); + this.cacheRepository.put(key, new CachedCompletion(completion.response(), completion.reasoning(), completion.finishReason(), completion.statistics().inTokens(), completion.statistics().cachedInTokens(), completion.statistics().outTokens(), completion.statistics().reasoningOutTokens(), OffsetDateTime.now())); } return completion; } diff --git a/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java b/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java index f91149b..240e44c 100644 --- a/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java +++ b/src/main/java/de/extio/lmlib/client/cached/CachedCompletion.java @@ -4,6 +4,6 @@ import java.time.OffsetDateTime; import de.extio.lmlib.client.CompletionFinishReason; -public record CachedCompletion(String response, String reasoning, CompletionFinishReason finishReason, long inTokens, long outTokens, OffsetDateTime created) { +public record CachedCompletion(String response, String reasoning, CompletionFinishReason finishReason, long inTokens, long cachedInTokens, long outTokens, long reasoningOutTokens, OffsetDateTime created) { } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index c6efa57..048a656 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -140,32 +140,64 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean } } - protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) { + protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final Timings timings, final String prompt, final String response, final String reasoning) { int inTokens = 0; + int cachedInTokens = 0; int outTokens = 0; + int reasoningOutTokens = 0; + + // Usage data has priority (official OAI spec) if (usage != null && usage.getPromptTokens() > 0) { - inTokens = usage.getPromptTokens(); - outTokens = usage.getCompletionTokens(); + cachedInTokens = usage.getCachedTokens(); + inTokens = usage.getPromptTokens() - cachedInTokens; + reasoningOutTokens = usage.getReasoningTokens(); + outTokens = usage.getCompletionTokens() - reasoningOutTokens; } - else { - if (prompt != null) { - inTokens = this.tokenizer.count(prompt, modelProfile); + // If no usage data, try to get token counts from timings (e.g. llama.cpp, prompt_n does not include cache_n) + if (timings != null) { + if (inTokens <= 0 && cachedInTokens <= 0 && (timings.getPromptN() > 0 || timings.getCacheN() > 0)) { + inTokens = Math.max(0, timings.getPromptN()); + cachedInTokens = Math.max(0, timings.getCacheN()); } - if (response != null) { - outTokens = this.tokenizer.count(response, modelProfile); + else if (cachedInTokens <= 0 && timings.getCacheN() > 0) { + cachedInTokens = Math.max(0, timings.getCacheN()); + inTokens -= cachedInTokens; + } + if (outTokens <= 0 && timings.getPredictedN() > 0) { + outTokens = timings.getPredictedN(); } } + // Fallback to tokenizer estimation + if (inTokens <= 0 && prompt != null && !prompt.isBlank()) { + inTokens = this.tokenizer.count(prompt, modelProfile); + } + if (reasoningOutTokens <= 0 && reasoning != null && !reasoning.isBlank()) { + reasoningOutTokens = this.tokenizer.count(reasoning, modelProfile); + outTokens -= reasoningOutTokens; + } + if (outTokens <= 0 && response != null && !response.isBlank()) { + outTokens = this.tokenizer.count(response, modelProfile); + } + + // Calculate cost - inTokens and outTokens are already exclusive of cached/reasoning tokens + BigDecimal cost = BigDecimal.ZERO; + cost = cost.add(new BigDecimal(inTokens).multiply(modelProfile.costPerInToken())); + cost = cost.add(new BigDecimal(cachedInTokens).multiply(modelProfile.costPerCachedInToken())); + cost = cost.add(new BigDecimal(outTokens).multiply(modelProfile.costPerOutToken())); + cost = cost.add(new BigDecimal(reasoningOutTokens).multiply(modelProfile.costPerReasoningOutToken())); final var statistics = new de.extio.lmlib.client.CompletionStatistics( 1, Duration.between(start, LocalDateTime.now()), inTokens, + cachedInTokens, outTokens, - new BigDecimal(inTokens).multiply(modelProfile.costPerInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costPerOutToken())), + reasoningOutTokens, + cost, false); LOGGER.debug("{}", statistics); if (this.collectStatistics) { - this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost()); + this.totalStatistics.add(statistics.duration(), inTokens, cachedInTokens, outTokens, reasoningOutTokens, statistics.cost()); } return statistics; } @@ -187,15 +219,21 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean private long inTokens; + private long cachedInTokens; + private long outTokens; + private long reasoningOutTokens; + private BigDecimal cost = BigDecimal.ZERO; - synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) { + synchronized void add(final Duration duration, final int inTokens, final int cachedInTokens, final int outTokens, final int reasoningOutTokens, final BigDecimal cost) { this.requests++; this.duration = this.duration.plus(duration); this.inTokens += inTokens; + this.cachedInTokens += cachedInTokens; this.outTokens += outTokens; + this.reasoningOutTokens += reasoningOutTokens; this.cost = this.cost.add(cost); } @@ -208,19 +246,24 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean builder.append(this.duration); builder.append(", inTokens="); builder.append(this.inTokens); + builder.append(", cachedInTokens="); + builder.append(this.cachedInTokens); builder.append(", outTokens="); builder.append(this.outTokens); + builder.append(", reasoningOutTokens="); + builder.append(this.reasoningOutTokens); builder.append(", averageTps="); - builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests))); + final long totalOutTokens = this.outTokens + this.reasoningOutTokens; + builder.append(new DecimalFormat("0.##").format(((double) (totalOutTokens + inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests))); builder.append(", averageOutTps="); - builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0))); + builder.append(new DecimalFormat("0.##").format((double) totalOutTokens / ((double) this.duration.toMillis() / 1000.0))); builder.append(", effectiveDuration="); final var effectiveDuration = Duration.between(this.start, Instant.now()); builder.append(effectiveDuration); builder.append(", effectiveTps="); - builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests))); + builder.append(new DecimalFormat("0.##").format(((double) (totalOutTokens + inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests))); builder.append(", effectiveOutTps="); - builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0))); + builder.append(new DecimalFormat("0.##").format((double) totalOutTokens / ((double) effectiveDuration.toMillis() / 1000.0))); builder.append(", cost="); builder.append(new DecimalFormat("0.###").format(cost.doubleValue())); builder.append("]"); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java index 8e5531d..fc9f357 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionRequest.java @@ -16,6 +16,8 @@ public abstract class AbstractCompletionRequest { protected StreamOptions streamOptions; + protected Boolean usage; + @JsonProperty("model") public String getModel() { return this.model; @@ -69,4 +71,13 @@ public abstract class AbstractCompletionRequest { public void setStreamOptions(final StreamOptions streamOptions) { this.streamOptions = streamOptions; } + + @JsonProperty("usage") + public Boolean getUsage() { + return this.usage; + } + + public void setUsage(final Boolean usage) { + this.usage = usage; + } } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java index 275b7c8..a9f9a13 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionResponse.java @@ -11,6 +11,8 @@ public abstract class AbstractCompletionResponse { protected Integer created; protected Usage usage; + + protected Timings timings; @JsonProperty("id") public String getId() { @@ -41,10 +43,19 @@ public abstract class AbstractCompletionResponse { @JsonProperty("usage") public Usage getUsage() { - return usage; + return this.usage; } public void setUsage(final Usage usage) { this.usage = usage; } + + @JsonProperty("timings") + public Timings getTimings() { + return this.timings; + } + + public void setTimings(final Timings timings) { + this.timings = timings; + } } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java b/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java new file mode 100644 index 0000000..0ddfe0c --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java @@ -0,0 +1,111 @@ +package de.extio.lmlib.client.oai.completion; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public final class Timings { + + private int cacheN; + + private int predictedN; + + private double predictedMs; + + private double predictedPerTokenMs; + + private double predictedPerSecond; + + private int promptN; + + private double promptMs; + + private double promptPerTokenMs; + + private double promptPerSecond; + + @JsonProperty("cache_n") + public int getCacheN() { + return this.cacheN; + } + + public void setCacheN(final int cacheN) { + this.cacheN = cacheN; + } + + @JsonProperty("predicted_n") + public int getPredictedN() { + return this.predictedN; + } + + public void setPredictedN(final int predictedN) { + this.predictedN = predictedN; + } + + @JsonProperty("predicted_ms") + public double getPredictedMs() { + return this.predictedMs; + } + + public void setPredictedMs(final double predictedMs) { + this.predictedMs = predictedMs; + } + + @JsonProperty("predicted_per_token_ms") + public double getPredictedPerTokenMs() { + return this.predictedPerTokenMs; + } + + public void setPredictedPerTokenMs(final double predictedPerTokenMs) { + this.predictedPerTokenMs = predictedPerTokenMs; + } + + @JsonProperty("predicted_per_second") + public double getPredictedPerSecond() { + return this.predictedPerSecond; + } + + public void setPredictedPerSecond(final double predictedPerSecond) { + this.predictedPerSecond = predictedPerSecond; + } + + @JsonProperty("prompt_n") + public int getPromptN() { + return this.promptN; + } + + public void setPromptN(final int promptN) { + this.promptN = promptN; + } + + @JsonProperty("prompt_ms") + public double getPromptMs() { + return this.promptMs; + } + + public void setPromptMs(final double promptMs) { + this.promptMs = promptMs; + } + + @JsonProperty("prompt_per_token_ms") + public double getPromptPerTokenMs() { + return this.promptPerTokenMs; + } + + public void setPromptPerTokenMs(final double promptPerTokenMs) { + this.promptPerTokenMs = promptPerTokenMs; + } + + @JsonProperty("prompt_per_second") + public double getPromptPerSecond() { + return this.promptPerSecond; + } + + public void setPromptPerSecond(final double promptPerSecond) { + this.promptPerSecond = promptPerSecond; + } + + @Override + public String toString() { + return "Timings [cacheN=" + this.cacheN + ", predictedN=" + this.predictedN + ", predictedMs=" + this.predictedMs + ", predictedPerTokenMs=" + this.predictedPerTokenMs + ", predictedPerSecond=" + this.predictedPerSecond + ", promptN=" + this.promptN + ", promptMs=" + this.promptMs + ", promptPerTokenMs=" + this.promptPerTokenMs + ", promptPerSecond=" + this.promptPerSecond + "]"; + } + +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java index e0d3f62..6911a70 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java @@ -10,36 +10,114 @@ public final class Usage { private int totalTokens; + private InputTokensDetails inputTokensDetails; + + private OutputTokensDetails outputTokensDetails; + @JsonProperty("completion_tokens") public int getCompletionTokens() { - return completionTokens; + return this.completionTokens; } public void setCompletionTokens(final int completionTokens) { this.completionTokens = completionTokens; } + @JsonProperty("output_tokens") + public void setOutputTokens(final int outputTokens) { + this.completionTokens = outputTokens; + } + @JsonProperty("prompt_tokens") public int getPromptTokens() { - return promptTokens; + return this.promptTokens; } public void setPromptTokens(final int promptTokens) { this.promptTokens = promptTokens; } + @JsonProperty("input_tokens") + public void setInputTokens(final int inputTokens) { + this.promptTokens = inputTokens; + } + @JsonProperty("total_tokens") public int getTotalTokens() { - return totalTokens; + return this.totalTokens; } public void setTotalTokens(final int totalTokens) { this.totalTokens = totalTokens; } + @JsonProperty("input_tokens_details") + public InputTokensDetails getInputTokensDetails() { + return this.inputTokensDetails; + } + + public void setInputTokensDetails(final InputTokensDetails inputTokensDetails) { + this.inputTokensDetails = inputTokensDetails; + } + + @JsonProperty("output_tokens_details") + public OutputTokensDetails getOutputTokensDetails() { + return this.outputTokensDetails; + } + + public void setOutputTokensDetails(final OutputTokensDetails outputTokensDetails) { + this.outputTokensDetails = outputTokensDetails; + } + + public int getCachedTokens() { + return this.inputTokensDetails != null ? this.inputTokensDetails.getCachedTokens() : 0; + } + + public int getReasoningTokens() { + return this.outputTokensDetails != null ? this.outputTokensDetails.getReasoningTokens() : 0; + } + @Override public String toString() { - return "Usage [completionTokens=" + completionTokens + ", promptTokens=" + promptTokens + ", totalTokens=" + totalTokens + "]"; + return "Usage [completionTokens=" + this.completionTokens + ", promptTokens=" + this.promptTokens + ", totalTokens=" + this.totalTokens + ", inputTokensDetails=" + this.inputTokensDetails + ", outputTokensDetails=" + this.outputTokensDetails + "]"; + } + + public static final class InputTokensDetails { + + private int cachedTokens; + + @JsonProperty("cached_tokens") + public int getCachedTokens() { + return this.cachedTokens; + } + + public void setCachedTokens(final int cachedTokens) { + this.cachedTokens = cachedTokens; + } + + @Override + public String toString() { + return "InputTokensDetails [cachedTokens=" + this.cachedTokens + "]"; + } + } + + public static final class OutputTokensDetails { + + private int reasoningTokens; + + @JsonProperty("reasoning_tokens") + public int getReasoningTokens() { + return this.reasoningTokens; + } + + public void setReasoningTokens(final int reasoningTokens) { + this.reasoningTokens = reasoningTokens; + } + + @Override + public String toString() { + return "OutputTokensDetails [reasoningTokens=" + this.reasoningTokens + "]"; + } } } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index 08cb869..c054f7f 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -52,6 +52,8 @@ public class ChatCompletionClient extends AbstractCompletionClient { request.setStream(chunkConsumer != null); if (chunkConsumer != null) { this.configureStreamOptions(true, request::setStreamOptions); + } else { + request.setUsage(true); } if (modelProfile.reasoningEffort() != null && !modelProfile.reasoningEffort().isBlank()) { request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning()); @@ -86,10 +88,14 @@ public class ChatCompletionClient extends AbstractCompletionClient { final LocalDateTime start = LocalDateTime.now(); final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); - final var response = webClient + var requestSpec = webClient .method(HttpMethod.POST) .uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build()) - .header("Content-Type", "application/json") + .header("Content-Type", "application/json"); + if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { + requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); + } + final var response = requestSpec .bodyValue(requestBody) .exchangeToMono(clientResponse -> { if (clientResponse.statusCode().isError()) { @@ -123,6 +129,9 @@ public class ChatCompletionClient extends AbstractCompletionClient { if (streamResponse.getUsage() != null) { chatCompletionResponse.setUsage(streamResponse.getUsage()); } + if (streamResponse.getTimings() != null) { + chatCompletionResponse.setTimings(streamResponse.getTimings()); + } if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) { if (streamResponse.getChoices().getFirst().getFinishReason() != null) { chatCompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason()); @@ -165,7 +174,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { final var content = response.getChoices().getFirst().getMessage().getContent(); final var reasoning = response.getChoices().getFirst().getMessage().getReasoningContent(); final var finishReason = this.mapFinishReason(choice.getFinishReason()); - final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content); + final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), response.getTimings(), null, content, reasoning); return new Completion(content, reasoning, finishReason, statistics); } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 31d4e32..35824fb 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -57,6 +57,8 @@ public class TextCompletionClient extends AbstractCompletionClient { request.setStream(chunkConsumer != null); if (chunkConsumer != null) { this.configureStreamOptions(true, request::setStreamOptions); + } else { + request.setUsage(true); } String requestBody; @@ -70,10 +72,14 @@ public class TextCompletionClient extends AbstractCompletionClient { LOGGER.debug("Requesting completion at {}", modelProfile.url()); final LocalDateTime start = LocalDateTime.now(); final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); - final var response = webClient + var requestSpec = webClient .method(HttpMethod.POST) .uri(uriBuilder -> uriBuilder.path("/v1/completions").build()) - .header("Content-Type", "application/json") + .header("Content-Type", "application/json"); + if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { + requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); + } + final var response = requestSpec .bodyValue(requestBody) .exchangeToMono(clientResponse -> { if (clientResponse.statusCode().isError()) { @@ -112,6 +118,9 @@ public class TextCompletionClient extends AbstractCompletionClient { if (streamResponse.getUsage() != null) { CompletionResponse.setUsage(streamResponse.getUsage()); } + if (streamResponse.getTimings() != null) { + CompletionResponse.setTimings(streamResponse.getTimings()); + } if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) { if (streamResponse.getChoices().getFirst().getFinishReason() != null) { CompletionResponse.getChoices().getFirst().setFinishReason(streamResponse.getChoices().getFirst().getFinishReason()); @@ -172,7 +181,7 @@ public class TextCompletionClient extends AbstractCompletionClient { } } - final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent()); + final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), response.getTimings(), prompt, content, reasoning); return new Completion(content, reasoning, finishReason, statistics); } diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfile.java b/src/main/java/de/extio/lmlib/profile/ModelProfile.java index 784eb66..491d43c 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfile.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfile.java @@ -14,7 +14,9 @@ public record ModelProfile( String url, String apiKey, BigDecimal costPerInToken, + BigDecimal costPerCachedInToken, BigDecimal costPerOutToken, + BigDecimal costPerReasoningOutToken, String reasoningEffort, String reasoningSummaryDetails, String category) { diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java index 454ba08..459e698 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java @@ -76,11 +76,19 @@ public class ModelProfileService { .map(value -> new BigDecimal(value)) .orElse(BigDecimal.ZERO) .divide(new BigDecimal(1000000.0)); + final var costPerCachedInToken = Optional.ofNullable(resource.getProperty("cost1MCachedInTokens")) + .map(Object::toString) + .map(value -> new BigDecimal(value).divide(new BigDecimal(1000000.0))) + .orElse(costPerInToken); final var costPerOutToken = Optional.ofNullable(resource.getProperty("cost1MOutTokens")) .map(Object::toString) .map(value -> new BigDecimal(value)) .orElse(BigDecimal.ZERO) .divide(new BigDecimal(1000000.0)); + final var costPerReasoningOutToken = Optional.ofNullable(resource.getProperty("cost1MReasoningOutTokens")) + .map(Object::toString) + .map(value -> new BigDecimal(value).divide(new BigDecimal(1000000.0))) + .orElse(BigDecimal.ZERO); final var reasoningEffort = Optional.ofNullable(resource.getProperty("reasoningEffort")) .map(Object::toString) .orElse(null); @@ -103,7 +111,7 @@ public class ModelProfileService { .orElse(null); } - return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerOutToken, reasoningEffort, reasoningSummaryDetails, category); + return new ModelProfile(promptTemplate, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category); } catch (final IOException e) { LOGGER.error("Error while reading model profile", e); diff --git a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java index 0949e44..d6ce451 100755 --- a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java +++ b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java @@ -28,7 +28,7 @@ import de.extio.lmlib.client.oai.completion.text.TextCompletionClient; import de.extio.lmlib.grader.Grader; import de.extio.lmlib.profile.ModelCategory; -@Disabled("This test requires a running Llama server") +// @Disabled("This test requires a running Llama server") @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @ComponentScan(basePackages = "de.extio.lmlib") diff --git a/src/test/resources/llama3.3-70b-azure.properties b/src/test/resources/llama3.3-70b-azure.properties index 1a82041..38de7d8 100644 --- a/src/test/resources/llama3.3-70b-azure.properties +++ b/src/test/resources/llama3.3-70b-azure.properties @@ -10,4 +10,6 @@ modelName=Llama-3.3-70B-Instruct url=XXX apiKey=XXX cost1MInTokens=0.71 +cost1MCachedInTokens=0.71 cost1MOutTokens=0.71 +cost1MReasoningOutTokens=0 diff --git a/src/test/resources/local-chatcompletion.properties b/src/test/resources/local-chatcompletion.properties index 0efbc09..7d9078a 100755 --- a/src/test/resources/local-chatcompletion.properties +++ b/src/test/resources/local-chatcompletion.properties @@ -10,6 +10,8 @@ modelName= url=http://localhost:5001 apiKey= cost1MInTokens=0 +cost1MCachedInTokens=0 cost1MOutTokens=0 +cost1MReasoningOutTokens=0 reasoningEffort=medium reasoningSummaryDetails=auto \ No newline at end of file diff --git a/src/test/resources/local-completion.properties b/src/test/resources/local-completion.properties index f1256e4..ce18a8a 100755 --- a/src/test/resources/local-completion.properties +++ b/src/test/resources/local-completion.properties @@ -10,4 +10,6 @@ modelName= url=http://localhost:5001 apiKey= cost1MInTokens=0 +cost1MCachedInTokens=0 cost1MOutTokens=0 +cost1MReasoningOutTokens=0