From 1e5adefd137cb3937eb561b9ee7374706f75f873 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 14 Mar 2026 14:27:39 +0100 Subject: [PATCH] feat: Implement skipCache functionality across conversation handling --- .../de/extio/lmlib/agent/AgentContext.java | 11 ++++ .../java/de/extio/lmlib/agent/BaseAgent.java | 10 ++-- .../java/de/extio/lmlib/client/Client.java | 14 ++--- .../de/extio/lmlib/client/ClientService.java | 23 ++++---- .../lmlib/client/cached/CachedClient.java | 26 ++++------ .../completion/AbstractCompletionClient.java | 19 +++---- .../java/de/extio/lmlib/grader/Grader.java | 19 ++++--- .../java/de/extio/lmlib/grader/Grader2.java | 3 +- .../lmlib/grader/agent/AnswerGrader.java | 4 +- .../lmlib/grader/agent/AnswerGrader2.java | 3 +- .../grader/agent/AnswerGraderBinary.java | 1 + .../lmlib/client/cached/CachedClientTest.java | 52 ++++++++++++++----- .../client/oai/CompletionClientTest.java | 52 ++++++++++--------- 13 files changed, 131 insertions(+), 106 deletions(-) diff --git a/src/main/java/de/extio/lmlib/agent/AgentContext.java b/src/main/java/de/extio/lmlib/agent/AgentContext.java index 497eea0..a2e4a1b 100755 --- a/src/main/java/de/extio/lmlib/agent/AgentContext.java +++ b/src/main/java/de/extio/lmlib/agent/AgentContext.java @@ -25,6 +25,8 @@ public final class AgentContext { private volatile boolean error; private volatile boolean skipNextCompletion; + + private volatile boolean skipCache; private volatile Conversation conversation; @@ -74,6 +76,7 @@ public final class AgentContext { this.agentContextUpdateConsumer = other.agentContextUpdateConsumer; this.error = other.error; this.skipNextCompletion = other.skipNextCompletion; + this.skipCache = other.skipCache; this.streaming = other.streaming; synchronized (other.graph) { this.graph.addAll(other.graph); @@ -201,6 +204,14 @@ public final class AgentContext { public void setSkipNextCompletion(final boolean skipNextCompletion) { this.skipNextCompletion = skipNextCompletion; } + + public boolean isSkipCache() { + return this.skipCache; + } + + public void setSkipCache(final boolean skipCache) { + this.skipCache = skipCache; + } Consumer getAgentContextUpdateConsumer() { return agentContextUpdateConsumer; diff --git a/src/main/java/de/extio/lmlib/agent/BaseAgent.java b/src/main/java/de/extio/lmlib/agent/BaseAgent.java index 4f1bdfe..9f73b39 100644 --- a/src/main/java/de/extio/lmlib/agent/BaseAgent.java +++ b/src/main/java/de/extio/lmlib/agent/BaseAgent.java @@ -103,6 +103,8 @@ public interface BaseAgent { split.context().getGraph().add(this.name()); if (!skipCompletion) { + final boolean skipCache = split.context().isSkipCache(); + split.context().setSkipCache(false); final var conversation = this.setupConversation(split); boolean parseable = false; @@ -126,21 +128,21 @@ public interface BaseAgent { }; if (modelProfile != null) { requestStart = System.currentTimeMillis(); - completion = client.streamConversation(modelProfile, conversation, chunkConsumer); + completion = client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache); } else { requestStart = System.currentTimeMillis(); - completion = client.streamConversation(this.modelCategory(split.context()), conversation, chunkConsumer); + completion = client.streamConversation(this.modelCategory(split.context()), conversation, chunkConsumer, skipCache); } } else { if (modelProfile != null) { requestStart = System.currentTimeMillis(); - completion = client.conversation(modelProfile, conversation); + completion = client.conversation(modelProfile, conversation, skipCache); } else { requestStart = System.currentTimeMillis(); - completion = client.conversation(this.modelCategory(split.context()), conversation); + completion = client.conversation(this.modelCategory(split.context()), conversation, skipCache); } } final var durationMs = System.currentTimeMillis() - requestStart; diff --git a/src/main/java/de/extio/lmlib/client/Client.java b/src/main/java/de/extio/lmlib/client/Client.java index 84d1a93..9014584 100755 --- a/src/main/java/de/extio/lmlib/client/Client.java +++ b/src/main/java/de/extio/lmlib/client/Client.java @@ -8,19 +8,13 @@ import de.extio.lmlib.profile.ModelProfile.ModelProvider; public interface Client { - /** - * @deprecated use {@link #conversation(ModelCategory, Conversation)} instead. Conversation can be converted from a system and text by Conversation.create(system, text) - */ - @Deprecated - Completion completion(ModelCategory modelCategory, String system, String text); + Completion conversation(ModelCategory modelCategory, Conversation conversation, boolean skipCache); - Completion conversation(ModelCategory modelCategory, Conversation conversation); - - Completion conversation(ModelProfile modelProfile, Conversation conversation); + Completion conversation(ModelProfile modelProfile, Conversation conversation, boolean skipCache); - Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer chunkConsumer); + Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer chunkConsumer, boolean skipCache); - Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer chunkConsumer); + Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer chunkConsumer, boolean skipCache); ModelProvider getModelProvider(); } diff --git a/src/main/java/de/extio/lmlib/client/ClientService.java b/src/main/java/de/extio/lmlib/client/ClientService.java index d134f0d..2b3704d 100755 --- a/src/main/java/de/extio/lmlib/client/ClientService.java +++ b/src/main/java/de/extio/lmlib/client/ClientService.java @@ -57,29 +57,24 @@ public class ClientService { } @Override - public Completion completion(final ModelCategory modelCategory, final String system, final String text) { - return this.client.completion(modelCategory, system, text); + public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_, final boolean skipCache) { + final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory); + return this.conversation(modelProfile, conversation_, skipCache); } @Override - public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_) { + public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer chunkConsumer, final boolean skipCache) { final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory); - return this.conversation(modelProfile, conversation_); - } - - @Override - public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer chunkConsumer) { - final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory); - return this.streamConversation(modelProfile, conversation_, chunkConsumer); + return this.streamConversation(modelProfile, conversation_, chunkConsumer, skipCache); } @Override - public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_) { + public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_, final boolean skipCache) { Conversation conversation = conversation_; for (final var interceptor : this.completionInterceptors) { conversation = interceptor.before(modelProfile, conversation); } - var completion = this.client.conversation(modelProfile, conversation); + var completion = this.client.conversation(modelProfile, conversation, skipCache); for (final var interceptor : this.completionInterceptors) { completion = interceptor.after(modelProfile, conversation, completion); } @@ -87,12 +82,12 @@ public class ClientService { } @Override - public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer chunkConsumer) { + public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer chunkConsumer, final boolean skipCache) { Conversation conversation = conversation_; for (final var interceptor : this.completionInterceptors) { conversation = interceptor.before(modelProfile, conversation); } - var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer); + var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache); for (final var interceptor : this.completionInterceptors) { completion = interceptor.after(modelProfile, conversation, completion); } diff --git a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java index 446ce16..8e505fe 100644 --- a/src/main/java/de/extio/lmlib/client/cached/CachedClient.java +++ b/src/main/java/de/extio/lmlib/client/cached/CachedClient.java @@ -6,7 +6,6 @@ import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.OffsetDateTime; import java.util.HexFormat; -import java.util.Objects; import java.util.function.Consumer; import java.util.function.Supplier; @@ -36,23 +35,18 @@ public class CachedClient implements Client { } @Override - public Completion completion(final ModelCategory modelCategory, final String system, final String text) { - return this.getCachedResponse(modelCategory, null, () -> this.client.completion(modelCategory, system, text), system, text); + public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final boolean skipCache) { + return this.getCachedResponse(modelCategory, null, skipCache, () -> this.client.conversation(modelCategory, conversation, skipCache), conversation.getConversation().toString()); } @Override - public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) { - return this.getCachedResponse(modelCategory, null, () -> this.client.conversation(modelCategory, conversation), conversation.getConversation().toString()); - } - - @Override - public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) { - return this.getCachedResponse(null, modelProfile, () -> this.client.conversation(modelProfile, conversation), conversation.getConversation().toString()); + public Completion conversation(final ModelProfile modelProfile, final Conversation conversation, final boolean skipCache) { + return this.getCachedResponse(null, modelProfile, skipCache, () -> this.client.conversation(modelProfile, conversation, skipCache), conversation.getConversation().toString()); } @Override - public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer chunkConsumer) { - final var response = this.getCachedResponse(modelCategory, null, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString()); + public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer chunkConsumer, final boolean skipCache) { + final var response = this.getCachedResponse(modelCategory, null, skipCache, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer, skipCache), conversation.getConversation().toString()); if (response.statistics().cached()) { chunkConsumer.accept(new Chunk(response.response(), response.reasoning())); } @@ -60,8 +54,8 @@ public class CachedClient implements Client { } @Override - public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer chunkConsumer) { - final var response = this.getCachedResponse(null, modelProfile, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer), conversation.getConversation().toString()); + public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer chunkConsumer, final boolean skipCache) { + final var response = this.getCachedResponse(null, modelProfile, skipCache, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache), conversation.getConversation().toString()); if (response.statistics().cached()) { chunkConsumer.accept(new Chunk(response.response(), response.reasoning())); } @@ -73,10 +67,10 @@ public class CachedClient implements Client { return this.client.getModelProvider(); } - private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final Supplier supplier, final String... keys) { + private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final boolean skipCache, final Supplier supplier, final String... keys) { final var key = this.createCacheKey(modelCategory, modelProfile, keys); - final var cachedCompletion = this.cacheRepository.get(key); + final var cachedCompletion = skipCache ? null : this.cacheRepository.get(key); if (cachedCompletion != null) { return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.cachedInTokens(), cachedCompletion.outTokens(), cachedCompletion.reasoningOutTokens(), BigDecimal.ZERO, true)); } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index 373856e..05737fa 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -82,29 +82,24 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean } @Override - public Completion completion(final ModelCategory modelCategory, final String system, final String text) { - return this.conversation(modelCategory, Conversation.create(system, text)); + public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final boolean skipCache) { + return this.streamConversation(modelCategory, conversation, null, skipCache); } @Override - public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) { - return this.streamConversation(modelCategory, conversation, null); + public Completion conversation(final ModelProfile modelProfile, final Conversation conversation, final boolean skipCache) { + return this.streamConversation(modelProfile, conversation, null, skipCache); } @Override - public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) { - return this.streamConversation(modelProfile, conversation, null); - } - - @Override - public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer chunkConsumer) { + public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer chunkConsumer, final boolean skipCache) { final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_; final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory); - return this.streamConversation(modelProfile, conversation, chunkConsumer); + return this.streamConversation(modelProfile, conversation, chunkConsumer, skipCache); } @Override - public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer chunkConsumer) { + public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer chunkConsumer, final boolean skipCache) { if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) { throw new IllegalArgumentException("Invalid ModelProfile"); } diff --git a/src/main/java/de/extio/lmlib/grader/Grader.java b/src/main/java/de/extio/lmlib/grader/Grader.java index 3ef8489..93d5811 100644 --- a/src/main/java/de/extio/lmlib/grader/Grader.java +++ b/src/main/java/de/extio/lmlib/grader/Grader.java @@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory; import de.extio.lmlib.client.Client; import de.extio.lmlib.client.ClientService; +import de.extio.lmlib.client.Conversation; import de.extio.lmlib.profile.ModelCategory; /** @@ -25,15 +26,17 @@ public class Grader { var score = 0; while (Math.abs(score) < 3) { - final var completion = client.completion( + final var completion = client.conversation( ModelCategory.MEDIUM, - """ - You are a grader assessing the truthfulness of a given text to a user question. - Please provide a binary response 'true' or 'false' for the following text. - 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not. - Only provide the response as a single word and no preamble and no explanation.""", - "Here is the text:\n" + text + "\n\n" + - "Here is the user question: " + question); + Conversation.create( + """ + You are a grader assessing the truthfulness of a given text to a user question. + Please provide a binary response 'true' or 'false' for the following text. + 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not. + Only provide the response as a single word and no preamble and no explanation.""", + "Here is the text:\n" + text + "\n\n" + + "Here is the user question: " + question), + true); LOGGER.info(completion.response()); score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1; if (Math.abs(score) == 2) { diff --git a/src/main/java/de/extio/lmlib/grader/Grader2.java b/src/main/java/de/extio/lmlib/grader/Grader2.java index 95cf54c..80b5284 100644 --- a/src/main/java/de/extio/lmlib/grader/Grader2.java +++ b/src/main/java/de/extio/lmlib/grader/Grader2.java @@ -49,7 +49,8 @@ public class Grader2 { - 'true': The text provides a truthful answer. - 'false': The text does not provide a truthful answer. Only provide the response as a single word. Do not include any preamble or explanation. - """.formatted(question, text))); + """.formatted(question, text)), + true); LOGGER.info(completion.response()); score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1; if (Math.abs(score) == 2) { diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java index c68319e..8126334 100755 --- a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java @@ -31,7 +31,9 @@ public class AnswerGrader { The answer is: {{answer}} """, new JsonAgentResponseHandler(), - null, + context -> { + context.setSkipCache(true); + }, context -> { if (!context.getContext().containsKey("qaRating")) { context.setStringValue("qaRating", "0"); diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java index aa2daeb..4c87b78 100755 --- a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java @@ -52,7 +52,8 @@ public class AnswerGrader2 { context.setStringValue("qaOutputFormat", "Provide no preamble. Return the response in JSON format with the following field only: {\"qaRating\": rating}"); } context.getContext().remove("qaRating"); - context.getContext().remove("qaExplanation"); + context.getContext().remove("qaExplanation"); + context.setSkipCache(true); }, context -> { if (!context.getContext().containsKey("qaRating")) { diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java index 1322938..3c3bc9c 100644 --- a/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java @@ -44,6 +44,7 @@ public class AnswerGraderBinary { } context.getContext().remove("qaPassed"); context.getContext().remove("qaExplanation"); + context.setSkipCache(true); }, context -> { if (!context.getContext().containsKey("qaPassed")) { diff --git a/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java index 542b3b1..36e510b 100755 --- a/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java +++ b/src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java @@ -47,29 +47,29 @@ public class CachedClientTest { @Test void cacheCompletion() throws Exception { - var completion = this.clientService.getClient(ModelCategory.SMALL).completion( + var completion = this.clientService.getClient(ModelCategory.SMALL).conversation( ModelCategory.SMALL, - "You are a funny person", - "Tell me a cat joke"); + Conversation.create("You are a funny person", "Tell me a cat joke"), + false); LOGGER.info(completion.toString()); assertFalse(completion.statistics().cached()); assertEquals(1, completion.statistics().requests()); final var firstResponse = completion.response(); assertFalse(firstResponse.isBlank()); - completion = this.clientService.getClient(ModelCategory.SMALL).completion( + completion = this.clientService.getClient(ModelCategory.SMALL).conversation( ModelCategory.SMALL, - "You are a funny person", - "Tell me a cat joke"); + Conversation.create("You are a funny person", "Tell me a cat joke"), + false); LOGGER.info(completion.toString()); assertTrue(completion.statistics().cached()); assertEquals(0, completion.statistics().requests()); assertEquals(firstResponse, completion.response()); - completion = this.clientService.getClient(ModelCategory.SMALL).completion( + completion = this.clientService.getClient(ModelCategory.SMALL).conversation( ModelCategory.SMALL, - "You are a funny person", - "Tell me a dog joke"); + Conversation.create("You are a funny person", "Tell me a dog joke"), + false); LOGGER.info(completion.toString()); assertFalse(completion.statistics().cached()); assertEquals(1, completion.statistics().requests()); @@ -80,14 +80,14 @@ public class CachedClientTest { void cacheChat() throws Exception { final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Munich."); - var completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation); + var completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false); LOGGER.info(completion.toString()); assertFalse(completion.statistics().cached()); assertEquals(1, completion.statistics().requests()); final var firstResponse = completion.response(); assertFalse(firstResponse.isBlank()); - completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation); + completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false); LOGGER.info(completion.toString()); assertTrue(completion.statistics().cached()); assertEquals(0, completion.statistics().requests()); @@ -96,13 +96,37 @@ public class CachedClientTest { conversation.addTurn(new Turn(TurnType.ASSISTANT, "T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius.")); conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?")); - completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation); + completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false); LOGGER.info(completion.toString()); assertFalse(completion.statistics().cached()); assertEquals(1, completion.statistics().requests()); assertNotEquals(firstResponse, completion.response()); } + @Test + void bypassCacheStillStoresFreshResult() throws Exception { + final var conversation = Conversation.create("You are a witty assistant", "Give me a short joke about caches."); + + final var initialCompletion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false); + LOGGER.info(initialCompletion.toString()); + assertFalse(initialCompletion.statistics().cached()); + assertEquals(1, initialCompletion.statistics().requests()); + assertFalse(initialCompletion.response().isBlank()); + + final var bypassedCompletion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, true); + LOGGER.info(bypassedCompletion.toString()); + assertFalse(bypassedCompletion.statistics().cached()); + assertEquals(1, bypassedCompletion.statistics().requests()); + assertFalse(bypassedCompletion.response().isBlank()); + + final var cachedCompletion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false); + LOGGER.info(cachedCompletion.toString()); + assertTrue(cachedCompletion.statistics().cached()); + assertEquals(0, cachedCompletion.statistics().requests()); + assertEquals(bypassedCompletion.response(), cachedCompletion.response()); + assertEquals(bypassedCompletion.reasoning(), cachedCompletion.reasoning()); + } + @Test void cacheStream() throws Exception { final var conversation = Conversation.create("You are an assistant who talks extremely like a pirate", "Hi there, I would like to know the climate in the red sea."); @@ -116,7 +140,7 @@ public class CachedClientTest { sb.append(chunk.content()); System.out.print(chunk.content()); } - }); + }, false); assertFalse(completion.statistics().cached()); assertEquals(1, completion.statistics().requests()); assertTrue(!sb.isEmpty()); @@ -134,7 +158,7 @@ public class CachedClientTest { sb2.append(chunk.content()); System.out.print(chunk.content()); } - }); + }, false); assertTrue(completion.statistics().cached()); assertEquals(sb.toString(), sb2.toString()); assertEquals(sb2.toString(), completion.response()); diff --git a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java index a29f4b5..8717039 100755 --- a/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java +++ b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java @@ -27,6 +27,7 @@ import de.extio.lmlib.client.Conversation.TurnType; import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient; import de.extio.lmlib.client.oai.completion.text.TextCompletionClient; import de.extio.lmlib.grader.Grader2; +import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfileService; @Disabled("This test requires a running Llama server") @@ -55,7 +56,8 @@ public class CompletionClientTest { final var modelProfile = this.modelProfileService.getModelProfile("profile.model.textcompletion"); final var completion = this.textCompletionClient.conversation( modelProfile, - Conversation.create("You are a helpful assistant", "Say the color is green")); + Conversation.create("You are a helpful assistant", "Say the color is green"), + false); LOGGER.info(completion.reasoning()); LOGGER.info(completion.response()); @@ -69,7 +71,7 @@ public class CompletionClientTest { final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color."); conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?")); conversation.addTurn(new Turn(TurnType.USER, "You choose!")); - final var completion = this.textCompletionClient.conversation(modelProfile, conversation); + final var completion = this.textCompletionClient.conversation(modelProfile, conversation, false); LOGGER.info(completion.reasoning()); LOGGER.info(completion.response()); @@ -92,7 +94,7 @@ public class CompletionClientTest { System.out.print(chunk.content()); sb.append(chunk.content()); } - }); + }, false); if (completion.reasoning() != null) { // Prompt template does not separate reasoning and response from the stream at the moment @@ -112,7 +114,7 @@ public class CompletionClientTest { final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color."); conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?")); conversation.addTurn(new Turn(TurnType.USER, "You choose!")); - final var completion = this.chatCompletionclient.conversation(modelProfile, conversation); + final var completion = this.chatCompletionclient.conversation(modelProfile, conversation, false); LOGGER.info(completion.reasoning()); LOGGER.info(completion.response()); @@ -135,7 +137,7 @@ public class CompletionClientTest { System.out.print(chunk.content()); sb.append(chunk.content()); } - }); + }, false); assertEquals(completion.response(), sb.toString()); assertTrue(Grader2.assessScoreBinary("Does the text explain a chess opening?", completion.response(), modelProfile, this.clientService)); @@ -151,10 +153,10 @@ public class CompletionClientTest { } sb.delete(sb.length() - 2, sb.length()); - final var completion = this.textCompletionClient.completion( - null, - "You are counting machine", - "How many numbers do you count?\n" + sb.toString()); + final var completion = this.textCompletionClient.conversation( + ModelCategory.MEDIUM, + Conversation.create("You are counting machine", "How many numbers do you count?\n" + sb.toString()), + false); LOGGER.info(completion.response()); } @@ -199,10 +201,10 @@ public class CompletionClientTest { private Tasks createTasks() { return new Tasks(() -> { try { - final var completion = this.textCompletionClient.completion( - null, - "You are a helpful assistant", - "How do I calculate the annual profit margin?"); + final var completion = this.textCompletionClient.conversation( + ModelCategory.MEDIUM, + Conversation.create("You are a helpful assistant", "How do I calculate the annual profit margin?"), + false); LOGGER.info(completion.response()); } catch (final Exception e) { @@ -210,10 +212,10 @@ public class CompletionClientTest { } }, () -> { try { - final var completion = this.textCompletionClient.completion( - null, - "You are a helpful assistant", - "How do I calculate the operating profit ratio?"); + final var completion = this.textCompletionClient.conversation( + ModelCategory.MEDIUM, + Conversation.create("You are a helpful assistant", "How do I calculate the operating profit ratio?"), + false); LOGGER.info(completion.response()); } catch (final Exception e) { @@ -221,10 +223,10 @@ public class CompletionClientTest { } }, () -> { try { - final var completion = this.textCompletionClient.completion( - null, - "You are a helpful assistant", - "How do I calculate the return of investment?"); + final var completion = this.textCompletionClient.conversation( + ModelCategory.MEDIUM, + Conversation.create("You are a helpful assistant", "How do I calculate the return of investment?"), + false); LOGGER.info(completion.response()); } catch (final Exception e) { @@ -232,10 +234,10 @@ public class CompletionClientTest { } }, () -> { try { - final var completion = this.textCompletionClient.completion( - null, - "You are a helpful assistant", - "How do I calculate the return on net worth?"); + final var completion = this.textCompletionClient.conversation( + ModelCategory.MEDIUM, + Conversation.create("You are a helpful assistant", "How do I calculate the return on net worth?"), + false); LOGGER.info(completion.response()); } catch (final Exception e) {