exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement skipCache functionality across conversation handling

Browse Source
This commit is contained in:
extio-de
2026-03-14 14:27:39 +01:00
parent c305740e10
commit 1e5adefd13
13 changed files with 131 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/de/extio/lmlib/agent/AgentContext.java
View File

@@ -25,6 +25,8 @@ public final class AgentContext {
private volatile boolean error;
private volatile boolean skipNextCompletion;
private volatile boolean skipCache;
private volatile Conversation conversation;
@@ -74,6 +76,7 @@ public final class AgentContext {
this.agentContextUpdateConsumer = other.agentContextUpdateConsumer;
this.error = other.error;
this.skipNextCompletion = other.skipNextCompletion;
this.skipCache = other.skipCache;
this.streaming = other.streaming;
synchronized (other.graph) {
this.graph.addAll(other.graph);
@@ -201,6 +204,14 @@ public final class AgentContext {
public void setSkipNextCompletion(final boolean skipNextCompletion) {
this.skipNextCompletion = skipNextCompletion;
}
public boolean isSkipCache() {
return this.skipCache;
}
public void setSkipCache(final boolean skipCache) {
this.skipCache = skipCache;
}
Consumer<AgentContext> getAgentContextUpdateConsumer() {
return agentContextUpdateConsumer;

10
src/main/java/de/extio/lmlib/agent/BaseAgent.java
View File

@@ -103,6 +103,8 @@ public interface BaseAgent {
split.context().getGraph().add(this.name());
if (!skipCompletion) {
final boolean skipCache = split.context().isSkipCache();
split.context().setSkipCache(false);
final var conversation = this.setupConversation(split);
boolean parseable = false;
@@ -126,21 +128,21 @@ public interface BaseAgent {
};
if (modelProfile != null) {
requestStart = System.currentTimeMillis();
completion = client.streamConversation(modelProfile, conversation, chunkConsumer);
completion = client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache);
}
else {
requestStart = System.currentTimeMillis();
completion = client.streamConversation(this.modelCategory(split.context()), conversation, chunkConsumer);
completion = client.streamConversation(this.modelCategory(split.context()), conversation, chunkConsumer, skipCache);
}
}
else {
if (modelProfile != null) {
requestStart = System.currentTimeMillis();
completion = client.conversation(modelProfile, conversation);
completion = client.conversation(modelProfile, conversation, skipCache);
}
else {
requestStart = System.currentTimeMillis();
completion = client.conversation(this.modelCategory(split.context()), conversation);
completion = client.conversation(this.modelCategory(split.context()), conversation, skipCache);
}
}
final var durationMs = System.currentTimeMillis() - requestStart;

14
src/main/java/de/extio/lmlib/client/Client.java
View File

@@ -8,19 +8,13 @@ import de.extio.lmlib.profile.ModelProfile.ModelProvider;
public interface Client {
/**
* @deprecated use {@link #conversation(ModelCategory, Conversation)} instead. Conversation can be converted from a system and text by Conversation.create(system, text)
*/
@Deprecated
Completion completion(ModelCategory modelCategory, String system, String text);
Completion conversation(ModelCategory modelCategory, Conversation conversation, boolean skipCache);
Completion conversation(ModelCategory modelCategory, Conversation conversation);
Completion conversation(ModelProfile modelProfile, Conversation conversation);
Completion conversation(ModelProfile modelProfile, Conversation conversation, boolean skipCache);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer, boolean skipCache);
Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer<Chunk> chunkConsumer);
Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer<Chunk> chunkConsumer, boolean skipCache);
ModelProvider getModelProvider();
}

23
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -57,29 +57,24 @@ public class ClientService {
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.client.completion(modelCategory, system, text);
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_, final boolean skipCache) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.conversation(modelProfile, conversation_, skipCache);
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_) {
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.conversation(modelProfile, conversation_);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation_, chunkConsumer);
return this.streamConversation(modelProfile, conversation_, chunkConsumer, skipCache);
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_) {
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_, final boolean skipCache) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.conversation(modelProfile, conversation);
var completion = this.client.conversation(modelProfile, conversation, skipCache);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelProfile, conversation, completion);
}
@@ -87,12 +82,12 @@ public class ClientService {
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer);
var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelProfile, conversation, completion);
}

26
src/main/java/de/extio/lmlib/client/cached/CachedClient.java
View File

@@ -6,7 +6,6 @@ import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.HexFormat;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.function.Supplier;
@@ -36,23 +35,18 @@ public class CachedClient implements Client {
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.getCachedResponse(modelCategory, null, () -> this.client.completion(modelCategory, system, text), system, text);
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final boolean skipCache) {
return this.getCachedResponse(modelCategory, null, skipCache, () -> this.client.conversation(modelCategory, conversation, skipCache), conversation.getConversation().toString());
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.getCachedResponse(modelCategory, null, () -> this.client.conversation(modelCategory, conversation), conversation.getConversation().toString());
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.getCachedResponse(null, modelProfile, () -> this.client.conversation(modelProfile, conversation), conversation.getConversation().toString());
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation, final boolean skipCache) {
return this.getCachedResponse(null, modelProfile, skipCache, () -> this.client.conversation(modelProfile, conversation, skipCache), conversation.getConversation().toString());
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(modelCategory, null, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final var response = this.getCachedResponse(modelCategory, null, skipCache, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer, skipCache), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
@@ -60,8 +54,8 @@ public class CachedClient implements Client {
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(null, modelProfile, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer), conversation.getConversation().toString());
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final var response = this.getCachedResponse(null, modelProfile, skipCache, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
@@ -73,10 +67,10 @@ public class CachedClient implements Client {
return this.client.getModelProvider();
}
private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final Supplier<Completion> supplier, final String... keys) {
private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final boolean skipCache, final Supplier<Completion> supplier, final String... keys) {
final var key = this.createCacheKey(modelCategory, modelProfile, keys);
final var cachedCompletion = this.cacheRepository.get(key);
final var cachedCompletion = skipCache ? null : this.cacheRepository.get(key);
if (cachedCompletion != null) {
return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.cachedInTokens(), cachedCompletion.outTokens(), cachedCompletion.reasoningOutTokens(), BigDecimal.ZERO, true));
}

19
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
View File

@@ -82,29 +82,24 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.conversation(modelCategory, Conversation.create(system, text));
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final boolean skipCache) {
return this.streamConversation(modelCategory, conversation, null, skipCache);
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.streamConversation(modelCategory, conversation, null);
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation, final boolean skipCache) {
return this.streamConversation(modelProfile, conversation, null, skipCache);
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.streamConversation(modelProfile, conversation, null);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation, chunkConsumer);
return this.streamConversation(modelProfile, conversation, chunkConsumer, skipCache);
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("Invalid ModelProfile");
}

19
src/main/java/de/extio/lmlib/grader/Grader.java
View File

@@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.profile.ModelCategory;
/**
@@ -25,15 +26,17 @@ public class Grader {
var score = 0;
while (Math.abs(score) < 3) {
final var completion = client.completion(
final var completion = client.conversation(
ModelCategory.MEDIUM,
"""
You are a grader assessing the truthfulness of a given text to a user question.
Please provide a binary response 'true' or 'false' for the following text.
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
Only provide the response as a single word and no preamble and no explanation.""",
"Here is the text:\n" + text + "\n\n" +
"Here is the user question: " + question);
Conversation.create(
"""
You are a grader assessing the truthfulness of a given text to a user question.
Please provide a binary response 'true' or 'false' for the following text.
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
Only provide the response as a single word and no preamble and no explanation.""",
"Here is the text:\n" + text + "\n\n" +
"Here is the user question: " + question),
true);
LOGGER.info(completion.response());
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
if (Math.abs(score) == 2) {

3
src/main/java/de/extio/lmlib/grader/Grader2.java
View File

@@ -49,7 +49,8 @@ public class Grader2 {
- 'true': The text provides a truthful answer.
- 'false': The text does not provide a truthful answer.
Only provide the response as a single word. Do not include any preamble or explanation.
""".formatted(question, text)));
""".formatted(question, text)),
true);
LOGGER.info(completion.response());
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
if (Math.abs(score) == 2) {

4
src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java
View File

@@ -31,7 +31,9 @@ public class AnswerGrader {
The answer is: {{answer}}
""",
new JsonAgentResponseHandler(),
null,
context -> {
context.setSkipCache(true);
},
context -> {
if (!context.getContext().containsKey("qaRating")) {
context.setStringValue("qaRating", "0");

3
src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java
View File

@@ -52,7 +52,8 @@ public class AnswerGrader2 {
context.setStringValue("qaOutputFormat", "Provide no preamble. Return the response in JSON format with the following field only: {\"qaRating\": rating}");
}
context.getContext().remove("qaRating");
context.getContext().remove("qaExplanation");
context.getContext().remove("qaExplanation");
context.setSkipCache(true);
},
context -> {
if (!context.getContext().containsKey("qaRating")) {

1
src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java
View File

@@ -44,6 +44,7 @@ public class AnswerGraderBinary {
}
context.getContext().remove("qaPassed");
context.getContext().remove("qaExplanation");
context.setSkipCache(true);
},
context -> {
if (!context.getContext().containsKey("qaPassed")) {