exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement skipCache functionality across conversation handling

Browse Source
This commit is contained in:
extio-de
2026-03-14 14:27:39 +01:00
parent c305740e10
commit 1e5adefd13
13 changed files with 131 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/de/extio/lmlib/client/Client.java
View File

@@ -8,19 +8,13 @@ import de.extio.lmlib.profile.ModelProfile.ModelProvider;
public interface Client {
/**
* @deprecated use {@link #conversation(ModelCategory, Conversation)} instead. Conversation can be converted from a system and text by Conversation.create(system, text)
*/
@Deprecated
Completion completion(ModelCategory modelCategory, String system, String text);
Completion conversation(ModelCategory modelCategory, Conversation conversation, boolean skipCache);
Completion conversation(ModelCategory modelCategory, Conversation conversation);
Completion conversation(ModelProfile modelProfile, Conversation conversation);
Completion conversation(ModelProfile modelProfile, Conversation conversation, boolean skipCache);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer);
Completion streamConversation(ModelCategory modelCategory, Conversation conversation, Consumer<Chunk> chunkConsumer, boolean skipCache);
Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer<Chunk> chunkConsumer);
Completion streamConversation(ModelProfile modelProfile, Conversation conversation, Consumer<Chunk> chunkConsumer, boolean skipCache);
ModelProvider getModelProvider();
}

23
src/main/java/de/extio/lmlib/client/ClientService.java
View File

@@ -57,29 +57,24 @@ public class ClientService {
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.client.completion(modelCategory, system, text);
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_, final boolean skipCache) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.conversation(modelProfile, conversation_, skipCache);
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_) {
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.conversation(modelProfile, conversation_);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation_, chunkConsumer);
return this.streamConversation(modelProfile, conversation_, chunkConsumer, skipCache);
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_) {
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation_, final boolean skipCache) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.conversation(modelProfile, conversation);
var completion = this.client.conversation(modelProfile, conversation, skipCache);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelProfile, conversation, completion);
}
@@ -87,12 +82,12 @@ public class ClientService {
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer<Chunk> chunkConsumer) {
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation_, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
Conversation conversation = conversation_;
for (final var interceptor : this.completionInterceptors) {
conversation = interceptor.before(modelProfile, conversation);
}
var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer);
var completion = this.client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache);
for (final var interceptor : this.completionInterceptors) {
completion = interceptor.after(modelProfile, conversation, completion);
}

26
src/main/java/de/extio/lmlib/client/cached/CachedClient.java
View File

@@ -6,7 +6,6 @@ import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.HexFormat;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.function.Supplier;
@@ -36,23 +35,18 @@ public class CachedClient implements Client {
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.getCachedResponse(modelCategory, null, () -> this.client.completion(modelCategory, system, text), system, text);
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final boolean skipCache) {
return this.getCachedResponse(modelCategory, null, skipCache, () -> this.client.conversation(modelCategory, conversation, skipCache), conversation.getConversation().toString());
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.getCachedResponse(modelCategory, null, () -> this.client.conversation(modelCategory, conversation), conversation.getConversation().toString());
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.getCachedResponse(null, modelProfile, () -> this.client.conversation(modelProfile, conversation), conversation.getConversation().toString());
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation, final boolean skipCache) {
return this.getCachedResponse(null, modelProfile, skipCache, () -> this.client.conversation(modelProfile, conversation, skipCache), conversation.getConversation().toString());
}
@Override
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(modelCategory, null, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer), conversation.getConversation().toString());
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final var response = this.getCachedResponse(modelCategory, null, skipCache, () -> this.client.streamConversation(modelCategory, conversation, chunkConsumer, skipCache), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
@@ -60,8 +54,8 @@ public class CachedClient implements Client {
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
final var response = this.getCachedResponse(null, modelProfile, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer), conversation.getConversation().toString());
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final var response = this.getCachedResponse(null, modelProfile, skipCache, () -> this.client.streamConversation(modelProfile, conversation, chunkConsumer, skipCache), conversation.getConversation().toString());
if (response.statistics().cached()) {
chunkConsumer.accept(new Chunk(response.response(), response.reasoning()));
}
@@ -73,10 +67,10 @@ public class CachedClient implements Client {
return this.client.getModelProvider();
}
private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final Supplier<Completion> supplier, final String... keys) {
private Completion getCachedResponse(final ModelCategory modelCategory, final ModelProfile modelProfile, final boolean skipCache, final Supplier<Completion> supplier, final String... keys) {
final var key = this.createCacheKey(modelCategory, modelProfile, keys);
final var cachedCompletion = this.cacheRepository.get(key);
final var cachedCompletion = skipCache ? null : this.cacheRepository.get(key);
if (cachedCompletion != null) {
return new Completion(cachedCompletion.response(), cachedCompletion.reasoning(), CompletionFinishReason.DONE, new CompletionStatistics(0, Duration.ZERO, cachedCompletion.inTokens(), cachedCompletion.cachedInTokens(), cachedCompletion.outTokens(), cachedCompletion.reasoningOutTokens(), BigDecimal.ZERO, true));
}

19
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
View File

@@ -82,29 +82,24 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.conversation(modelCategory, Conversation.create(system, text));
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final boolean skipCache) {
return this.streamConversation(modelCategory, conversation, null, skipCache);
}
@Override
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
return this.streamConversation(modelCategory, conversation, null);
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation, final boolean skipCache) {
return this.streamConversation(modelProfile, conversation, null, skipCache);
}
@Override
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
return this.streamConversation(modelProfile, conversation, null);
}
@Override
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
public Completion streamConversation(final ModelCategory modelCategory_, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
final ModelCategory modelCategory = (modelCategory_ == null) ? ModelCategory.MEDIUM : modelCategory_;
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
return this.streamConversation(modelProfile, conversation, chunkConsumer);
return this.streamConversation(modelProfile, conversation, chunkConsumer, skipCache);
}
@Override
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer, final boolean skipCache) {
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("Invalid ModelProfile");
}