exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement skipCache functionality across conversation handling

Browse Source
This commit is contained in:
extio-de
2026-03-14 14:27:39 +01:00
parent c305740e10
commit 1e5adefd13
13 changed files with 131 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
src/test/java/de/extio/lmlib/client/cached/CachedClientTest.java
View File

@@ -47,29 +47,29 @@ public class CachedClientTest {
@Test
void cacheCompletion() throws Exception {
var completion = this.clientService.getClient(ModelCategory.SMALL).completion(
var completion = this.clientService.getClient(ModelCategory.SMALL).conversation(
ModelCategory.SMALL,
"You are a funny person",
"Tell me a cat joke");
Conversation.create("You are a funny person", "Tell me a cat joke"),
false);
LOGGER.info(completion.toString());
assertFalse(completion.statistics().cached());
assertEquals(1, completion.statistics().requests());
final var firstResponse = completion.response();
assertFalse(firstResponse.isBlank());
completion = this.clientService.getClient(ModelCategory.SMALL).completion(
completion = this.clientService.getClient(ModelCategory.SMALL).conversation(
ModelCategory.SMALL,
"You are a funny person",
"Tell me a cat joke");
Conversation.create("You are a funny person", "Tell me a cat joke"),
false);
LOGGER.info(completion.toString());
assertTrue(completion.statistics().cached());
assertEquals(0, completion.statistics().requests());
assertEquals(firstResponse, completion.response());
completion = this.clientService.getClient(ModelCategory.SMALL).completion(
completion = this.clientService.getClient(ModelCategory.SMALL).conversation(
ModelCategory.SMALL,
"You are a funny person",
"Tell me a dog joke");
Conversation.create("You are a funny person", "Tell me a dog joke"),
false);
LOGGER.info(completion.toString());
assertFalse(completion.statistics().cached());
assertEquals(1, completion.statistics().requests());
@@ -80,14 +80,14 @@ public class CachedClientTest {
void cacheChat() throws Exception {
final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Munich.");
var completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
var completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false);
LOGGER.info(completion.toString());
assertFalse(completion.statistics().cached());
assertEquals(1, completion.statistics().requests());
final var firstResponse = completion.response();
assertFalse(firstResponse.isBlank());
completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false);
LOGGER.info(completion.toString());
assertTrue(completion.statistics().cached());
assertEquals(0, completion.statistics().requests());
@@ -96,13 +96,37 @@ public class CachedClientTest {
conversation.addTurn(new Turn(TurnType.ASSISTANT,
"T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius."));
conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?"));
completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation);
completion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false);
LOGGER.info(completion.toString());
assertFalse(completion.statistics().cached());
assertEquals(1, completion.statistics().requests());
assertNotEquals(firstResponse, completion.response());
}
@Test
void bypassCacheStillStoresFreshResult() throws Exception {
final var conversation = Conversation.create("You are a witty assistant", "Give me a short joke about caches.");
final var initialCompletion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false);
LOGGER.info(initialCompletion.toString());
assertFalse(initialCompletion.statistics().cached());
assertEquals(1, initialCompletion.statistics().requests());
assertFalse(initialCompletion.response().isBlank());
final var bypassedCompletion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, true);
LOGGER.info(bypassedCompletion.toString());
assertFalse(bypassedCompletion.statistics().cached());
assertEquals(1, bypassedCompletion.statistics().requests());
assertFalse(bypassedCompletion.response().isBlank());
final var cachedCompletion = this.clientService.getClient(ModelCategory.MEDIUM).conversation(ModelCategory.MEDIUM, conversation, false);
LOGGER.info(cachedCompletion.toString());
assertTrue(cachedCompletion.statistics().cached());
assertEquals(0, cachedCompletion.statistics().requests());
assertEquals(bypassedCompletion.response(), cachedCompletion.response());
assertEquals(bypassedCompletion.reasoning(), cachedCompletion.reasoning());
}
@Test
void cacheStream() throws Exception {
final var conversation = Conversation.create("You are an assistant who talks extremely like a pirate", "Hi there, I would like to know the climate in the red sea.");
@@ -116,7 +140,7 @@ public class CachedClientTest {
sb.append(chunk.content());
System.out.print(chunk.content());
}
});
}, false);
assertFalse(completion.statistics().cached());
assertEquals(1, completion.statistics().requests());
assertTrue(!sb.isEmpty());
@@ -134,7 +158,7 @@ public class CachedClientTest {
sb2.append(chunk.content());
System.out.print(chunk.content());
}
});
}, false);
assertTrue(completion.statistics().cached());
assertEquals(sb.toString(), sb2.toString());
assertEquals(sb2.toString(), completion.response());

52
src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java
View File

@@ -27,6 +27,7 @@ import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient;
import de.extio.lmlib.client.oai.completion.text.TextCompletionClient;
import de.extio.lmlib.grader.Grader2;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfileService;
@Disabled("This test requires a running Llama server")
@@ -55,7 +56,8 @@ public class CompletionClientTest {
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.textcompletion");
final var completion = this.textCompletionClient.conversation(
modelProfile,
Conversation.create("You are a helpful assistant", "Say the color is green"));
Conversation.create("You are a helpful assistant", "Say the color is green"),
false);
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
@@ -69,7 +71,7 @@ public class CompletionClientTest {
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.textCompletionClient.conversation(modelProfile, conversation);
final var completion = this.textCompletionClient.conversation(modelProfile, conversation, false);
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
@@ -92,7 +94,7 @@ public class CompletionClientTest {
System.out.print(chunk.content());
sb.append(chunk.content());
}
});
}, false);
if (completion.reasoning() != null) {
// Prompt template does not separate reasoning and response from the stream at the moment
@@ -112,7 +114,7 @@ public class CompletionClientTest {
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.chatCompletionclient.conversation(modelProfile, conversation);
final var completion = this.chatCompletionclient.conversation(modelProfile, conversation, false);
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
@@ -135,7 +137,7 @@ public class CompletionClientTest {
System.out.print(chunk.content());
sb.append(chunk.content());
}
});
}, false);
assertEquals(completion.response(), sb.toString());
assertTrue(Grader2.assessScoreBinary("Does the text explain a chess opening?", completion.response(), modelProfile, this.clientService));
@@ -151,10 +153,10 @@ public class CompletionClientTest {
}
sb.delete(sb.length() - 2, sb.length());
final var completion = this.textCompletionClient.completion(
null,
"You are counting machine",
"How many numbers do you count?\n" + sb.toString());
final var completion = this.textCompletionClient.conversation(
ModelCategory.MEDIUM,
Conversation.create("You are counting machine", "How many numbers do you count?\n" + sb.toString()),
false);
LOGGER.info(completion.response());
}
@@ -199,10 +201,10 @@ public class CompletionClientTest {
private Tasks createTasks() {
return new Tasks(() -> {
try {
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the annual profit margin?");
final var completion = this.textCompletionClient.conversation(
ModelCategory.MEDIUM,
Conversation.create("You are a helpful assistant", "How do I calculate the annual profit margin?"),
false);
LOGGER.info(completion.response());
}
catch (final Exception e) {
@@ -210,10 +212,10 @@ public class CompletionClientTest {
}
}, () -> {
try {
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the operating profit ratio?");
final var completion = this.textCompletionClient.conversation(
ModelCategory.MEDIUM,
Conversation.create("You are a helpful assistant", "How do I calculate the operating profit ratio?"),
false);
LOGGER.info(completion.response());
}
catch (final Exception e) {
@@ -221,10 +223,10 @@ public class CompletionClientTest {
}
}, () -> {
try {
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the return of investment?");
final var completion = this.textCompletionClient.conversation(
ModelCategory.MEDIUM,
Conversation.create("You are a helpful assistant", "How do I calculate the return of investment?"),
false);
LOGGER.info(completion.response());
}
catch (final Exception e) {
@@ -232,10 +234,10 @@ public class CompletionClientTest {
}
}, () -> {
try {
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the return on net worth?");
final var completion = this.textCompletionClient.conversation(
ModelCategory.MEDIUM,
Conversation.create("You are a helpful assistant", "How do I calculate the return on net worth?"),
false);
LOGGER.info(completion.response());
}
catch (final Exception e) {