exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Refactor grading logic to utilize ModelProfile and ClientService in Grader2 and related tests

Browse Source
This commit is contained in:
extio-de
2025-12-02 21:41:07 +01:00
parent e539ec44e1
commit 0bc9468827
8 changed files with 84 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java
View File

@@ -1,5 +1,7 @@
package de.extio.lmlib.client.azureai;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
@@ -11,10 +13,12 @@ import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.grader.Grader2;
import de.extio.lmlib.profile.ModelProfileService;
@Disabled("This test requires an Azure subscription (setup azure key in model profile)")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@@ -28,41 +32,56 @@ public class AzureAiClientTest {
@Autowired
private AzureAiClient client;
@Autowired
private ModelProfileService modelProfileService;
@Autowired
private ClientService clientService;
@Test
void completion() throws Exception {
final var completion = this.client.completion(
ModelCategory.MEDIUM,
"You are a clever story teller",
"Continue this story:\nOnce upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
LOGGER.info(completion.toString());
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
final var completion = this.client.conversation(
modelProfile,
Conversation.create("You are a clever story teller", "Continue this story:\nOnce upon a time, there was a spaceship travelling to a black hole. The crew were cats!"));
LOGGER.info(completion.response());
assertTrue(Grader2.assessScoreBinary("Does the story continue about a spaceship with cat crew?", completion.response(), modelProfile, this.clientService));
}
@Test
void completionLarge() throws Exception {
final var completion = this.client.completion(
ModelCategory.LARGE,
"You are a clever story teller",
"Continue this story:\nOnce upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
final var completion = this.client.conversation(
modelProfile,
Conversation.create("You are a clever story teller", "Continue this story:\nOnce upon a time, there was a spaceship travelling to a black hole. The crew were cats!"));
LOGGER.info(completion.response());
assertTrue(Grader2.assessScoreBinary("Does the story continue about a spaceship with cat crew?", completion.response(), modelProfile, this.clientService));
}
@Test
void chat() throws Exception {
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Berlin.");
conversation.addTurn(new Turn(TurnType.ASSISTANT,
"T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius."));
conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?"));
final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation);
final var completion = this.client.conversation(modelProfile, conversation);
LOGGER.info(completion.response());
assertTrue(Grader2.assessScoreBinary("Does the response mention summer weather?", completion.response(), modelProfile, this.clientService));
}
@Test
void longPrompt() throws Exception {
final var completion = this.client.completion(
null,
"You are a helpful assistant",
"Count the numbers: 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9");
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
final var completion = this.client.conversation(
modelProfile,
Conversation.create("You are a helpful assistant", "Count the numbers: 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9"));
LOGGER.info(completion.response());
assertTrue(Grader2.assessScoreBinary("Did the assistant count the numbers?", completion.response(), modelProfile, this.clientService));
}
}

41
src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java
View File

@@ -20,13 +20,14 @@ import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient;
import de.extio.lmlib.client.oai.completion.text.TextCompletionClient;
import de.extio.lmlib.grader.Grader;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.grader.Grader2;
import de.extio.lmlib.profile.ModelProfileService;
@Disabled("This test requires a running Llama server")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@@ -43,39 +44,47 @@ public class CompletionClientTest {
@Autowired
private ChatCompletionClient chatCompletionclient;
@Autowired
private ModelProfileService modelProfileService;
@Autowired
private ClientService clientService;
@Test
void completion() throws Exception {
final var completion = this.textCompletionClient.completion(
ModelCategory.SMALL,
"You are a helpful assistant",
"Say the color is green");
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.textcompletion");
final var completion = this.textCompletionClient.conversation(
modelProfile,
Conversation.create("You are a helpful assistant", "Say the color is green"));
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.chatCompletionclient));
assertTrue(Grader2.assessScoreBinary("Does the text mention the color green?", completion.response(), modelProfile, this.clientService));
}
@Test
void chatTextCompletion() throws Exception {
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.textcompletion");
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.textCompletionClient.conversation(ModelCategory.SMALL, conversation);
final var completion = this.textCompletionClient.conversation(modelProfile, conversation);
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.chatCompletionclient));
assertTrue(Grader2.assessScoreBinary("Does the text mention a color?", completion.response(), modelProfile, this.clientService));
}
@Test
void streamTextCompletion() throws Exception {
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.textcompletion");
final var conversation = Conversation.create("You are a helpful assistant", "I would like to play a role-playing game with you.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which character do you choose?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose! But I need you to explain the character attributes to me."));
final StringBuilder sb = new StringBuilder();
final var completion = this.textCompletionClient.streamConversation(ModelCategory.SMALL, conversation, chunk -> {
final var completion = this.textCompletionClient.streamConversation(modelProfile, conversation, chunk -> {
if (chunk.reasoningContent() != null) {
System.out.print(chunk.reasoningContent());
}
@@ -94,29 +103,31 @@ public class CompletionClientTest {
else {
assertEquals(completion.response(), sb.toString());
}
assertTrue(Grader.assessScoreBinary("Does the text mention a role-playing game character?", completion.response(), this.chatCompletionclient));
assertTrue(Grader2.assessScoreBinary("Does the text mention a role-playing game character?", completion.response(), modelProfile, this.clientService));
}
@Test
void chatCompletion() throws Exception {
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.chatcompletion");
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.chatCompletionclient.conversation(ModelCategory.MEDIUM, conversation);
final var completion = this.chatCompletionclient.conversation(modelProfile, conversation);
LOGGER.info(completion.reasoning());
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.chatCompletionclient));
assertTrue(Grader2.assessScoreBinary("Does the text mention a color?", completion.response(), modelProfile, this.clientService));
}
@Test
void streamChatCompletion() throws Exception {
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.chatcompletion");
final var conversation = Conversation.create("You are a helpful assistant", "I would like to play a game of chess.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which opening do you choose?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose! But I need you to explain the opening to me."));
final StringBuilder sb = new StringBuilder();
final var completion = this.chatCompletionclient.streamConversation(ModelCategory.MEDIUM, conversation, chunk -> {
final var completion = this.chatCompletionclient.streamConversation(modelProfile, conversation, chunk -> {
if (chunk.reasoningContent() != null) {
System.out.print(chunk.reasoningContent());
}
@@ -127,7 +138,7 @@ public class CompletionClientTest {
});
assertEquals(completion.response(), sb.toString());
assertTrue(Grader.assessScoreBinary("Does the text explain a chess opening?", completion.response(), this.chatCompletionclient));
assertTrue(Grader2.assessScoreBinary("Does the text explain a chess opening?", completion.response(), modelProfile, this.clientService));
}
@Disabled

10
src/test/java/de/extio/lmlib/grader/GraderTest.java
View File

@@ -39,8 +39,9 @@ import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.grader.agent.AnswerGrader;
import de.extio.lmlib.grader.agent.AnswerGrader2;
import de.extio.lmlib.grader.agent.AnswerGraderBinary;
import de.extio.lmlib.profile.ModelCategory;
@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
// @Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@@ -56,6 +57,9 @@ public class GraderTest {
@Autowired
private ClientService clientService;
@Autowired
private de.extio.lmlib.profile.ModelProfileService modelProfileService;
private long testStartNano;
private final Map<String, Long> methodTotalNanos = new HashMap<>();
@@ -244,10 +248,10 @@ public class GraderTest {
@ParameterizedTest
@MethodSource("gradingScenarios")
void graderCurrent(final String question, final String answer, final boolean expectedPass) {
final boolean result = runBinaryGrader("Grader2", question, answer, (q, a) -> Grader2.assessScoreBinary(q, a, this.clientService));
final boolean result = runBinaryGrader("Grader2", question, answer, (q, a) -> Grader2.assessScoreBinary(q, a, ModelCategory.MEDIUM, this.modelProfileService, this.clientService));
assertEquals(expectedPass, result, "Grader2 result mismatch for question: " + question);
}
private boolean runAnswerGrader(final String label, final BaseAgent agent, final Consumer<AgentContext> contextConfigurer) {
final var context = new AgentContext(Map.of("AnswerGrader", agent));
contextConfigurer.accept(context);