From 04beaf6b5fe977996453cec1f7f44e9d7eba3a49 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 22 Nov 2025 18:40:35 +0100 Subject: [PATCH] feat: Introduce Grader2 and AnswerGrader2 for improved grading functionality; deprecate legacy Grader and AnswerGrader --- README.md | 125 ++++++- .../java/de/extio/lmlib/grader/Grader.java | 4 + .../java/de/extio/lmlib/grader/Grader2.java | 62 ++++ .../lmlib/grader/agent/AnswerGrader.java | 4 + .../lmlib/grader/agent/AnswerGrader2.java | 91 ++++++ .../grader/agent/AnswerGraderBinary.java | 81 +++++ .../de/extio/lmlib/grader/GraderTest.java | 309 ++++++++++++++++++ 7 files changed, 672 insertions(+), 4 deletions(-) create mode 100644 src/main/java/de/extio/lmlib/grader/Grader2.java create mode 100755 src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java create mode 100644 src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java create mode 100644 src/test/java/de/extio/lmlib/grader/GraderTest.java diff --git a/README.md b/README.md index 64ac979..3fd5e08 100644 --- a/README.md +++ b/README.md @@ -353,13 +353,130 @@ public class MyInterceptor implements CompletionInterceptor { The client will autowire all `CompletionInterceptor` implementations and apply them to requests automatically. -# Grader +# Graders -The library includes a grader component that evaluates the correctness of agent responses. +The library includes multiple grader implementations for evaluating the correctness and quality of responses. Graders can be used either as standalone utilities or integrated into agentic flows. Each grader uses a different evaluation approach and offers trade-offs between accuracy and performance. -Example: +**Performance Note:** The performance characteristics of agent graders vary significantly depending on the model type. Reasoning models have substantial inherent overhead, which means the additional complexity of agent-based approaches has relatively less impact on total execution time. Non-reasoning models show more pronounced relative differences because they have lower baseline overhead. All performance factors are relative to `Grader2` (baseline) and measured under identical conditions. - assertTrue(Grader.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getStringValue("blogPost"), this.clientService)); +## Conventional Graders + +Standalone graders for use outside agentic flows. These are simple utilities you can call directly from your code without requiring an `AgentContext`. + +### Grader2 (Recommended for Conventional Grading) + +**Approach:** Majority voting with multiple independent assessments. + +Uses the grader logic multiple times and returns the majority result. This approach is slower but more robust and reliable, reducing the impact of individual LLM evaluation errors. + +**Performance:** Baseline (1.0x) + +**Feedback Loop:** Not supported (conventional grader) + +**Use Case:** General-purpose binary evaluation (pass/fail) outside of agentic flows. + +**Usage:** +```java +final var result = Grader2.assessScoreBinary("Does the text mention feature X?", textToGrade, clientService); +``` + +### Grader (Deprecated) + +Legacy majority voting grader. Use `Grader2` instead. + +## Agent Graders + +Specialized graders designed to be integrated into agentic flows as part of an `AgentContext`. These agents can be composed within larger workflows and support feedback loops for iterative improvement. + +### AnswerGrader2 (Recommended for Detailed Feedback in Flows) + +**Approach:** Agentic evaluation with detailed scoring and feedback loop. + +An agent-based grader that provides a detailed rating (0-100) for each answer along with an explanation. Supports optional descriptions via the `qaOutputDescription` context key. Includes a feedback loop mechanism (up to 2 attempts) to improve scoring accuracy when initial ratings are below threshold (≤60). + +**Performance:** +- **With descriptions:** ~1.5x - 7x slower than baseline — provides detailed explanations +- **Without descriptions:** ~1.2x - 2.3x slower than baseline — faster, ratings only + +*Note: Performance varies significantly by model type. Reasoning models show minimal relative overhead (~1.5x/1.2x) due to their substantial baseline processing time, while non-reasoning models show greater relative differences (~7x/2.3x).* + +**Feedback Loop:** Supported (retries up to 2 attempts when rating ≤60) + +**Use Case:** Quality assessment within agentic flows where detailed feedback and scoring are required, with optional retry logic for borderline cases. + +**Context Keys:** +- `qaQuestion`: The question to evaluate +- `qaAnswer`: The answer to grade +- `qaOutputDescription`: Set to `Boolean.TRUE` (default) for detailed explanations or `Boolean.FALSE` for ratings only (required for feedback loop to provide meaningful feedback) +- `qaFeedbackLoopDestination` (required for feedback loop): Agent name to return to for answer refinement + +**Usage:** +```java +context.setStringValue("qaQuestion", "Is this a good solution?"); +context.setStringValue("qaAnswer", answerText); +context.setValue("qaOutputDescription", Boolean.FALSE); // For faster evaluation +context.setStringValue("qaFeedbackLoopDestination", "AnswerRefiner"); +final var result = agentExecutorService.walk("AnswerGrader2", context); +``` + +### AnswerGraderBinary (Recommended for Speed in Flows) + +**Approach:** Agentic binary evaluation with feedback loop. + +A lightweight agent-based grader that determines if an answer is a successful resolution (true/false) with optional explanations. Includes a feedback loop mechanism (up to 2 attempts) for improved accuracy when evaluations fail. + +**Performance:** +- **With descriptions:** ~1.3x - 6.3x slower than baseline — includes brief explanations +- **Without descriptions:** ~1.0x - 2.2x slower than baseline — fastest agent grader option + +*Note: Performance varies significantly by model type. Reasoning models show minimal relative overhead (~1.3x/1.0x) due to their substantial baseline processing time, while non-reasoning models show greater relative differences (~6x/2x).* + +**Feedback Loop:** Supported (retries up to 2 attempts when evaluation fails) + +**Use Case:** Quick pass/fail evaluation within agentic flows where speed is prioritized, with optional feedback for failed evaluations. + +**Context Keys:** +- `qaQuestion`: The question to evaluate +- `qaAnswer`: The answer to evaluate +- `qaOutputDescription`: Set to `Boolean.TRUE` (default) for explanations or `Boolean.FALSE` for boolean only (required for feedback loop to provide meaningful feedback) +- `qaFeedbackLoopDestination` (required for feedback loop): Agent name to return to for answer refinement + +**Usage:** +```java +context.setStringValue("qaQuestion", "Does this answer the question?"); +context.setStringValue("qaAnswer", answerText); +context.setValue("qaOutputDescription", Boolean.FALSE); // For maximum speed +context.setStringValue("qaFeedbackLoopDestination", "AnswerRefiner"); +final var result = agentExecutorService.walk("AnswerGraderBinary", context); +``` + +### AnswerGrader (Deprecated) + +Legacy agent-based grader. Use `AnswerGrader2` instead. + +## Feedback Loop Mechanism in Agent Graders + +Both `AnswerGrader2` and `AnswerGraderBinary` support an optional feedback loop for iterative improvement within agentic flows. The feedback loop provides actionable feedback to the source agent to refine their answer. When configured: + +1. If the evaluation fails and fewer than 2 attempts have been made, the grader: + - Constructs feedback from the previous evaluation (using `qaExplanation` from the grader output) + - Returns to the specified feedback loop destination (the source agent) with a retry flag + - The source agent receives the feedback via the `qaFeedback` context key and can refine the answer + - The refined answer is resubmitted for evaluation + +2. After 2 attempts or if the evaluation passes, evaluation completes with the final result + +**Note:** For the feedback loop to provide meaningful feedback, `qaOutputDescription` should be set to `Boolean.TRUE` to ensure the grader generates explanations that can guide the source agent's refinement. + +**Example with Feedback Loop:** +```java +// Set up an agentic flow with a feedback loop +context.setStringValue("qaQuestion", "Explain machine learning"); +context.setStringValue("qaAnswer", initialAnswer); +context.setStringValue("qaFeedbackLoopDestination", "AnswerEnhancer"); +final var result = agentExecutorService.walk("AnswerGraderBinary", context); +// If grading fails, will loop back to AnswerEnhancer for one retry +``` # Agentic Flows diff --git a/src/main/java/de/extio/lmlib/grader/Grader.java b/src/main/java/de/extio/lmlib/grader/Grader.java index 8363e90..3ef8489 100644 --- a/src/main/java/de/extio/lmlib/grader/Grader.java +++ b/src/main/java/de/extio/lmlib/grader/Grader.java @@ -7,6 +7,10 @@ import de.extio.lmlib.client.Client; import de.extio.lmlib.client.ClientService; import de.extio.lmlib.profile.ModelCategory; +/** + * @deprecated Use {@link Grader2} instead. + */ +@Deprecated public class Grader { private static final Logger LOGGER = LoggerFactory.getLogger(Grader.class); diff --git a/src/main/java/de/extio/lmlib/grader/Grader2.java b/src/main/java/de/extio/lmlib/grader/Grader2.java new file mode 100644 index 0000000..3657f72 --- /dev/null +++ b/src/main/java/de/extio/lmlib/grader/Grader2.java @@ -0,0 +1,62 @@ +package de.extio.lmlib.grader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.extio.lmlib.client.Client; +import de.extio.lmlib.client.ClientService; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.profile.ModelCategory; + +public class Grader2 { + + private static final Logger LOGGER = LoggerFactory.getLogger(Grader2.class); + + public static boolean assessScoreBinary(final String question, final String text, final ClientService clientService) { + final var client = clientService.getClient(ModelCategory.MEDIUM); + return assessScoreBinary(question, text, client); + } + + public static boolean assessScoreBinary(final String question, final String text, final Client client) { + LOGGER.info(question); + + var score = 0; + while (Math.abs(score) < 3) { + final var completion = client.conversation( + ModelCategory.MEDIUM, + Conversation.create( + """ + You are an expert grader. Your role is to assess the truthfulness of a given text in response to a user question. + """, + """ + I need you to determine if the provided text is a truthful answer to the question. + + ### Input Data + **Question:** + %s + + **Text:** + %s + + ### Task + Assess whether the text provides a truthful answer to the question. + + ### Output Format + Provide a binary response 'true' or 'false'. + - 'true': The text provides a truthful answer. + - 'false': The text does not provide a truthful answer. + Only provide the response as a single word. Do not include any preamble or explanation. + """.formatted(question, text))); + LOGGER.info(completion.response()); + score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1; + if (Math.abs(score) == 2) { + break; + } + } + + final var result = score > 0; + LOGGER.info("->" + String.valueOf(result)); + return result; + } + +} diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java index c33980a..c68319e 100755 --- a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java @@ -8,6 +8,10 @@ import de.extio.lmlib.agent.AgentType; import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler; import de.extio.lmlib.profile.ModelCategory; +/** + * @deprecated Use {@link AnswerGrader2} instead. + */ +@Deprecated public class AnswerGrader { public static Agent createAgent() { diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java new file mode 100755 index 0000000..801a568 --- /dev/null +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java @@ -0,0 +1,91 @@ +package de.extio.lmlib.grader.agent; + +import java.util.List; + +import de.extio.lmlib.agent.Agent; +import de.extio.lmlib.agent.AgentNext; +import de.extio.lmlib.agent.AgentType; +import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler; +import de.extio.lmlib.profile.ModelCategory; + +public class AnswerGrader2 { + + public static Agent createAgent() { + return new Agent( + "AnswerGrader", + AgentType.COMPLETION, + ModelCategory.MEDIUM, + null, + """ + You are an expert grader agent. Your role is to evaluate the quality of an answer provided for a specific question. + """, + """ + I need you to evaluate the quality of the answer provided below. + + ### Input Data + **Question:** + {{{qaQuestion}}} + + **Answer:** + {{{qaAnswer}}} + + ### Task + Evaluate the answer and provide a rating between 0 and 100 based on its quality and relevance to the question. Use the following scale: + - 100: Directly addresses the question, is accurate, and provides a clear and concise solution or explanation. + - 90-99: Mostly accurate and provides a good solution or explanation, but may lack some detail or clarity. + - 80-89: Generally accurate but may require some additional information or clarification to fully understand the solution or explanation. + - 60-74: Partially accurate or relevant, provides a useful partial solution or explanation, but misses some key information. + - 50-59: Relevant to the general topic but provides an incomplete solution or explanation that does not sufficiently answer the specific question. + - 25-49: Does not directly answer the question, is speculative, contains mainly instructions for changes, or refers to external teams without providing substantial information, but may provide some relevant information or context. + - 0-24: Not related to the question, is inaccurate, or provides no useful information. + + ### Output Format + {{qaOutputFormat}} + """, + new JsonAgentResponseHandler(), + context -> { + final Boolean outputDescription = context.getValue("qaOutputDescription", Boolean.class); + if (outputDescription == null || outputDescription.booleanValue()) { + context.setStringValue("qaOutputFormat", "Provide no preamble and a short explanation. Return the response in JSON format with the following fields: {\"qaRating\": rating, \"qaExplanation\": \"brief explanation\"}"); + } + else { + context.setStringValue("qaOutputFormat", "Provide no preamble. Return the response in JSON format with the following field only: {\"qaRating\": rating}"); + } + context.getContext().remove("qaRating"); + context.getContext().remove("qaExplanation"); + }, + context -> { + if (!context.getContext().containsKey("qaRating")) { + context.setStringValue("qaRating", "0"); + context.setStringValue("qaExplanation", "The grader agent did not provide an answer"); + } + else if (!context.getContext().containsKey("qaExplanation")) { + final var qaRatingEntry = context.getStringValue("qaRating"); + context.setStringValue("qaExplanation", "Rating was not sufficient: " + qaRatingEntry + " out of 100."); + } + + context.getContext().compute("qaIteration", (k, v) -> { + if (v == null) { + return List.of(1); + } + else { + return List.of((Integer) v.get(0) + 1); + } + }); + }, + null, + context -> { + final var qaRatingEntry = context.getStringValue("qaRating"); + final var qaRating = qaRatingEntry != null ? Integer.parseInt(qaRatingEntry) : 0; + final var qaPassed = qaRating > 60; + context.getContext().put("qaPassed", List.of(qaPassed)); + + if (!qaPassed && (int) context.getContext().get("qaIteration").get(0) < 2) { + context.setStringValue("qaFeedback", "Feedback from the previous attempt to consider: " + context.getStringValue("qaExplanation")); + return new AgentNext(context.getStringValue("qaFeedbackLoopDestination"), Boolean.FALSE); + } + + return new AgentNext(null, qaPassed); + }); + } +} diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java new file mode 100644 index 0000000..f9fc3e0 --- /dev/null +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java @@ -0,0 +1,81 @@ +package de.extio.lmlib.grader.agent; + +import java.util.List; + +import de.extio.lmlib.agent.Agent; +import de.extio.lmlib.agent.AgentNext; +import de.extio.lmlib.agent.AgentType; +import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler; +import de.extio.lmlib.profile.ModelCategory; + +public class AnswerGraderBinary { + + public static Agent createAgent() { + return new Agent( + "AnswerGraderBinary", + AgentType.COMPLETION, + ModelCategory.MEDIUM, + null, + """ + You are a focused binary grader that determines whether an assistant answer sufficiently resolves a question. + """, + """ + Evaluate whether the provided response truthfully and directly answers the user question. + + Input Question: + {{{qaQuestion}}} + + Input Answer: + {{{qaAnswer}}} + + Task: Decide if the answer can be classified as a successful resolution (true) or not (false) for the given question. + + Output Format: + {{qaOutputFormat}} + """, + new JsonAgentResponseHandler(), + context -> { + final Boolean outputDescription = context.getValue("qaOutputDescription", Boolean.class); + if (outputDescription == null || outputDescription.booleanValue()) { + context.setStringValue("qaOutputFormat", "Return a JSON object with exactly these keys: {\"qaPassed\": boolean, \"qaExplanation\": \"brief rationale\"}"); + } + else { + context.setStringValue("qaOutputFormat", "Return a JSON object with exactly this key: {\"qaPassed\": boolean}"); + } + context.getContext().remove("qaPassed"); + context.getContext().remove("qaExplanation"); + }, + context -> { + if (!context.getContext().containsKey("qaPassed")) { + context.setStringValue("qaPassed", "false"); + context.setStringValue("qaExplanation", "The grader agent did not provide a pass/fail output"); + } + else if (!context.getContext().containsKey("qaExplanation")) { + context.setStringValue("qaExplanation", "The answer did not pass evaluation"); + } + + context.getContext().compute("qaIteration", (k, v) -> { + if (v == null) { + return List.of(1); + } + else { + return List.of((Integer) v.get(0) + 1); + } + }); + }, + null, + context -> { + final var qaPassedEntry = context.getStringValue("qaPassed"); + final var qaPassed = qaPassedEntry != null && Boolean.parseBoolean(qaPassedEntry); + context.getContext().put("qaPassed", List.of(qaPassed)); + + if (!qaPassed && (int) context.getContext().get("qaIteration").get(0) < 2) { + context.setStringValue("qaFeedback", "Feedback from the previous attempt to consider: " + context.getStringValue("qaExplanation")); + return new AgentNext(context.getStringValue("qaFeedbackLoopDestination"), Boolean.FALSE); + } + + return new AgentNext(null, qaPassed); + }); + } +} + diff --git a/src/test/java/de/extio/lmlib/grader/GraderTest.java b/src/test/java/de/extio/lmlib/grader/GraderTest.java new file mode 100644 index 0000000..0f0b0b6 --- /dev/null +++ b/src/test/java/de/extio/lmlib/grader/GraderTest.java @@ -0,0 +1,309 @@ +package de.extio.lmlib.grader; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.TestWatcher; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.PropertySource; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +import de.extio.lmlib.agent.AgentContext; +import de.extio.lmlib.agent.AgentExecutorService; +import de.extio.lmlib.agent.BaseAgent; +import de.extio.lmlib.client.ClientService; +import de.extio.lmlib.grader.agent.AnswerGrader; +import de.extio.lmlib.grader.agent.AnswerGrader2; +import de.extio.lmlib.grader.agent.AnswerGraderBinary; + +@SpringBootTest(webEnvironment = WebEnvironment.NONE) +@SpringBootConfiguration +@ComponentScan(basePackages = "de.extio.lmlib") +@PropertySource("classpath:/application-test.properties") +@TestInstance(Lifecycle.PER_CLASS) +public class GraderTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(GraderTest.class); + + @Autowired + private AgentExecutorService agentExecutor; + + @Autowired + private ClientService clientService; + + private long testStartNano; + + private final Map methodTotalNanos = new HashMap<>(); + private final Map methodInvocationCount = new HashMap<>(); + private final Map methodFailureCount = new HashMap<>(); + + static Stream gradingScenarios() { + return Stream.of( + // Geography + Arguments.of( + "What is the capital of France?", + "The capital of France is Paris.", + true), + Arguments.of( + "What is the capital of France?", + "I don't know.", + false), + Arguments.of( + "Name three countries in Southeast Asia.", + "Thailand, Vietnam, and Indonesia are three countries in Southeast Asia.", + true), + Arguments.of( + "Name three countries in Southeast Asia.", + "Some countries exist there.", + false), + // Technology Support + Arguments.of( + "How do I reset my password?", + "You can reset your password by clicking on the 'Forgot Password' link on the login page and following the instructions sent to your email.", + true), + Arguments.of( + "How do I reset my password?", + "Contact support.", + false), + Arguments.of( + "How do I reset my password?", + "Read the manual!", + false), + Arguments.of( + "How do I enable two-factor authentication?", + "Go to Settings > Security > Two-Factor Authentication, then choose your preferred method (SMS, email, or authenticator app) and follow the setup steps.", + true), + Arguments.of( + "How do I enable two-factor authentication?", + "Ask your administrator.", + false), + // Science + Arguments.of( + "What is the boiling point of water?", + "100 degrees Celsius at sea level.", + true), + Arguments.of( + "What is the boiling point of water?", + "It is hot.", + false), + Arguments.of( + "Explain photosynthesis in one sentence.", + "Photosynthesis is the process where plants convert sunlight, water, and carbon dioxide into glucose and oxygen.", + true), + Arguments.of( + "Explain photosynthesis in one sentence.", + "Plants do something with light.", + false), + // Mathematics + Arguments.of( + "What is 15% of 200?", + "15% of 200 is 30.", + true), + Arguments.of( + "What is 15% of 200?", + "A lot.", + false), + Arguments.of( + "Solve for x: 2x + 5 = 15", + "x = 5, because 2(5) + 5 = 10 + 5 = 15.", + true), + Arguments.of( + "Solve for x: 2x + 5 = 15", + "Maybe x is 5 or something else.", + false), + Arguments.of( + "Solve for x: 2x + 5 = 15", + "x is 10", + false), + // History + Arguments.of( + "In what year did World War II end?", + "World War II ended in 1945.", + true), + Arguments.of( + "In what year did World War II end?", + "A long time ago.", + false), + Arguments.of( + "Who was the first President of the United States?", + "George Washington was the first President of the United States, serving from 1789 to 1797.", + true), + Arguments.of( + "Who was the first President of the United States?", + "Some important person.", + false), + // Programming + Arguments.of( + "What is the time complexity of binary search?", + "The time complexity of binary search is O(log n).", + true), + Arguments.of( + "What is the time complexity of binary search?", + "It's fast.", + false), + Arguments.of( + "Explain the difference between an array and a linked list.", + "Arrays provide constant-time access but fixed size, while linked lists provide dynamic size but require linear-time access.", + true), + Arguments.of( + "Explain the difference between an array and a linked list.", + "They are different data structures.", + false)); + } + + @SuppressWarnings("deprecation") + @ParameterizedTest + @MethodSource("gradingScenarios") + void answerGraderLegacy(final String question, final String answer, final boolean expectedPass) { + LOGGER.info("Legacy AnswerGrader - Question: '{}' Answer: '{}'", question, answer); + final var agent = AnswerGrader.createAgent(); + final var context = new AgentContext(Map.of("AnswerGrader", agent)); + context.getContext().put("questions", List.of(question)); + context.getContext().put("answer", List.of(answer)); + final var result = this.agentExecutor.walk(agent, context).getFirst(); + final boolean passed = (boolean) result.getContext().get("qaPassed").getFirst(); + LOGGER.info("AnswerGrader - Rating: {}, Passed: {}, Explanation: {}", result.getStringValue("qaRating"), passed, result.getStringValue("qaExplanation")); + assertEquals(expectedPass, passed, "AnswerGrader result mismatch for question: " + question); + } + + @ParameterizedTest + @MethodSource("gradingScenarios") + void answerGraderCurrent(final String question, final String answer, final boolean expectedPass) { + final boolean passed = runAnswerGrader("AnswerGrader2", AnswerGrader2.createAgent(), context -> { + context.getContext().put("qaQuestion", List.of(question)); + context.getContext().put("qaAnswer", List.of(answer)); + }); + assertEquals(expectedPass, passed, "AnswerGrader2 result mismatch for question: " + question); + } + + @ParameterizedTest + @MethodSource("gradingScenarios") + void answerGraderBinary(final String question, final String answer, final boolean expectedPass) { + final boolean passed = runAnswerGrader("AnswerGraderBinary", AnswerGraderBinary.createAgent(), context -> { + context.getContext().put("qaQuestion", List.of(question)); + context.getContext().put("qaAnswer", List.of(answer)); + }); + assertEquals(expectedPass, passed, "AnswerGraderBinary result mismatch for question: " + question); + } + + @ParameterizedTest + @MethodSource("gradingScenarios") + void answerGraderCurrentNoDesc(final String question, final String answer, final boolean expectedPass) { + final boolean passed = runAnswerGrader("AnswerGrader2 (no description)", AnswerGrader2.createAgent(), context -> { + context.getContext().put("qaQuestion", List.of(question)); + context.getContext().put("qaAnswer", List.of(answer)); + context.setValue("qaOutputDescription", Boolean.FALSE); + }); + assertEquals(expectedPass, passed, "AnswerGrader2 (no description) result mismatch for question: " + question); + } + + @ParameterizedTest + @MethodSource("gradingScenarios") + void answerGraderBinaryNoDesc(final String question, final String answer, final boolean expectedPass) { + final boolean passed = runAnswerGrader("AnswerGraderBinary (no description)", AnswerGraderBinary.createAgent(), context -> { + context.getContext().put("qaQuestion", List.of(question)); + context.getContext().put("qaAnswer", List.of(answer)); + context.setValue("qaOutputDescription", Boolean.FALSE); + }); + assertEquals(expectedPass, passed, "AnswerGraderBinary (no description) result mismatch for question: " + question); + } + + @SuppressWarnings("deprecation") + @ParameterizedTest + @MethodSource("gradingScenarios") + void graderLegacy(final String question, final String answer, final boolean expectedPass) { + final boolean result = runBinaryGrader("Grader", question, answer, (q, a) -> Grader.assessScoreBinary(q, a, this.clientService)); + assertEquals(expectedPass, result, "Grader result mismatch for question: " + question); + } + + @ParameterizedTest + @MethodSource("gradingScenarios") + void graderCurrent(final String question, final String answer, final boolean expectedPass) { + final boolean result = runBinaryGrader("Grader2", question, answer, (q, a) -> Grader2.assessScoreBinary(q, a, this.clientService)); + assertEquals(expectedPass, result, "Grader2 result mismatch for question: " + question); + } + + private boolean runAnswerGrader(final String label, final BaseAgent agent, final Consumer contextConfigurer) { + final var context = new AgentContext(Map.of("AnswerGrader", agent)); + contextConfigurer.accept(context); + final var result = this.agentExecutor.walk(agent, context).getFirst(); + final boolean passed = (boolean) result.getContext().get("qaPassed").getFirst(); + LOGGER.info("{} - Rating: {}, Passed: {}, Explanation: {}", label, result.getStringValue("qaRating"), passed, + result.getStringValue("qaExplanation")); + return passed; + } + + private boolean runBinaryGrader(final String label, final String question, final String answer, + final BiFunction grader) { + final boolean result = grader.apply(question, answer); + LOGGER.info("{} - Result: {}", label, result); + return result; + } + + @BeforeEach + void captureStart(TestInfo info) { + this.testStartNano = System.nanoTime(); + LOGGER.info("Starting {}", info.getDisplayName()); + } + + @AfterEach + void logDuration(TestInfo info) { + final var elapsedMs = (System.nanoTime() - this.testStartNano) / 1_000_000.0; + LOGGER.info("Completed {} in {} ms", info.getDisplayName(), String.format("%.2f", elapsedMs)); + recordDuration(info, (long) (elapsedMs * 1_000_000)); + } + + @AfterAll + void logTotals() { + this.methodTotalNanos.forEach((method, total) -> { + final var count = this.methodInvocationCount.get(method); + LOGGER.info("{} total time: {} ms over {} runs", + method, + String.format("%.2f", total / 1_000_000.0), + count); + }); + this.methodFailureCount.forEach((method, failures) -> LOGGER.info("{} failures: {}", method, failures)); + } + + private void recordDuration(final TestInfo info, final long nanos) { + final var method = info.getTestMethod().map(Method::getName).orElse(info.getDisplayName()); + this.methodTotalNanos.merge(method, nanos, Long::sum); + this.methodInvocationCount.merge(method, 1, Integer::sum); + } + + @RegisterExtension + final TestWatcher watcher = new TestWatcher() { + + private String methodName(final ExtensionContext context) { + return context.getTestMethod().map(Method::getName).orElse(context.getDisplayName()); + } + + @Override + public void testFailed(final ExtensionContext context, final Throwable cause) { + GraderTest.this.methodFailureCount.merge(methodName(context), 1, Integer::sum); + } + }; +}