exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Introduce Grader2 and AnswerGrader2 for improved grading functionality; deprecate legacy Grader and AnswerGrader

Browse Source
This commit is contained in:
extio-de
2025-11-22 18:40:35 +01:00
parent ae322743d6
commit 04beaf6b5f
7 changed files with 672 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmlib/grader/Grader.java
View File

@@ -7,6 +7,10 @@ import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
/**
* @deprecated Use {@link Grader2} instead.
*/
@Deprecated
public class Grader {
private static final Logger LOGGER = LoggerFactory.getLogger(Grader.class);

62
src/main/java/de/extio/lmlib/grader/Grader2.java Normal file
View File

@@ -0,0 +1,62 @@
package de.extio.lmlib.grader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.profile.ModelCategory;
public class Grader2 {
private static final Logger LOGGER = LoggerFactory.getLogger(Grader2.class);
public static boolean assessScoreBinary(final String question, final String text, final ClientService clientService) {
final var client = clientService.getClient(ModelCategory.MEDIUM);
return assessScoreBinary(question, text, client);
}
public static boolean assessScoreBinary(final String question, final String text, final Client client) {
LOGGER.info(question);
var score = 0;
while (Math.abs(score) < 3) {
final var completion = client.conversation(
ModelCategory.MEDIUM,
Conversation.create(
"""
You are an expert grader. Your role is to assess the truthfulness of a given text in response to a user question.
""",
"""
I need you to determine if the provided text is a truthful answer to the question.
### Input Data
**Question:**
%s
**Text:**
%s
### Task
Assess whether the text provides a truthful answer to the question.
### Output Format
Provide a binary response 'true' or 'false'.
- 'true': The text provides a truthful answer.
- 'false': The text does not provide a truthful answer.
Only provide the response as a single word. Do not include any preamble or explanation.
""".formatted(question, text)));
LOGGER.info(completion.response());
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
if (Math.abs(score) == 2) {
break;
}
}
final var result = score > 0;
LOGGER.info("->" + String.valueOf(result));
return result;
}
}

4
src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java
View File

@@ -8,6 +8,10 @@ import de.extio.lmlib.agent.AgentType;
import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler;
import de.extio.lmlib.profile.ModelCategory;
/**
* @deprecated Use {@link AnswerGrader2} instead.
*/
@Deprecated
public class AnswerGrader {
public static Agent createAgent() {

91
src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java Executable file
View File

@@ -0,0 +1,91 @@
package de.extio.lmlib.grader.agent;
import java.util.List;
import de.extio.lmlib.agent.Agent;
import de.extio.lmlib.agent.AgentNext;
import de.extio.lmlib.agent.AgentType;
import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler;
import de.extio.lmlib.profile.ModelCategory;
public class AnswerGrader2 {
public static Agent createAgent() {
return new Agent(
"AnswerGrader",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"""
You are an expert grader agent. Your role is to evaluate the quality of an answer provided for a specific question.
""",
"""
I need you to evaluate the quality of the answer provided below.
### Input Data
**Question:**
{{{qaQuestion}}}
**Answer:**
{{{qaAnswer}}}
### Task
Evaluate the answer and provide a rating between 0 and 100 based on its quality and relevance to the question. Use the following scale:
- 100: Directly addresses the question, is accurate, and provides a clear and concise solution or explanation.
- 90-99: Mostly accurate and provides a good solution or explanation, but may lack some detail or clarity.
- 80-89: Generally accurate but may require some additional information or clarification to fully understand the solution or explanation.
- 60-74: Partially accurate or relevant, provides a useful partial solution or explanation, but misses some key information.
- 50-59: Relevant to the general topic but provides an incomplete solution or explanation that does not sufficiently answer the specific question.
- 25-49: Does not directly answer the question, is speculative, contains mainly instructions for changes, or refers to external teams without providing substantial information, but may provide some relevant information or context.
- 0-24: Not related to the question, is inaccurate, or provides no useful information.
### Output Format
{{qaOutputFormat}}
""",
new JsonAgentResponseHandler(),
context -> {
final Boolean outputDescription = context.getValue("qaOutputDescription", Boolean.class);
if (outputDescription == null || outputDescription.booleanValue()) {
context.setStringValue("qaOutputFormat", "Provide no preamble and a short explanation. Return the response in JSON format with the following fields: {\"qaRating\": rating, \"qaExplanation\": \"brief explanation\"}");
}
else {
context.setStringValue("qaOutputFormat", "Provide no preamble. Return the response in JSON format with the following field only: {\"qaRating\": rating}");
}
context.getContext().remove("qaRating");
context.getContext().remove("qaExplanation");
},
context -> {
if (!context.getContext().containsKey("qaRating")) {
context.setStringValue("qaRating", "0");
context.setStringValue("qaExplanation", "The grader agent did not provide an answer");
}
else if (!context.getContext().containsKey("qaExplanation")) {
final var qaRatingEntry = context.getStringValue("qaRating");
context.setStringValue("qaExplanation", "Rating was not sufficient: " + qaRatingEntry + " out of 100.");
}
context.getContext().compute("qaIteration", (k, v) -> {
if (v == null) {
return List.of(1);
}
else {
return List.of((Integer) v.get(0) + 1);
}
});
},
null,
context -> {
final var qaRatingEntry = context.getStringValue("qaRating");
final var qaRating = qaRatingEntry != null ? Integer.parseInt(qaRatingEntry) : 0;
final var qaPassed = qaRating > 60;
context.getContext().put("qaPassed", List.of(qaPassed));
if (!qaPassed && (int) context.getContext().get("qaIteration").get(0) < 2) {
context.setStringValue("qaFeedback", "Feedback from the previous attempt to consider: " + context.getStringValue("qaExplanation"));
return new AgentNext(context.getStringValue("qaFeedbackLoopDestination"), Boolean.FALSE);
}
return new AgentNext(null, qaPassed);
});
}
}

81
src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java Normal file
View File

@@ -0,0 +1,81 @@
package de.extio.lmlib.grader.agent;
import java.util.List;
import de.extio.lmlib.agent.Agent;
import de.extio.lmlib.agent.AgentNext;
import de.extio.lmlib.agent.AgentType;
import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler;
import de.extio.lmlib.profile.ModelCategory;
public class AnswerGraderBinary {
public static Agent createAgent() {
return new Agent(
"AnswerGraderBinary",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
null,
"""
You are a focused binary grader that determines whether an assistant answer sufficiently resolves a question.
""",
"""
Evaluate whether the provided response truthfully and directly answers the user question.
Input Question:
{{{qaQuestion}}}
Input Answer:
{{{qaAnswer}}}
Task: Decide if the answer can be classified as a successful resolution (true) or not (false) for the given question.
Output Format:
{{qaOutputFormat}}
""",
new JsonAgentResponseHandler(),
context -> {
final Boolean outputDescription = context.getValue("qaOutputDescription", Boolean.class);
if (outputDescription == null || outputDescription.booleanValue()) {
context.setStringValue("qaOutputFormat", "Return a JSON object with exactly these keys: {\"qaPassed\": boolean, \"qaExplanation\": \"brief rationale\"}");
}
else {
context.setStringValue("qaOutputFormat", "Return a JSON object with exactly this key: {\"qaPassed\": boolean}");
}
context.getContext().remove("qaPassed");
context.getContext().remove("qaExplanation");
},
context -> {
if (!context.getContext().containsKey("qaPassed")) {
context.setStringValue("qaPassed", "false");
context.setStringValue("qaExplanation", "The grader agent did not provide a pass/fail output");
}
else if (!context.getContext().containsKey("qaExplanation")) {
context.setStringValue("qaExplanation", "The answer did not pass evaluation");
}
context.getContext().compute("qaIteration", (k, v) -> {
if (v == null) {
return List.of(1);
}
else {
return List.of((Integer) v.get(0) + 1);
}
});
},
null,
context -> {
final var qaPassedEntry = context.getStringValue("qaPassed");
final var qaPassed = qaPassedEntry != null && Boolean.parseBoolean(qaPassedEntry);
context.getContext().put("qaPassed", List.of(qaPassed));
if (!qaPassed && (int) context.getContext().get("qaIteration").get(0) < 2) {
context.setStringValue("qaFeedback", "Feedback from the previous attempt to consider: " + context.getStringValue("qaExplanation"));
return new AgentNext(context.getStringValue("qaFeedbackLoopDestination"), Boolean.FALSE);
}
return new AgentNext(null, qaPassed);
});
}
}