exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Grader to library

Browse Source
This commit is contained in:
extio-de
2025-06-23 20:16:23 +02:00
parent cf6b868d3c
commit d76e2e8f4f
3 changed files with 44 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/main/java/de/extio/lmlib/grader/Grader.java Normal file
View File

@@ -0,0 +1,41 @@
package de.extio.lmlib.grader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.profile.ModelCategory;
public class Grader {
private static final Logger LOGGER = LoggerFactory.getLogger(Grader.class);
public static boolean assessScoreBinary(final String question, final String text, final ClientService clientService) {
LOGGER.info(question);
final var client = clientService.getClient(ModelCategory.MEDIUM);
var score = 0;
while (Math.abs(score) < 3) {
final var completion = client.completion(
ModelCategory.MEDIUM,
"""
You are a grader assessing the truthfulness of a given text to a user question.
Please provide a binary response 'true' or 'false' for the following text.
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
Only provide the response as a single word and no preamble and no explanation.""",
"Here is the text:\n" + text,
"\nHere is the user question: " + question);
LOGGER.info(completion.response());
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
if (Math.abs(score) == 2) {
break;
}
}
final var result = score > 0;
LOGGER.info("->" + String.valueOf(result));
return result;
}
}

31
src/test/java/de/extio/lmlib/agent/AgentTest.java
View File

@@ -20,6 +20,7 @@ import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.grader.Grader;
import de.extio.lmlib.profile.ModelCategory;
import io.netty.util.internal.ThreadLocalRandom;
@@ -114,7 +115,7 @@ public class AgentTest {
assertEquals(1, resultContext.getContext().get("blogPost").size());
assertEquals(8, resultContext.getRequestStatistic().getRequests().get());
for (final var feature : resultContext.getContext().get("features")) {
assertTrue(this.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString()));
assertTrue(Grader.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString(), this.clientService));
}
}
@@ -197,32 +198,4 @@ public class AgentTest {
}
}
private boolean assessScoreBinary(final String question, final String text) {
LOGGER.info(question);
final var client = this.clientService.getClient(ModelCategory.MEDIUM);
var score = 0;
while (Math.abs(score) < 3) {
final var completion = client.completion(
ModelCategory.MEDIUM,
"""
You are a grader assessing the truthfulness of a given text to a user question.
Please provide a binary response 'true' or 'false' for the following text.
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
Only provide the response as a single word and no preamble and no explanation.""",
"Here is the text:\n" + text,
"\nHere is the user question: " + question);
LOGGER.info(completion.response());
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
if (Math.abs(score) == 2) {
break;
}
}
final var result = score > 0;
LOGGER.info("->" + String.valueOf(result));
return result;
}
}