diff --git a/pom.xml b/pom.xml index 1a8b94f..0a43aec 100755 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ lm-lib de.extio lm-lib - 1.5.4 + 1.5.5 21 diff --git a/src/main/java/de/extio/lmlib/grader/Grader.java b/src/main/java/de/extio/lmlib/grader/Grader.java new file mode 100644 index 0000000..3ab405b --- /dev/null +++ b/src/main/java/de/extio/lmlib/grader/Grader.java @@ -0,0 +1,41 @@ +package de.extio.lmlib.grader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.extio.lmlib.client.ClientService; +import de.extio.lmlib.profile.ModelCategory; + +public class Grader { + + private static final Logger LOGGER = LoggerFactory.getLogger(Grader.class); + + public static boolean assessScoreBinary(final String question, final String text, final ClientService clientService) { + LOGGER.info(question); + + final var client = clientService.getClient(ModelCategory.MEDIUM); + + var score = 0; + while (Math.abs(score) < 3) { + final var completion = client.completion( + ModelCategory.MEDIUM, + """ + You are a grader assessing the truthfulness of a given text to a user question. + Please provide a binary response 'true' or 'false' for the following text. + 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not. + Only provide the response as a single word and no preamble and no explanation.""", + "Here is the text:\n" + text, + "\nHere is the user question: " + question); + LOGGER.info(completion.response()); + score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1; + if (Math.abs(score) == 2) { + break; + } + } + + final var result = score > 0; + LOGGER.info("->" + String.valueOf(result)); + return result; + } + +} diff --git a/src/test/java/de/extio/lmlib/agent/AgentTest.java b/src/test/java/de/extio/lmlib/agent/AgentTest.java index 008226f..cb0f9f3 100644 --- a/src/test/java/de/extio/lmlib/agent/AgentTest.java +++ b/src/test/java/de/extio/lmlib/agent/AgentTest.java @@ -20,6 +20,7 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.PropertySource; import de.extio.lmlib.client.ClientService; +import de.extio.lmlib.grader.Grader; import de.extio.lmlib.profile.ModelCategory; import io.netty.util.internal.ThreadLocalRandom; @@ -114,7 +115,7 @@ public class AgentTest { assertEquals(1, resultContext.getContext().get("blogPost").size()); assertEquals(8, resultContext.getRequestStatistic().getRequests().get()); for (final var feature : resultContext.getContext().get("features")) { - assertTrue(this.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString())); + assertTrue(Grader.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString(), this.clientService)); } } @@ -197,32 +198,4 @@ public class AgentTest { } } - private boolean assessScoreBinary(final String question, final String text) { - LOGGER.info(question); - - final var client = this.clientService.getClient(ModelCategory.MEDIUM); - - var score = 0; - while (Math.abs(score) < 3) { - final var completion = client.completion( - ModelCategory.MEDIUM, - """ - You are a grader assessing the truthfulness of a given text to a user question. - Please provide a binary response 'true' or 'false' for the following text. - 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not. - Only provide the response as a single word and no preamble and no explanation.""", - "Here is the text:\n" + text, - "\nHere is the user question: " + question); - LOGGER.info(completion.response()); - score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1; - if (Math.abs(score) == 2) { - break; - } - } - - final var result = score > 0; - LOGGER.info("->" + String.valueOf(result)); - return result; - } - }