diff --git a/pom.xml b/pom.xml
index 1a8b94f..0a43aec 100755
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
lm-lib
de.extio
lm-lib
- 1.5.4
+ 1.5.5
21
diff --git a/src/main/java/de/extio/lmlib/grader/Grader.java b/src/main/java/de/extio/lmlib/grader/Grader.java
new file mode 100644
index 0000000..3ab405b
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/grader/Grader.java
@@ -0,0 +1,41 @@
+package de.extio.lmlib.grader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.extio.lmlib.client.ClientService;
+import de.extio.lmlib.profile.ModelCategory;
+
+public class Grader {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(Grader.class);
+
+ public static boolean assessScoreBinary(final String question, final String text, final ClientService clientService) {
+ LOGGER.info(question);
+
+ final var client = clientService.getClient(ModelCategory.MEDIUM);
+
+ var score = 0;
+ while (Math.abs(score) < 3) {
+ final var completion = client.completion(
+ ModelCategory.MEDIUM,
+ """
+ You are a grader assessing the truthfulness of a given text to a user question.
+ Please provide a binary response 'true' or 'false' for the following text.
+ 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
+ Only provide the response as a single word and no preamble and no explanation.""",
+ "Here is the text:\n" + text,
+ "\nHere is the user question: " + question);
+ LOGGER.info(completion.response());
+ score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
+ if (Math.abs(score) == 2) {
+ break;
+ }
+ }
+
+ final var result = score > 0;
+ LOGGER.info("->" + String.valueOf(result));
+ return result;
+ }
+
+}
diff --git a/src/test/java/de/extio/lmlib/agent/AgentTest.java b/src/test/java/de/extio/lmlib/agent/AgentTest.java
index 008226f..cb0f9f3 100644
--- a/src/test/java/de/extio/lmlib/agent/AgentTest.java
+++ b/src/test/java/de/extio/lmlib/agent/AgentTest.java
@@ -20,6 +20,7 @@ import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.ClientService;
+import de.extio.lmlib.grader.Grader;
import de.extio.lmlib.profile.ModelCategory;
import io.netty.util.internal.ThreadLocalRandom;
@@ -114,7 +115,7 @@ public class AgentTest {
assertEquals(1, resultContext.getContext().get("blogPost").size());
assertEquals(8, resultContext.getRequestStatistic().getRequests().get());
for (final var feature : resultContext.getContext().get("features")) {
- assertTrue(this.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString()));
+ assertTrue(Grader.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString(), this.clientService));
}
}
@@ -197,32 +198,4 @@ public class AgentTest {
}
}
- private boolean assessScoreBinary(final String question, final String text) {
- LOGGER.info(question);
-
- final var client = this.clientService.getClient(ModelCategory.MEDIUM);
-
- var score = 0;
- while (Math.abs(score) < 3) {
- final var completion = client.completion(
- ModelCategory.MEDIUM,
- """
- You are a grader assessing the truthfulness of a given text to a user question.
- Please provide a binary response 'true' or 'false' for the following text.
- 'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
- Only provide the response as a single word and no preamble and no explanation.""",
- "Here is the text:\n" + text,
- "\nHere is the user question: " + question);
- LOGGER.info(completion.response());
- score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
- if (Math.abs(score) == 2) {
- break;
- }
- }
-
- final var result = score > 0;
- LOGGER.info("->" + String.valueOf(result));
- return result;
- }
-
}