feat: Implement skipCache functionality across conversation handling
This commit is contained in:
@@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory;
|
||||
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.ClientService;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
|
||||
/**
|
||||
@@ -25,15 +26,17 @@ public class Grader {
|
||||
|
||||
var score = 0;
|
||||
while (Math.abs(score) < 3) {
|
||||
final var completion = client.completion(
|
||||
final var completion = client.conversation(
|
||||
ModelCategory.MEDIUM,
|
||||
"""
|
||||
You are a grader assessing the truthfulness of a given text to a user question.
|
||||
Please provide a binary response 'true' or 'false' for the following text.
|
||||
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
|
||||
Only provide the response as a single word and no preamble and no explanation.""",
|
||||
"Here is the text:\n" + text + "\n\n" +
|
||||
"Here is the user question: " + question);
|
||||
Conversation.create(
|
||||
"""
|
||||
You are a grader assessing the truthfulness of a given text to a user question.
|
||||
Please provide a binary response 'true' or 'false' for the following text.
|
||||
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
|
||||
Only provide the response as a single word and no preamble and no explanation.""",
|
||||
"Here is the text:\n" + text + "\n\n" +
|
||||
"Here is the user question: " + question),
|
||||
true);
|
||||
LOGGER.info(completion.response());
|
||||
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
|
||||
if (Math.abs(score) == 2) {
|
||||
|
||||
@@ -49,7 +49,8 @@ public class Grader2 {
|
||||
- 'true': The text provides a truthful answer.
|
||||
- 'false': The text does not provide a truthful answer.
|
||||
Only provide the response as a single word. Do not include any preamble or explanation.
|
||||
""".formatted(question, text)));
|
||||
""".formatted(question, text)),
|
||||
true);
|
||||
LOGGER.info(completion.response());
|
||||
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
|
||||
if (Math.abs(score) == 2) {
|
||||
|
||||
@@ -31,7 +31,9 @@ public class AnswerGrader {
|
||||
The answer is: {{answer}}
|
||||
""",
|
||||
new JsonAgentResponseHandler(),
|
||||
null,
|
||||
context -> {
|
||||
context.setSkipCache(true);
|
||||
},
|
||||
context -> {
|
||||
if (!context.getContext().containsKey("qaRating")) {
|
||||
context.setStringValue("qaRating", "0");
|
||||
|
||||
@@ -52,7 +52,8 @@ public class AnswerGrader2 {
|
||||
context.setStringValue("qaOutputFormat", "Provide no preamble. Return the response in JSON format with the following field only: {\"qaRating\": rating}");
|
||||
}
|
||||
context.getContext().remove("qaRating");
|
||||
context.getContext().remove("qaExplanation");
|
||||
context.getContext().remove("qaExplanation");
|
||||
context.setSkipCache(true);
|
||||
},
|
||||
context -> {
|
||||
if (!context.getContext().containsKey("qaRating")) {
|
||||
|
||||
@@ -44,6 +44,7 @@ public class AnswerGraderBinary {
|
||||
}
|
||||
context.getContext().remove("qaPassed");
|
||||
context.getContext().remove("qaExplanation");
|
||||
context.setSkipCache(true);
|
||||
},
|
||||
context -> {
|
||||
if (!context.getContext().containsKey("qaPassed")) {
|
||||
|
||||
Reference in New Issue
Block a user