From d2f5c46847ea69344bc69fcdbeb4d80ae7c63741 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sat, 13 Dec 2025 15:26:38 +0100 Subject: [PATCH] Convert spaces to tabs --- src/main/java/de/extio/lmlib/agent/Agent.java | 164 ++--- .../java/de/extio/lmlib/agent/AgentNext.java | 8 +- .../CsvAgentResponseHandler.java | 322 +++++----- .../CsvUnmarshallingAgentResponseHandler.java | 512 +++++++-------- .../StreamedAgentResponseHandler.java | 8 +- .../lmlib/client/CompletionFinishReason.java | 24 +- .../client/oai/completion/FinishReasons.java | 4 +- .../client/oai/completion/StreamOptions.java | 16 +- .../lmlib/client/oai/completion/Timings.java | 210 +++--- .../lmlib/client/oai/completion/Usage.java | 234 +++---- .../oai/completion/chat/ChatMessage.java | 84 +-- .../lmlib/grader/agent/AnswerGrader2.java | 4 +- .../grader/agent/AnswerGraderBinary.java | 6 +- .../lmlib/prompt/GptOssPromptStrategy.java | 202 +++--- .../extio/lmlib/prompt/NoPromptStrategy.java | 56 +- .../CsvAgentResponseHandlerTest.java | 608 +++++++++--------- ...UnmarshallingAgentResponseHandlerTest.java | 494 +++++++------- .../JsonAgentResponseHandlerTest.java | 2 +- 18 files changed, 1479 insertions(+), 1479 deletions(-) diff --git a/src/main/java/de/extio/lmlib/agent/Agent.java b/src/main/java/de/extio/lmlib/agent/Agent.java index 62c8908..4d16bde 100755 --- a/src/main/java/de/extio/lmlib/agent/Agent.java +++ b/src/main/java/de/extio/lmlib/agent/Agent.java @@ -9,89 +9,89 @@ import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfile; public record Agent(String name, - AgentType agentType, - ModelCategory modelCategory, - ModelProfile modelProfile, - String systemPrompt, - String textTemplate, - AgentResponseHandler responseHandler, - Consumer preProcessor, - Consumer postProcessor, - Function, - List> merger, - Function chooseNext) implements BaseAgent { - - public Agent { - if (name == null) { - throw new IllegalArgumentException("Name cannot be null"); - } - if (agentType == null) { - throw new IllegalArgumentException("Agent type cannot be null"); - } - } + AgentType agentType, + ModelCategory modelCategory, + ModelProfile modelProfile, + String systemPrompt, + String textTemplate, + AgentResponseHandler responseHandler, + Consumer preProcessor, + Consumer postProcessor, + Function, + List> merger, + Function chooseNext) implements BaseAgent { + + public Agent { + if (name == null) { + throw new IllegalArgumentException("Name cannot be null"); + } + if (agentType == null) { + throw new IllegalArgumentException("Agent type cannot be null"); + } + } - @Override - public AgentType agentType(final AgentContext context) { - if (agentType != null) { - return agentType; - } - return BaseAgent.super.agentType(context); - } - - @Override - public ModelCategory modelCategory(final AgentContext context) { - if (modelCategory != null) { - return modelCategory; - } - return BaseAgent.super.modelCategory(context); - } + @Override + public AgentType agentType(final AgentContext context) { + if (agentType != null) { + return agentType; + } + return BaseAgent.super.agentType(context); + } + + @Override + public ModelCategory modelCategory(final AgentContext context) { + if (modelCategory != null) { + return modelCategory; + } + return BaseAgent.super.modelCategory(context); + } - @Override - public ModelProfile modelProfile(final AgentContext context) { - if (modelProfile != null) { - return modelProfile; - } - return BaseAgent.super.modelProfile(context); - } - - @Override - public AgentResponseHandler responseHandler(final AgentContext context) { - if (responseHandler != null) { - return responseHandler; - } - return BaseAgent.super.responseHandler(context); - } + @Override + public ModelProfile modelProfile(final AgentContext context) { + if (modelProfile != null) { + return modelProfile; + } + return BaseAgent.super.modelProfile(context); + } + + @Override + public AgentResponseHandler responseHandler(final AgentContext context) { + if (responseHandler != null) { + return responseHandler; + } + return BaseAgent.super.responseHandler(context); + } - @Override - public void preProcess(final AgentContext context) { - if (preProcessor != null) { - preProcessor.accept(context); - } - BaseAgent.super.preProcess(context); - } - - @Override - public void postProcess(final AgentContext context) { - if (postProcessor != null) { - postProcessor.accept(context); - } - BaseAgent.super.postProcess(context); - } - - @Override - public List merge(final List contexts) { - if (merger != null) { - return merger.apply(contexts); - } - return BaseAgent.super.merge(contexts); - } - - @Override - public AgentNext chooseNextAgent(final AgentContext context) { - if (chooseNext != null) { - return chooseNext.apply(context); - } - return BaseAgent.super.chooseNextAgent(context); - } - + @Override + public void preProcess(final AgentContext context) { + if (preProcessor != null) { + preProcessor.accept(context); + } + BaseAgent.super.preProcess(context); + } + + @Override + public void postProcess(final AgentContext context) { + if (postProcessor != null) { + postProcessor.accept(context); + } + BaseAgent.super.postProcess(context); + } + + @Override + public List merge(final List contexts) { + if (merger != null) { + return merger.apply(contexts); + } + return BaseAgent.super.merge(contexts); + } + + @Override + public AgentNext chooseNextAgent(final AgentContext context) { + if (chooseNext != null) { + return chooseNext.apply(context); + } + return BaseAgent.super.chooseNextAgent(context); + } + } diff --git a/src/main/java/de/extio/lmlib/agent/AgentNext.java b/src/main/java/de/extio/lmlib/agent/AgentNext.java index bb22053..2cdc65c 100755 --- a/src/main/java/de/extio/lmlib/agent/AgentNext.java +++ b/src/main/java/de/extio/lmlib/agent/AgentNext.java @@ -2,9 +2,9 @@ package de.extio.lmlib.agent; public final record AgentNext(String name, Boolean gradingPassed) { - public static final AgentNext END = new AgentNext(null, null); + public static final AgentNext END = new AgentNext(null, null); - public static AgentNext end(final AgentContext context) { - return END; - } + public static AgentNext end(final AgentContext context) { + return END; + } } diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java index 98c6de6..5f5a5c2 100644 --- a/src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java @@ -12,165 +12,165 @@ import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.Conversation; public class CsvAgentResponseHandler implements AgentResponseHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(CsvAgentResponseHandler.class); - - private final List headings; - - private final String keyPrefix; - - private final String fieldSeparator; - - private final String fieldSeparatorQuoted; - - private final String errorPromptSuffix; - - public CsvAgentResponseHandler(final List headings, final String keyPrefix) { - this(headings, keyPrefix, ";"); - } - - public CsvAgentResponseHandler(final List headings, final String keyPrefix, final String fieldSeparator) { - if (headings == null || headings.isEmpty()) { - throw new IllegalArgumentException("Headings list cannot be null or empty"); - } - this.headings = List.copyOf(headings); - this.keyPrefix = keyPrefix != null ? keyPrefix : ""; - this.fieldSeparator = fieldSeparator; - this.fieldSeparatorQuoted = Pattern.quote(fieldSeparator); - - this.errorPromptSuffix = "\n\nThe previous response could not be fully processed or validated. " + - "Please format the response as CSV with '" + fieldSeparator + "' as field separator. " + - "Expected columns: " + String.join(", ", headings) + ". One row per line."; - } - - @Override - public boolean handle(final Completion completion, final AgentContext context) { - final String resp = completion.response(); - if (resp == null || resp.isBlank()) { - LOGGER.warn("Empty completion response for CSV parsing"); - return false; - } - - try { - final boolean success = this.parseCsv(resp, context); - if (!success) { - LOGGER.warn("No valid CSV data parsed from response"); - this.addCsvResponseErrorPrompt(context); - return false; - } - - return true; - } - catch (final Exception ex) { - LOGGER.warn("Failed to parse CSV response", ex); - this.addCsvResponseErrorPrompt(context); - return false; - } - } - - private boolean parseCsv(final String csvText, final AgentContext context) { - final String[] lines = csvText.split("\n"); - - int startLine = 0; - - for (int i = 0; i < lines.length; i++) { - final String line = lines[i].trim(); - - if (line.isEmpty()) { - continue; - } - - if (!line.contains(this.fieldSeparator)) { - startLine = i + 1; - continue; - } - - final String lowerLine = line.toLowerCase(); - boolean isHeader = false; - for (final String heading : this.headings) { - if (lowerLine.contains(heading.toLowerCase())) { - isHeader = true; - break; - } - } - - if (isHeader) { - startLine = i + 1; - continue; - } - - break; - } - - final List> columnData = new ArrayList<>(); - for (int i = 0; i < this.headings.size(); i++) { - columnData.add(new ArrayList<>()); - } - - int rowCount = 0; - for (int i = startLine; i < lines.length; i++) { - final String line = lines[i].trim(); - - if (line.isEmpty()) { - continue; - } - - if (!line.contains(this.fieldSeparator)) { - continue; - } - - try { - final String[] parts = line.split(this.fieldSeparatorQuoted, -1); - if (parts.length != this.headings.size()) { - LOGGER.debug("Column count mismatch: expected {}, got {} for line '{}'", - this.headings.size(), parts.length, line); - continue; - } - - for (int col = 0; col < parts.length; col++) { - final String value = this.stripQuotes(parts[col].trim()); - columnData.get(col).add(value); - } - rowCount++; - } - catch (final Exception ex) { - LOGGER.debug("Failed to parse line '{}': {}", line, ex.getMessage()); - } - } - - if (rowCount == 0) { - return false; - } - - for (int i = 0; i < this.headings.size(); i++) { - final String key = this.keyPrefix + this.headings.get(i); - context.setStringValues(key, columnData.get(i)); - LOGGER.debug("Stored {} values in context key '{}'", columnData.get(i).size(), key); - } - - LOGGER.debug("Parsed {} CSV rows into {} columns", rowCount, this.headings.size()); - return true; - } - - private void addCsvResponseErrorPrompt(final AgentContext context) { - final var turn = context.getConversation().getConversation().getLast(); - context.getConversation().replaceTurn(new Conversation.Turn( - turn.type(), - turn.text() + this.errorPromptSuffix)); - } - - private String stripQuotes(final String value) { - if (value == null || value.length() < 2) { - return value; - } - - if ((value.startsWith("\"") - && value.endsWith("\"")) - || (value.startsWith("'") - && value.endsWith("'"))) { - return value.substring(1, value.length() - 1); - } - - return value; - } + + private static final Logger LOGGER = LoggerFactory.getLogger(CsvAgentResponseHandler.class); + + private final List headings; + + private final String keyPrefix; + + private final String fieldSeparator; + + private final String fieldSeparatorQuoted; + + private final String errorPromptSuffix; + + public CsvAgentResponseHandler(final List headings, final String keyPrefix) { + this(headings, keyPrefix, ";"); + } + + public CsvAgentResponseHandler(final List headings, final String keyPrefix, final String fieldSeparator) { + if (headings == null || headings.isEmpty()) { + throw new IllegalArgumentException("Headings list cannot be null or empty"); + } + this.headings = List.copyOf(headings); + this.keyPrefix = keyPrefix != null ? keyPrefix : ""; + this.fieldSeparator = fieldSeparator; + this.fieldSeparatorQuoted = Pattern.quote(fieldSeparator); + + this.errorPromptSuffix = "\n\nThe previous response could not be fully processed or validated. " + + "Please format the response as CSV with '" + fieldSeparator + "' as field separator. " + + "Expected columns: " + String.join(", ", headings) + ". One row per line."; + } + + @Override + public boolean handle(final Completion completion, final AgentContext context) { + final String resp = completion.response(); + if (resp == null || resp.isBlank()) { + LOGGER.warn("Empty completion response for CSV parsing"); + return false; + } + + try { + final boolean success = this.parseCsv(resp, context); + if (!success) { + LOGGER.warn("No valid CSV data parsed from response"); + this.addCsvResponseErrorPrompt(context); + return false; + } + + return true; + } + catch (final Exception ex) { + LOGGER.warn("Failed to parse CSV response", ex); + this.addCsvResponseErrorPrompt(context); + return false; + } + } + + private boolean parseCsv(final String csvText, final AgentContext context) { + final String[] lines = csvText.split("\n"); + + int startLine = 0; + + for (int i = 0; i < lines.length; i++) { + final String line = lines[i].trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.contains(this.fieldSeparator)) { + startLine = i + 1; + continue; + } + + final String lowerLine = line.toLowerCase(); + boolean isHeader = false; + for (final String heading : this.headings) { + if (lowerLine.contains(heading.toLowerCase())) { + isHeader = true; + break; + } + } + + if (isHeader) { + startLine = i + 1; + continue; + } + + break; + } + + final List> columnData = new ArrayList<>(); + for (int i = 0; i < this.headings.size(); i++) { + columnData.add(new ArrayList<>()); + } + + int rowCount = 0; + for (int i = startLine; i < lines.length; i++) { + final String line = lines[i].trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.contains(this.fieldSeparator)) { + continue; + } + + try { + final String[] parts = line.split(this.fieldSeparatorQuoted, -1); + if (parts.length != this.headings.size()) { + LOGGER.debug("Column count mismatch: expected {}, got {} for line '{}'", + this.headings.size(), parts.length, line); + continue; + } + + for (int col = 0; col < parts.length; col++) { + final String value = this.stripQuotes(parts[col].trim()); + columnData.get(col).add(value); + } + rowCount++; + } + catch (final Exception ex) { + LOGGER.debug("Failed to parse line '{}': {}", line, ex.getMessage()); + } + } + + if (rowCount == 0) { + return false; + } + + for (int i = 0; i < this.headings.size(); i++) { + final String key = this.keyPrefix + this.headings.get(i); + context.setStringValues(key, columnData.get(i)); + LOGGER.debug("Stored {} values in context key '{}'", columnData.get(i).size(), key); + } + + LOGGER.debug("Parsed {} CSV rows into {} columns", rowCount, this.headings.size()); + return true; + } + + private void addCsvResponseErrorPrompt(final AgentContext context) { + final var turn = context.getConversation().getConversation().getLast(); + context.getConversation().replaceTurn(new Conversation.Turn( + turn.type(), + turn.text() + this.errorPromptSuffix)); + } + + private String stripQuotes(final String value) { + if (value == null || value.length() < 2) { + return value; + } + + if ((value.startsWith("\"") + && value.endsWith("\"")) + || (value.startsWith("'") + && value.endsWith("'"))) { + return value.substring(1, value.length() - 1); + } + + return value; + } } diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandler.java index 2c8b6ab..a550a53 100644 --- a/src/main/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandler.java +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandler.java @@ -17,260 +17,260 @@ import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.Conversation; public class CsvUnmarshallingAgentResponseHandler implements AgentResponseHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(CsvUnmarshallingAgentResponseHandler.class); - - private final Class targetClass; - - private final String contextKey; - - private final String fieldSeparator; - - private final String listSeparator; - - private final Field[] fields; - - private final List fieldNames; - - private final Constructor constructor; - - private final String fieldSeparatorQuoted; - - private final String listSeparatorQuoted; - - private final String errorPromptSuffix; - - public CsvUnmarshallingAgentResponseHandler(final Class targetClass, final String contextKey) { - this(targetClass, contextKey, ";", ","); - } - - public CsvUnmarshallingAgentResponseHandler(final Class targetClass, final String contextKey, - final String fieldSeparator, final String listSeparator) { - this.targetClass = targetClass; - this.contextKey = contextKey; - this.fieldSeparator = fieldSeparator; - this.listSeparator = listSeparator; - - // Pre-compute static data - this.fields = targetClass.getDeclaredFields(); - this.fieldNames = Arrays.stream(this.fields) - .map(Field::getName) - .toList(); - this.constructor = this.findRecordConstructor(); - if (this.constructor == null) { - throw new IllegalStateException("No suitable constructor found for " + targetClass.getName()); - } - this.fieldSeparatorQuoted = Pattern.quote(fieldSeparator); - this.listSeparatorQuoted = (listSeparator != null && !listSeparator.isEmpty()) - ? Pattern.quote(listSeparator) - : null; - - // Pre-compute error message suffix - final String listInfo = (listSeparator != null && !listSeparator.isEmpty()) - ? " and '" + listSeparator + "' for list items" - : ""; - this.errorPromptSuffix = "\n\nThe previous response could not be fully processed or validated. " + - "Please format the response as CSV with '" + fieldSeparator + "' as field separator" + - listInfo + ". One object per line."; - } - - private Constructor findRecordConstructor() { - final Field[] targetFields = this.targetClass.getDeclaredFields(); - for (final Constructor ctor : this.targetClass.getDeclaredConstructors()) { - if (ctor.getParameterCount() == targetFields.length) { - return ctor; - } - } - return null; - } - - @Override - public boolean handle(final Completion completion, final AgentContext context) { - final String resp = completion.response(); - if (resp == null || resp.isBlank()) { - LOGGER.warn("Empty completion response for CSV parsing"); - return false; - } - - try { - final List parsedObjects = this.parseCsv(resp); - if (parsedObjects.isEmpty()) { - LOGGER.warn("No valid CSV data parsed from response"); - this.addCsvResponseErrorPrompt(context); - return false; - } - - context.setValues(this.contextKey, parsedObjects); - LOGGER.debug("Parsed {} CSV objects", parsedObjects.size()); - return true; - } - catch (final Exception ex) { - LOGGER.warn("Failed to parse CSV response", ex); - this.addCsvResponseErrorPrompt(context); - return false; - } - } - - private List parseCsv(final String csvText) throws Exception { - final List result = new ArrayList<>(); - final String[] lines = csvText.split("\n"); - - int startLine = 0; - - for (int i = 0; i < lines.length; i++) { - final String line = lines[i].trim(); - - if (line.isEmpty()) { - continue; - } - - if (!line.contains(this.fieldSeparator)) { - startLine = i + 1; - continue; - } - - final String lowerLine = line.toLowerCase(); - boolean isHeader = false; - for (final String fieldName : this.fieldNames) { - if (lowerLine.contains(fieldName.toLowerCase())) { - isHeader = true; - break; - } - } - - if (isHeader) { - startLine = i + 1; - continue; - } - - break; - } - - for (int i = startLine; i < lines.length; i++) { - final String line = lines[i].trim(); - - if (line.isEmpty()) { - continue; - } - - if (!line.contains(this.fieldSeparator)) { - continue; - } - - try { - final Object obj = this.parseLine(line); - if (obj != null) { - result.add(obj); - LOGGER.debug("Parsed CSV object: {}", obj); - } - } - catch (final Exception ex) { - LOGGER.debug("Failed to parse line '{}': {}", line, ex.getMessage()); - } - } - - return result; - } - - private Object parseLine(final String line) throws Exception { - final String[] parts = line.split(this.fieldSeparatorQuoted, -1); - if (parts.length != this.fields.length) { - LOGGER.debug("Field count mismatch: expected {}, got {} for line '{}'", - this.fields.length, parts.length, line); - return null; - } - - final Object[] args = new Object[this.fields.length]; - - for (int i = 0; i < this.fields.length; i++) { - final String value = parts[i].trim(); - args[i] = this.parseValue(value, this.fields[i]); - } - - return this.constructor.newInstance(args); - } - - private Object parseValue(final String value, final Field field) { - final String cleanValue = this.stripQuotes(value); - final Class fieldType = field.getType(); - - if (cleanValue.isEmpty() || cleanValue.equalsIgnoreCase("null")) { - if (fieldType == List.class) { - return new ArrayList<>(); - } - return null; - } - - if (fieldType == String.class) { - return cleanValue; - } - - if (fieldType == Integer.class || fieldType == int.class) { - return Integer.parseInt(cleanValue); - } - - if (fieldType == Long.class || fieldType == long.class) { - return Long.parseLong(cleanValue); - } - - if (fieldType == Double.class || fieldType == double.class) { - return Double.parseDouble(cleanValue); - } - - if (fieldType == Boolean.class || fieldType == boolean.class) { - return Boolean.parseBoolean(cleanValue); - } - - if (fieldType == List.class) { - if (this.listSeparatorQuoted == null) { - return new ArrayList<>(); - } - - final Type genericType = field.getGenericType(); - if (genericType instanceof ParameterizedType) { - final ParameterizedType paramType = (ParameterizedType) genericType; - final Type[] typeArgs = paramType.getActualTypeArguments(); - - if (typeArgs.length > 0 && typeArgs[0] == String.class) { - if (cleanValue.isEmpty()) { - return new ArrayList(); - } - - final String[] items = cleanValue.split(this.listSeparatorQuoted); - final List list = new ArrayList<>(); - for (final String item : items) { - final String trimmed = item.trim(); - if (!trimmed.isEmpty()) { - list.add(trimmed); - } - } - return list; - } - } - } - - throw new IllegalArgumentException("Unsupported field type: " + fieldType.getName()); - } - - private void addCsvResponseErrorPrompt(final AgentContext context) { - final var turn = context.getConversation().getConversation().getLast(); - context.getConversation().replaceTurn(new Conversation.Turn( - turn.type(), - turn.text() + this.errorPromptSuffix)); - } - - private String stripQuotes(final String value) { - if (value == null || value.length() < 2) { - return value; - } - - if ((value.startsWith("\"") - && value.endsWith("\"")) - || (value.startsWith("'") - && value.endsWith("'"))) { - return value.substring(1, value.length() - 1); - } - - return value; - } + + private static final Logger LOGGER = LoggerFactory.getLogger(CsvUnmarshallingAgentResponseHandler.class); + + private final Class targetClass; + + private final String contextKey; + + private final String fieldSeparator; + + private final String listSeparator; + + private final Field[] fields; + + private final List fieldNames; + + private final Constructor constructor; + + private final String fieldSeparatorQuoted; + + private final String listSeparatorQuoted; + + private final String errorPromptSuffix; + + public CsvUnmarshallingAgentResponseHandler(final Class targetClass, final String contextKey) { + this(targetClass, contextKey, ";", ","); + } + + public CsvUnmarshallingAgentResponseHandler(final Class targetClass, final String contextKey, + final String fieldSeparator, final String listSeparator) { + this.targetClass = targetClass; + this.contextKey = contextKey; + this.fieldSeparator = fieldSeparator; + this.listSeparator = listSeparator; + + // Pre-compute static data + this.fields = targetClass.getDeclaredFields(); + this.fieldNames = Arrays.stream(this.fields) + .map(Field::getName) + .toList(); + this.constructor = this.findRecordConstructor(); + if (this.constructor == null) { + throw new IllegalStateException("No suitable constructor found for " + targetClass.getName()); + } + this.fieldSeparatorQuoted = Pattern.quote(fieldSeparator); + this.listSeparatorQuoted = (listSeparator != null && !listSeparator.isEmpty()) + ? Pattern.quote(listSeparator) + : null; + + // Pre-compute error message suffix + final String listInfo = (listSeparator != null && !listSeparator.isEmpty()) + ? " and '" + listSeparator + "' for list items" + : ""; + this.errorPromptSuffix = "\n\nThe previous response could not be fully processed or validated. " + + "Please format the response as CSV with '" + fieldSeparator + "' as field separator" + + listInfo + ". One object per line."; + } + + private Constructor findRecordConstructor() { + final Field[] targetFields = this.targetClass.getDeclaredFields(); + for (final Constructor ctor : this.targetClass.getDeclaredConstructors()) { + if (ctor.getParameterCount() == targetFields.length) { + return ctor; + } + } + return null; + } + + @Override + public boolean handle(final Completion completion, final AgentContext context) { + final String resp = completion.response(); + if (resp == null || resp.isBlank()) { + LOGGER.warn("Empty completion response for CSV parsing"); + return false; + } + + try { + final List parsedObjects = this.parseCsv(resp); + if (parsedObjects.isEmpty()) { + LOGGER.warn("No valid CSV data parsed from response"); + this.addCsvResponseErrorPrompt(context); + return false; + } + + context.setValues(this.contextKey, parsedObjects); + LOGGER.debug("Parsed {} CSV objects", parsedObjects.size()); + return true; + } + catch (final Exception ex) { + LOGGER.warn("Failed to parse CSV response", ex); + this.addCsvResponseErrorPrompt(context); + return false; + } + } + + private List parseCsv(final String csvText) throws Exception { + final List result = new ArrayList<>(); + final String[] lines = csvText.split("\n"); + + int startLine = 0; + + for (int i = 0; i < lines.length; i++) { + final String line = lines[i].trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.contains(this.fieldSeparator)) { + startLine = i + 1; + continue; + } + + final String lowerLine = line.toLowerCase(); + boolean isHeader = false; + for (final String fieldName : this.fieldNames) { + if (lowerLine.contains(fieldName.toLowerCase())) { + isHeader = true; + break; + } + } + + if (isHeader) { + startLine = i + 1; + continue; + } + + break; + } + + for (int i = startLine; i < lines.length; i++) { + final String line = lines[i].trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.contains(this.fieldSeparator)) { + continue; + } + + try { + final Object obj = this.parseLine(line); + if (obj != null) { + result.add(obj); + LOGGER.debug("Parsed CSV object: {}", obj); + } + } + catch (final Exception ex) { + LOGGER.debug("Failed to parse line '{}': {}", line, ex.getMessage()); + } + } + + return result; + } + + private Object parseLine(final String line) throws Exception { + final String[] parts = line.split(this.fieldSeparatorQuoted, -1); + if (parts.length != this.fields.length) { + LOGGER.debug("Field count mismatch: expected {}, got {} for line '{}'", + this.fields.length, parts.length, line); + return null; + } + + final Object[] args = new Object[this.fields.length]; + + for (int i = 0; i < this.fields.length; i++) { + final String value = parts[i].trim(); + args[i] = this.parseValue(value, this.fields[i]); + } + + return this.constructor.newInstance(args); + } + + private Object parseValue(final String value, final Field field) { + final String cleanValue = this.stripQuotes(value); + final Class fieldType = field.getType(); + + if (cleanValue.isEmpty() || cleanValue.equalsIgnoreCase("null")) { + if (fieldType == List.class) { + return new ArrayList<>(); + } + return null; + } + + if (fieldType == String.class) { + return cleanValue; + } + + if (fieldType == Integer.class || fieldType == int.class) { + return Integer.parseInt(cleanValue); + } + + if (fieldType == Long.class || fieldType == long.class) { + return Long.parseLong(cleanValue); + } + + if (fieldType == Double.class || fieldType == double.class) { + return Double.parseDouble(cleanValue); + } + + if (fieldType == Boolean.class || fieldType == boolean.class) { + return Boolean.parseBoolean(cleanValue); + } + + if (fieldType == List.class) { + if (this.listSeparatorQuoted == null) { + return new ArrayList<>(); + } + + final Type genericType = field.getGenericType(); + if (genericType instanceof ParameterizedType) { + final ParameterizedType paramType = (ParameterizedType) genericType; + final Type[] typeArgs = paramType.getActualTypeArguments(); + + if (typeArgs.length > 0 && typeArgs[0] == String.class) { + if (cleanValue.isEmpty()) { + return new ArrayList(); + } + + final String[] items = cleanValue.split(this.listSeparatorQuoted); + final List list = new ArrayList<>(); + for (final String item : items) { + final String trimmed = item.trim(); + if (!trimmed.isEmpty()) { + list.add(trimmed); + } + } + return list; + } + } + } + + throw new IllegalArgumentException("Unsupported field type: " + fieldType.getName()); + } + + private void addCsvResponseErrorPrompt(final AgentContext context) { + final var turn = context.getConversation().getConversation().getLast(); + context.getConversation().replaceTurn(new Conversation.Turn( + turn.type(), + turn.text() + this.errorPromptSuffix)); + } + + private String stripQuotes(final String value) { + if (value == null || value.length() < 2) { + return value; + } + + if ((value.startsWith("\"") + && value.endsWith("\"")) + || (value.startsWith("'") + && value.endsWith("'"))) { + return value.substring(1, value.length() - 1); + } + + return value; + } } diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java index f58cd24..145ca71 100644 --- a/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/StreamedAgentResponseHandler.java @@ -4,9 +4,9 @@ import de.extio.lmlib.agent.AgentContext; import de.extio.lmlib.client.Chunk; public interface StreamedAgentResponseHandler extends AgentResponseHandler { - - void beforeStream(AgentContext context); + + void beforeStream(AgentContext context); - boolean handleChunk(Chunk chunk, AgentContext context); - + boolean handleChunk(Chunk chunk, AgentContext context); + } diff --git a/src/main/java/de/extio/lmlib/client/CompletionFinishReason.java b/src/main/java/de/extio/lmlib/client/CompletionFinishReason.java index be8d271..136b58c 100644 --- a/src/main/java/de/extio/lmlib/client/CompletionFinishReason.java +++ b/src/main/java/de/extio/lmlib/client/CompletionFinishReason.java @@ -2,19 +2,19 @@ package de.extio.lmlib.client; public enum CompletionFinishReason { - /** - * Regular completion - */ - DONE, + /** + * Regular completion + */ + DONE, - /** - * Completion was stopped because the token limit was reached. You may continue the conversation. - */ - TOKEN_LIMIT_REACHED, + /** + * Completion was stopped because the token limit was reached. You may continue the conversation. + */ + TOKEN_LIMIT_REACHED, - /** - * Completion was stopped because the content was filtered. - */ - CONTENT_FILTERED + /** + * Completion was stopped because the content was filtered. + */ + CONTENT_FILTERED } \ No newline at end of file diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/FinishReasons.java b/src/main/java/de/extio/lmlib/client/oai/completion/FinishReasons.java index 3ddf153..3118afd 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/FinishReasons.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/FinishReasons.java @@ -2,10 +2,10 @@ package de.extio.lmlib.client.oai.completion; public final class FinishReasons { - public static final String FINISH_REASON_STOP = "stop"; + public static final String FINISH_REASON_STOP = "stop"; public static final String FINISH_REASON_LENGTH = "length"; public static final String FINISH_REASON_CONTENT_FILTER = "content_filter"; - + } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java b/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java index b9da1ed..465a641 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/StreamOptions.java @@ -4,14 +4,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class StreamOptions { - protected boolean includeUsage; + protected boolean includeUsage; - @JsonProperty("include_usage") - public boolean isIncludeUsage() { - return includeUsage; - } + @JsonProperty("include_usage") + public boolean isIncludeUsage() { + return includeUsage; + } - public void setIncludeUsage(final boolean includeUsage) { - this.includeUsage = includeUsage; - } + public void setIncludeUsage(final boolean includeUsage) { + this.includeUsage = includeUsage; + } } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java b/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java index 0ddfe0c..0b3e004 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/Timings.java @@ -3,109 +3,109 @@ package de.extio.lmlib.client.oai.completion; import com.fasterxml.jackson.annotation.JsonProperty; public final class Timings { - - private int cacheN; - - private int predictedN; - - private double predictedMs; - - private double predictedPerTokenMs; - - private double predictedPerSecond; - - private int promptN; - - private double promptMs; - - private double promptPerTokenMs; - - private double promptPerSecond; - - @JsonProperty("cache_n") - public int getCacheN() { - return this.cacheN; - } - - public void setCacheN(final int cacheN) { - this.cacheN = cacheN; - } - - @JsonProperty("predicted_n") - public int getPredictedN() { - return this.predictedN; - } - - public void setPredictedN(final int predictedN) { - this.predictedN = predictedN; - } - - @JsonProperty("predicted_ms") - public double getPredictedMs() { - return this.predictedMs; - } - - public void setPredictedMs(final double predictedMs) { - this.predictedMs = predictedMs; - } - - @JsonProperty("predicted_per_token_ms") - public double getPredictedPerTokenMs() { - return this.predictedPerTokenMs; - } - - public void setPredictedPerTokenMs(final double predictedPerTokenMs) { - this.predictedPerTokenMs = predictedPerTokenMs; - } - - @JsonProperty("predicted_per_second") - public double getPredictedPerSecond() { - return this.predictedPerSecond; - } - - public void setPredictedPerSecond(final double predictedPerSecond) { - this.predictedPerSecond = predictedPerSecond; - } - - @JsonProperty("prompt_n") - public int getPromptN() { - return this.promptN; - } - - public void setPromptN(final int promptN) { - this.promptN = promptN; - } - - @JsonProperty("prompt_ms") - public double getPromptMs() { - return this.promptMs; - } - - public void setPromptMs(final double promptMs) { - this.promptMs = promptMs; - } - - @JsonProperty("prompt_per_token_ms") - public double getPromptPerTokenMs() { - return this.promptPerTokenMs; - } - - public void setPromptPerTokenMs(final double promptPerTokenMs) { - this.promptPerTokenMs = promptPerTokenMs; - } - - @JsonProperty("prompt_per_second") - public double getPromptPerSecond() { - return this.promptPerSecond; - } - - public void setPromptPerSecond(final double promptPerSecond) { - this.promptPerSecond = promptPerSecond; - } - - @Override - public String toString() { - return "Timings [cacheN=" + this.cacheN + ", predictedN=" + this.predictedN + ", predictedMs=" + this.predictedMs + ", predictedPerTokenMs=" + this.predictedPerTokenMs + ", predictedPerSecond=" + this.predictedPerSecond + ", promptN=" + this.promptN + ", promptMs=" + this.promptMs + ", promptPerTokenMs=" + this.promptPerTokenMs + ", promptPerSecond=" + this.promptPerSecond + "]"; - } - + + private int cacheN; + + private int predictedN; + + private double predictedMs; + + private double predictedPerTokenMs; + + private double predictedPerSecond; + + private int promptN; + + private double promptMs; + + private double promptPerTokenMs; + + private double promptPerSecond; + + @JsonProperty("cache_n") + public int getCacheN() { + return this.cacheN; + } + + public void setCacheN(final int cacheN) { + this.cacheN = cacheN; + } + + @JsonProperty("predicted_n") + public int getPredictedN() { + return this.predictedN; + } + + public void setPredictedN(final int predictedN) { + this.predictedN = predictedN; + } + + @JsonProperty("predicted_ms") + public double getPredictedMs() { + return this.predictedMs; + } + + public void setPredictedMs(final double predictedMs) { + this.predictedMs = predictedMs; + } + + @JsonProperty("predicted_per_token_ms") + public double getPredictedPerTokenMs() { + return this.predictedPerTokenMs; + } + + public void setPredictedPerTokenMs(final double predictedPerTokenMs) { + this.predictedPerTokenMs = predictedPerTokenMs; + } + + @JsonProperty("predicted_per_second") + public double getPredictedPerSecond() { + return this.predictedPerSecond; + } + + public void setPredictedPerSecond(final double predictedPerSecond) { + this.predictedPerSecond = predictedPerSecond; + } + + @JsonProperty("prompt_n") + public int getPromptN() { + return this.promptN; + } + + public void setPromptN(final int promptN) { + this.promptN = promptN; + } + + @JsonProperty("prompt_ms") + public double getPromptMs() { + return this.promptMs; + } + + public void setPromptMs(final double promptMs) { + this.promptMs = promptMs; + } + + @JsonProperty("prompt_per_token_ms") + public double getPromptPerTokenMs() { + return this.promptPerTokenMs; + } + + public void setPromptPerTokenMs(final double promptPerTokenMs) { + this.promptPerTokenMs = promptPerTokenMs; + } + + @JsonProperty("prompt_per_second") + public double getPromptPerSecond() { + return this.promptPerSecond; + } + + public void setPromptPerSecond(final double promptPerSecond) { + this.promptPerSecond = promptPerSecond; + } + + @Override + public String toString() { + return "Timings [cacheN=" + this.cacheN + ", predictedN=" + this.predictedN + ", predictedMs=" + this.predictedMs + ", predictedPerTokenMs=" + this.predictedPerTokenMs + ", predictedPerSecond=" + this.predictedPerSecond + ", promptN=" + this.promptN + ", promptMs=" + this.promptMs + ", promptPerTokenMs=" + this.promptPerTokenMs + ", promptPerSecond=" + this.promptPerSecond + "]"; + } + } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java index 6911a70..27419fe 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java @@ -3,121 +3,121 @@ package de.extio.lmlib.client.oai.completion; import com.fasterxml.jackson.annotation.JsonProperty; public final class Usage { - - private int completionTokens; - - private int promptTokens; - - private int totalTokens; - - private InputTokensDetails inputTokensDetails; - - private OutputTokensDetails outputTokensDetails; - - @JsonProperty("completion_tokens") - public int getCompletionTokens() { - return this.completionTokens; - } - - public void setCompletionTokens(final int completionTokens) { - this.completionTokens = completionTokens; - } - - @JsonProperty("output_tokens") - public void setOutputTokens(final int outputTokens) { - this.completionTokens = outputTokens; - } - - @JsonProperty("prompt_tokens") - public int getPromptTokens() { - return this.promptTokens; - } - - public void setPromptTokens(final int promptTokens) { - this.promptTokens = promptTokens; - } - - @JsonProperty("input_tokens") - public void setInputTokens(final int inputTokens) { - this.promptTokens = inputTokens; - } - - @JsonProperty("total_tokens") - public int getTotalTokens() { - return this.totalTokens; - } - - public void setTotalTokens(final int totalTokens) { - this.totalTokens = totalTokens; - } - - @JsonProperty("input_tokens_details") - public InputTokensDetails getInputTokensDetails() { - return this.inputTokensDetails; - } - - public void setInputTokensDetails(final InputTokensDetails inputTokensDetails) { - this.inputTokensDetails = inputTokensDetails; - } - - @JsonProperty("output_tokens_details") - public OutputTokensDetails getOutputTokensDetails() { - return this.outputTokensDetails; - } - - public void setOutputTokensDetails(final OutputTokensDetails outputTokensDetails) { - this.outputTokensDetails = outputTokensDetails; - } - - public int getCachedTokens() { - return this.inputTokensDetails != null ? this.inputTokensDetails.getCachedTokens() : 0; - } - - public int getReasoningTokens() { - return this.outputTokensDetails != null ? this.outputTokensDetails.getReasoningTokens() : 0; - } - - @Override - public String toString() { - return "Usage [completionTokens=" + this.completionTokens + ", promptTokens=" + this.promptTokens + ", totalTokens=" + this.totalTokens + ", inputTokensDetails=" + this.inputTokensDetails + ", outputTokensDetails=" + this.outputTokensDetails + "]"; - } - - public static final class InputTokensDetails { - - private int cachedTokens; - - @JsonProperty("cached_tokens") - public int getCachedTokens() { - return this.cachedTokens; - } - - public void setCachedTokens(final int cachedTokens) { - this.cachedTokens = cachedTokens; - } - - @Override - public String toString() { - return "InputTokensDetails [cachedTokens=" + this.cachedTokens + "]"; - } - } - - public static final class OutputTokensDetails { - - private int reasoningTokens; - - @JsonProperty("reasoning_tokens") - public int getReasoningTokens() { - return this.reasoningTokens; - } - - public void setReasoningTokens(final int reasoningTokens) { - this.reasoningTokens = reasoningTokens; - } - - @Override - public String toString() { - return "OutputTokensDetails [reasoningTokens=" + this.reasoningTokens + "]"; - } - } - + + private int completionTokens; + + private int promptTokens; + + private int totalTokens; + + private InputTokensDetails inputTokensDetails; + + private OutputTokensDetails outputTokensDetails; + + @JsonProperty("completion_tokens") + public int getCompletionTokens() { + return this.completionTokens; + } + + public void setCompletionTokens(final int completionTokens) { + this.completionTokens = completionTokens; + } + + @JsonProperty("output_tokens") + public void setOutputTokens(final int outputTokens) { + this.completionTokens = outputTokens; + } + + @JsonProperty("prompt_tokens") + public int getPromptTokens() { + return this.promptTokens; + } + + public void setPromptTokens(final int promptTokens) { + this.promptTokens = promptTokens; + } + + @JsonProperty("input_tokens") + public void setInputTokens(final int inputTokens) { + this.promptTokens = inputTokens; + } + + @JsonProperty("total_tokens") + public int getTotalTokens() { + return this.totalTokens; + } + + public void setTotalTokens(final int totalTokens) { + this.totalTokens = totalTokens; + } + + @JsonProperty("input_tokens_details") + public InputTokensDetails getInputTokensDetails() { + return this.inputTokensDetails; + } + + public void setInputTokensDetails(final InputTokensDetails inputTokensDetails) { + this.inputTokensDetails = inputTokensDetails; + } + + @JsonProperty("output_tokens_details") + public OutputTokensDetails getOutputTokensDetails() { + return this.outputTokensDetails; + } + + public void setOutputTokensDetails(final OutputTokensDetails outputTokensDetails) { + this.outputTokensDetails = outputTokensDetails; + } + + public int getCachedTokens() { + return this.inputTokensDetails != null ? this.inputTokensDetails.getCachedTokens() : 0; + } + + public int getReasoningTokens() { + return this.outputTokensDetails != null ? this.outputTokensDetails.getReasoningTokens() : 0; + } + + @Override + public String toString() { + return "Usage [completionTokens=" + this.completionTokens + ", promptTokens=" + this.promptTokens + ", totalTokens=" + this.totalTokens + ", inputTokensDetails=" + this.inputTokensDetails + ", outputTokensDetails=" + this.outputTokensDetails + "]"; + } + + public static final class InputTokensDetails { + + private int cachedTokens; + + @JsonProperty("cached_tokens") + public int getCachedTokens() { + return this.cachedTokens; + } + + public void setCachedTokens(final int cachedTokens) { + this.cachedTokens = cachedTokens; + } + + @Override + public String toString() { + return "InputTokensDetails [cachedTokens=" + this.cachedTokens + "]"; + } + } + + public static final class OutputTokensDetails { + + private int reasoningTokens; + + @JsonProperty("reasoning_tokens") + public int getReasoningTokens() { + return this.reasoningTokens; + } + + public void setReasoningTokens(final int reasoningTokens) { + this.reasoningTokens = reasoningTokens; + } + + @Override + public String toString() { + return "OutputTokensDetails [reasoningTokens=" + this.reasoningTokens + "]"; + } + } + } diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java index c15f1b1..c33d684 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java @@ -3,46 +3,46 @@ package de.extio.lmlib.client.oai.completion.chat; import com.fasterxml.jackson.annotation.JsonProperty; final class ChatMessage { - - private String role; - - private String content; - - private String reasoningContent; - - public ChatMessage() { - } - - public ChatMessage(final String role, final String content, final String reasoningContent) { - this.role = role; - this.content = content; - this.reasoningContent = reasoningContent; - } - - @JsonProperty("role") - public String getRole() { - return role; - } - - public void setRole(final String role) { - this.role = role; - } - - @JsonProperty("content") - public String getContent() { - return content; - } - - public void setContent(final String content) { - this.content = content; - } - - @JsonProperty("reasoning_content") - public String getReasoningContent() { - return reasoningContent; - } - - public void setReasoningContent(final String reasoningContent) { - this.reasoningContent = reasoningContent; - } + + private String role; + + private String content; + + private String reasoningContent; + + public ChatMessage() { + } + + public ChatMessage(final String role, final String content, final String reasoningContent) { + this.role = role; + this.content = content; + this.reasoningContent = reasoningContent; + } + + @JsonProperty("role") + public String getRole() { + return role; + } + + public void setRole(final String role) { + this.role = role; + } + + @JsonProperty("content") + public String getContent() { + return content; + } + + public void setContent(final String content) { + this.content = content; + } + + @JsonProperty("reasoning_content") + public String getReasoningContent() { + return reasoningContent; + } + + public void setReasoningContent(final String reasoningContent) { + this.reasoningContent = reasoningContent; + } } diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java index 801a568..aa2daeb 100755 --- a/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGrader2.java @@ -51,8 +51,8 @@ public class AnswerGrader2 { else { context.setStringValue("qaOutputFormat", "Provide no preamble. Return the response in JSON format with the following field only: {\"qaRating\": rating}"); } - context.getContext().remove("qaRating"); - context.getContext().remove("qaExplanation"); + context.getContext().remove("qaRating"); + context.getContext().remove("qaExplanation"); }, context -> { if (!context.getContext().containsKey("qaRating")) { diff --git a/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java b/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java index f9fc3e0..1322938 100644 --- a/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java +++ b/src/main/java/de/extio/lmlib/grader/agent/AnswerGraderBinary.java @@ -42,8 +42,8 @@ public class AnswerGraderBinary { else { context.setStringValue("qaOutputFormat", "Return a JSON object with exactly this key: {\"qaPassed\": boolean}"); } - context.getContext().remove("qaPassed"); - context.getContext().remove("qaExplanation"); + context.getContext().remove("qaPassed"); + context.getContext().remove("qaExplanation"); }, context -> { if (!context.getContext().containsKey("qaPassed")) { @@ -70,7 +70,7 @@ public class AnswerGraderBinary { context.getContext().put("qaPassed", List.of(qaPassed)); if (!qaPassed && (int) context.getContext().get("qaIteration").get(0) < 2) { - context.setStringValue("qaFeedback", "Feedback from the previous attempt to consider: " + context.getStringValue("qaExplanation")); + context.setStringValue("qaFeedback", "Feedback from the previous attempt to consider: " + context.getStringValue("qaExplanation")); return new AgentNext(context.getStringValue("qaFeedbackLoopDestination"), Boolean.FALSE); } diff --git a/src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java b/src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java index 5a4052e..6f69b49 100644 --- a/src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java +++ b/src/main/java/de/extio/lmlib/prompt/GptOssPromptStrategy.java @@ -4,107 +4,107 @@ import org.springframework.stereotype.Component; @Component public class GptOssPromptStrategy implements PromptStrategy { - - private static final String finalMarker = "<|channel|>final<|message|>"; - - private static final String startMarker = "<|start|>"; + + private static final String finalMarker = "<|channel|>final<|message|>"; + + private static final String startMarker = "<|start|>"; - private static final String assistantStart = startMarker + "assistant"; - - private static final String analysisMarker = "<|channel|>analysis<|message|>"; + private static final String assistantStart = startMarker + "assistant"; + + private static final String analysisMarker = "<|channel|>analysis<|message|>"; - @Override - public StringBuilder start(final String system, final String user) { - final StringBuilder prompt = new StringBuilder(); - if (!system.isEmpty()) { - prompt.append("<|start|>system<|message|>"); - prompt.append(system); - prompt.append("<|end|>"); - } - prompt.append("<|start|>user<|message|>"); - prompt.append(user); - prompt.append("<|end|>"); - prompt.append(assistantStart); - - return prompt; - } - - @Override - public void next(final StringBuilder prompt, final String assistant, final String user) { - final String assistantContent = assistant == null ? "" : assistant; - if (assistantContent.startsWith(assistantStart)) { - final int assistantStartLength = assistantStart.length(); - if (prompt.length() >= assistantStartLength) { - boolean endsWithAssistantStart = true; - for (int i = 0; i < assistantStartLength; i++) { - if (prompt.charAt(prompt.length() - assistantStartLength + i) != assistantStart.charAt(i)) { - endsWithAssistantStart = false; - break; - } - } - if (endsWithAssistantStart) { - prompt.setLength(prompt.length() - assistantStartLength); - } - } - } - if (assistantContent.contains(finalMarker)) { - prompt.append(assistantContent); - if (!assistantContent.endsWith("<|end|>")) { - prompt.append("<|end|>"); - } - } - else { - prompt.append("<|channel|>final<|message|>"); - prompt.append(assistantContent); - prompt.append("<|end|>"); - } - prompt.append("<|start|>user<|message|>"); - prompt.append(user == null ? "" : user); - prompt.append("<|end|>"); - prompt.append(assistantStart); - } - - @Override - public String removeEOT(final String prompt) { - return prompt.strip().replace("<|end|>", "").replace("<|return|>", ""); - } - - @Override - public String getPromptName() { - return "gpt-oss"; - } - - @Override - public String getResponse(final String prompt) { - if (prompt == null) { - return null; - } - final int markerIndex = prompt.indexOf(finalMarker); - if (markerIndex == -1) { - return prompt; - } - final int start = markerIndex + finalMarker.length(); - int end = prompt.indexOf(startMarker, start); - if (end == -1) { - end = prompt.length(); - } - return prompt.substring(start, end).strip(); - } - - @Override - public String getReasoning(final String prompt) { - if (prompt == null) { - return null; - } - final int markerIndex = prompt.indexOf(analysisMarker); - if (markerIndex == -1) { - return null; - } - final int start = markerIndex + analysisMarker.length(); - int end = prompt.indexOf(startMarker, start); - if (end == -1) { - end = prompt.length(); - } - return prompt.substring(start, end).strip(); - } + @Override + public StringBuilder start(final String system, final String user) { + final StringBuilder prompt = new StringBuilder(); + if (!system.isEmpty()) { + prompt.append("<|start|>system<|message|>"); + prompt.append(system); + prompt.append("<|end|>"); + } + prompt.append("<|start|>user<|message|>"); + prompt.append(user); + prompt.append("<|end|>"); + prompt.append(assistantStart); + + return prompt; + } + + @Override + public void next(final StringBuilder prompt, final String assistant, final String user) { + final String assistantContent = assistant == null ? "" : assistant; + if (assistantContent.startsWith(assistantStart)) { + final int assistantStartLength = assistantStart.length(); + if (prompt.length() >= assistantStartLength) { + boolean endsWithAssistantStart = true; + for (int i = 0; i < assistantStartLength; i++) { + if (prompt.charAt(prompt.length() - assistantStartLength + i) != assistantStart.charAt(i)) { + endsWithAssistantStart = false; + break; + } + } + if (endsWithAssistantStart) { + prompt.setLength(prompt.length() - assistantStartLength); + } + } + } + if (assistantContent.contains(finalMarker)) { + prompt.append(assistantContent); + if (!assistantContent.endsWith("<|end|>")) { + prompt.append("<|end|>"); + } + } + else { + prompt.append("<|channel|>final<|message|>"); + prompt.append(assistantContent); + prompt.append("<|end|>"); + } + prompt.append("<|start|>user<|message|>"); + prompt.append(user == null ? "" : user); + prompt.append("<|end|>"); + prompt.append(assistantStart); + } + + @Override + public String removeEOT(final String prompt) { + return prompt.strip().replace("<|end|>", "").replace("<|return|>", ""); + } + + @Override + public String getPromptName() { + return "gpt-oss"; + } + + @Override + public String getResponse(final String prompt) { + if (prompt == null) { + return null; + } + final int markerIndex = prompt.indexOf(finalMarker); + if (markerIndex == -1) { + return prompt; + } + final int start = markerIndex + finalMarker.length(); + int end = prompt.indexOf(startMarker, start); + if (end == -1) { + end = prompt.length(); + } + return prompt.substring(start, end).strip(); + } + + @Override + public String getReasoning(final String prompt) { + if (prompt == null) { + return null; + } + final int markerIndex = prompt.indexOf(analysisMarker); + if (markerIndex == -1) { + return null; + } + final int start = markerIndex + analysisMarker.length(); + int end = prompt.indexOf(startMarker, start); + if (end == -1) { + end = prompt.length(); + } + return prompt.substring(start, end).strip(); + } } diff --git a/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java b/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java index 5ee4312..c4b55f9 100644 --- a/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java +++ b/src/main/java/de/extio/lmlib/prompt/NoPromptStrategy.java @@ -4,32 +4,32 @@ import org.springframework.stereotype.Component; @Component public class NoPromptStrategy implements PromptStrategy { - - @Override - public String getPromptName() { - return "noop"; - } - - @Override - public StringBuilder start(final String system, final String user) { - final var sb = new StringBuilder(); - if (system != null && !system.isBlank()) { - sb.append(system); - } - if (system != null && !system.isBlank() && user != null && !user.isBlank()) { - sb.append("\n"); - } - if (user != null && !user.isBlank()) { - sb.append(user); - } - return sb; - } - - @Override - public void next(final StringBuilder prompt, final String assistant, final String user) { - prompt.append(assistant); - prompt.append("\n"); - prompt.append(user); - } - + + @Override + public String getPromptName() { + return "noop"; + } + + @Override + public StringBuilder start(final String system, final String user) { + final var sb = new StringBuilder(); + if (system != null && !system.isBlank()) { + sb.append(system); + } + if (system != null && !system.isBlank() && user != null && !user.isBlank()) { + sb.append("\n"); + } + if (user != null && !user.isBlank()) { + sb.append(user); + } + return sb; + } + + @Override + public void next(final StringBuilder prompt, final String assistant, final String user) { + prompt.append(assistant); + prompt.append("\n"); + prompt.append(user); + } + } diff --git a/src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java b/src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java index 7213b88..c8e392a 100644 --- a/src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java +++ b/src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java @@ -15,308 +15,308 @@ import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.Conversation; public class CsvAgentResponseHandlerTest { - - @Test - public void testSimpleCsvParsing() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "xy_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - 1;2;3 - 4;5;6 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List aValues = context.getStringValues("xy_a"); - final List bValues = context.getStringValues("xy_b"); - final List cValues = context.getStringValues("xy_c"); - - assertNotNull(aValues); - assertNotNull(bValues); - assertNotNull(cValues); - - assertEquals(List.of("1", "4"), aValues); - assertEquals(List.of("2", "5"), bValues); - assertEquals(List.of("3", "6"), cValues); - } - - @Test - public void testWithHeader() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("name", "age", "city"), "person_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - name;age;city - Alice;30;Berlin - Bob;25;Munich - Charlie;35;Hamburg - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List names = context.getStringValues("person_name"); - final List ages = context.getStringValues("person_age"); - final List cities = context.getStringValues("person_city"); - - assertEquals(List.of("Alice", "Bob", "Charlie"), names); - assertEquals(List.of("30", "25", "35"), ages); - assertEquals(List.of("Berlin", "Munich", "Hamburg"), cities); - } - - @Test - public void testWithEmptyPrefix() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("x", "y"), ""); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - 10;20 - 30;40 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - assertEquals(List.of("10", "30"), context.getStringValues("x")); - assertEquals(List.of("20", "40"), context.getStringValues("y")); - } - - @Test - public void testWithNullPrefix() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("col1", "col2"), null); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - a;b - c;d - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - assertEquals(List.of("a", "c"), context.getStringValues("col1")); - assertEquals(List.of("b", "d"), context.getStringValues("col2")); - } - - @Test - public void testWithExtraTextBeforeData() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "test_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - Here is the CSV data: - - a;b;c - 1;2;3 - 4;5;6 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - assertEquals(List.of("1", "4"), context.getStringValues("test_a")); - assertEquals(List.of("2", "5"), context.getStringValues("test_b")); - assertEquals(List.of("3", "6"), context.getStringValues("test_c")); - } - - @Test - public void testWithEmptyLines() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "data_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - 1;2 - - 3;4 - - 5;6 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - assertEquals(List.of("1", "3", "5"), context.getStringValues("data_a")); - assertEquals(List.of("2", "4", "6"), context.getStringValues("data_b")); - } - - @Test - public void testWithWhitespace() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "ws_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - 1 ; 2 ; 3 - 4 ; 5 ; 6 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - assertEquals(List.of("1", "4"), context.getStringValues("ws_a")); - assertEquals(List.of("2", "5"), context.getStringValues("ws_b")); - assertEquals(List.of("3", "6"), context.getStringValues("ws_c")); - } - - @Test - public void testWithCustomSeparator() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "custom_", ","); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - 1,2,3 - 4,5,6 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - assertEquals(List.of("1", "4"), context.getStringValues("custom_a")); - assertEquals(List.of("2", "5"), context.getStringValues("custom_b")); - assertEquals(List.of("3", "6"), context.getStringValues("custom_c")); - } - - @Test - public void testColumnCountMismatch() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "mismatch_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - 1;2;3 - 4;5 - 7;8;9 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - // Only rows with correct column count should be included - assertEquals(List.of("1", "7"), context.getStringValues("mismatch_a")); - assertEquals(List.of("2", "8"), context.getStringValues("mismatch_b")); - assertEquals(List.of("3", "9"), context.getStringValues("mismatch_c")); - } - - @Test - public void testEmptyResponse() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "empty_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final Completion completion = new Completion("", null, null, null); - final boolean result = handler.handle(completion, context); - - assertFalse(result); - } - - @Test - public void testNullResponse() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "null_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final Completion completion = new Completion(null, null, null, null); - final boolean result = handler.handle(completion, context); - - assertFalse(result); - } - - @Test - public void testNoValidData() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "nodata_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - This is just text - without any valid CSV data - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertFalse(result); - } - - @Test - public void testSingleRow() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("x", "y", "z"), "single_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = "100;200;300"; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - assertEquals(List.of("100"), context.getStringValues("single_x")); - assertEquals(List.of("200"), context.getStringValues("single_y")); - assertEquals(List.of("300"), context.getStringValues("single_z")); - } - - @Test - public void testMultipleHeaders() { - final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("id", "value"), "multi_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - id;value - ID;VALUE - 1;100 - 2;200 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - // Should skip both header rows - assertEquals(List.of("1", "2"), context.getStringValues("multi_id")); - assertEquals(List.of("100", "200"), context.getStringValues("multi_value")); - } - - @Test - public void testNullHeadingsList() { - assertThrows(IllegalArgumentException.class, () -> { - new CsvAgentResponseHandler(null, "prefix_"); - }); - } - - @Test - public void testEmptyHeadingsList() { - assertThrows(IllegalArgumentException.class, () -> { - new CsvAgentResponseHandler(List.of(), "prefix_"); - }); - } + + @Test + public void testSimpleCsvParsing() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "xy_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + 1;2;3 + 4;5;6 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List aValues = context.getStringValues("xy_a"); + final List bValues = context.getStringValues("xy_b"); + final List cValues = context.getStringValues("xy_c"); + + assertNotNull(aValues); + assertNotNull(bValues); + assertNotNull(cValues); + + assertEquals(List.of("1", "4"), aValues); + assertEquals(List.of("2", "5"), bValues); + assertEquals(List.of("3", "6"), cValues); + } + + @Test + public void testWithHeader() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("name", "age", "city"), "person_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + name;age;city + Alice;30;Berlin + Bob;25;Munich + Charlie;35;Hamburg + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List names = context.getStringValues("person_name"); + final List ages = context.getStringValues("person_age"); + final List cities = context.getStringValues("person_city"); + + assertEquals(List.of("Alice", "Bob", "Charlie"), names); + assertEquals(List.of("30", "25", "35"), ages); + assertEquals(List.of("Berlin", "Munich", "Hamburg"), cities); + } + + @Test + public void testWithEmptyPrefix() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("x", "y"), ""); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + 10;20 + 30;40 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + assertEquals(List.of("10", "30"), context.getStringValues("x")); + assertEquals(List.of("20", "40"), context.getStringValues("y")); + } + + @Test + public void testWithNullPrefix() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("col1", "col2"), null); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + a;b + c;d + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + assertEquals(List.of("a", "c"), context.getStringValues("col1")); + assertEquals(List.of("b", "d"), context.getStringValues("col2")); + } + + @Test + public void testWithExtraTextBeforeData() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + Here is the CSV data: + + a;b;c + 1;2;3 + 4;5;6 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + assertEquals(List.of("1", "4"), context.getStringValues("test_a")); + assertEquals(List.of("2", "5"), context.getStringValues("test_b")); + assertEquals(List.of("3", "6"), context.getStringValues("test_c")); + } + + @Test + public void testWithEmptyLines() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "data_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + 1;2 + + 3;4 + + 5;6 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + assertEquals(List.of("1", "3", "5"), context.getStringValues("data_a")); + assertEquals(List.of("2", "4", "6"), context.getStringValues("data_b")); + } + + @Test + public void testWithWhitespace() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "ws_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + 1 ; 2 ; 3 + 4 ; 5 ; 6 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + assertEquals(List.of("1", "4"), context.getStringValues("ws_a")); + assertEquals(List.of("2", "5"), context.getStringValues("ws_b")); + assertEquals(List.of("3", "6"), context.getStringValues("ws_c")); + } + + @Test + public void testWithCustomSeparator() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "custom_", ","); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + 1,2,3 + 4,5,6 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + assertEquals(List.of("1", "4"), context.getStringValues("custom_a")); + assertEquals(List.of("2", "5"), context.getStringValues("custom_b")); + assertEquals(List.of("3", "6"), context.getStringValues("custom_c")); + } + + @Test + public void testColumnCountMismatch() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b", "c"), "mismatch_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + 1;2;3 + 4;5 + 7;8;9 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + // Only rows with correct column count should be included + assertEquals(List.of("1", "7"), context.getStringValues("mismatch_a")); + assertEquals(List.of("2", "8"), context.getStringValues("mismatch_b")); + assertEquals(List.of("3", "9"), context.getStringValues("mismatch_c")); + } + + @Test + public void testEmptyResponse() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "empty_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final Completion completion = new Completion("", null, null, null); + final boolean result = handler.handle(completion, context); + + assertFalse(result); + } + + @Test + public void testNullResponse() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "null_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final Completion completion = new Completion(null, null, null, null); + final boolean result = handler.handle(completion, context); + + assertFalse(result); + } + + @Test + public void testNoValidData() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("a", "b"), "nodata_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + This is just text + without any valid CSV data + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertFalse(result); + } + + @Test + public void testSingleRow() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("x", "y", "z"), "single_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = "100;200;300"; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + assertEquals(List.of("100"), context.getStringValues("single_x")); + assertEquals(List.of("200"), context.getStringValues("single_y")); + assertEquals(List.of("300"), context.getStringValues("single_z")); + } + + @Test + public void testMultipleHeaders() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(List.of("id", "value"), "multi_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + id;value + ID;VALUE + 1;100 + 2;200 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + // Should skip both header rows + assertEquals(List.of("1", "2"), context.getStringValues("multi_id")); + assertEquals(List.of("100", "200"), context.getStringValues("multi_value")); + } + + @Test + public void testNullHeadingsList() { + assertThrows(IllegalArgumentException.class, () -> { + new CsvAgentResponseHandler(null, "prefix_"); + }); + } + + @Test + public void testEmptyHeadingsList() { + assertThrows(IllegalArgumentException.class, () -> { + new CsvAgentResponseHandler(List.of(), "prefix_"); + }); + } } diff --git a/src/test/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandlerTest.java b/src/test/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandlerTest.java index e3cf71b..a0cd836 100644 --- a/src/test/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandlerTest.java +++ b/src/test/java/de/extio/lmlib/agent/responsehandler/CsvUnmarshallingAgentResponseHandlerTest.java @@ -13,251 +13,251 @@ import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.Conversation; public class CsvUnmarshallingAgentResponseHandlerTest { - - public static record TestObject(String name, List tags, Integer value) { - } - - @Test - public void testSimpleCsvParsing() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - item1;tag1,tag2;42 - item2;tag3;100 - item3;;50 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_test_", TestObject.class); - assertEquals(3, objects.size()); - - assertEquals("item1", objects.get(0).name()); - assertEquals(List.of("tag1", "tag2"), objects.get(0).tags()); - assertEquals(42, objects.get(0).value()); - - assertEquals("item2", objects.get(1).name()); - assertEquals(List.of("tag3"), objects.get(1).tags()); - assertEquals(100, objects.get(1).value()); - - assertEquals("item3", objects.get(2).name()); - assertTrue(objects.get(2).tags().isEmpty()); - assertEquals(50, objects.get(2).value()); - } - - @Test - public void testCsvWithHeader() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - name;tags;value - item1;tag1,tag2;42 - item2;tag3;100 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_test_", TestObject.class); - assertEquals(2, objects.size()); - - assertEquals("item1", objects.get(0).name()); - assertEquals("item2", objects.get(1).name()); - } - - @Test - public void testCsvWithPreamble() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - Here is the result: - - item1;tag1,tag2;42 - item2;tag3;100 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_test_", TestObject.class); - assertEquals(2, objects.size()); - } - - @Test - public void testCsvWithHeaderAndPreamble() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - Here are the results in CSV format: - - name;tags;value - item1;tag1,tag2;42 - item2;tag3;100 - item3;;50 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_test_", TestObject.class); - assertEquals(3, objects.size()); - - assertEquals("item1", objects.get(0).name()); - assertEquals(List.of("tag1", "tag2"), objects.get(0).tags()); - assertEquals(42, objects.get(0).value()); - } - - @Test - public void testCsvWithEmptyLines() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - - item1;tag1,tag2;42 - - item2;tag3;100 - - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_test_", TestObject.class); - assertEquals(2, objects.size()); - } - - @Test - public void testCsvWithNullValues() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - item1;tag1,tag2; - item2;;100 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_test_", TestObject.class); - assertEquals(2, objects.size()); - - assertEquals("item1", objects.get(0).name()); - assertNotNull(objects.get(0).tags()); - assertEquals(null, objects.get(0).value()); - - assertEquals("item2", objects.get(1).name()); - assertTrue(objects.get(1).tags().isEmpty()); - assertEquals(100, objects.get(1).value()); - } - - public static record TagObject(String tag, List aliases, Integer importance) { - } - - @Test - public void testTagObjectExample() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TagObject.class, "_tag_"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - location_dark_forest;location_forest,location_shadow_woods;25 - location_ancient_shrine;location_shrine,location_vine_shrine;65 - combat_dragon;combat_wyrm,combat_ancient_dragon;65 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_tag_", TagObject.class); - assertEquals(3, objects.size()); - - assertEquals("location_dark_forest", objects.get(0).tag()); - assertEquals(List.of("location_forest", "location_shadow_woods"), objects.get(0).aliases()); - assertEquals(25, objects.get(0).importance()); - - assertEquals("combat_dragon", objects.get(2).tag()); - assertEquals(List.of("combat_wyrm", "combat_ancient_dragon"), objects.get(2).aliases()); - assertEquals(65, objects.get(2).importance()); - } - - @Test - public void testCustomSeparators() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler( - TagObject.class, "_custom_", "|", ":"); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - location_dark_forest|location_forest:location_shadow_woods|25 - location_ancient_shrine|location_shrine:location_vine_shrine|65 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_custom_", TagObject.class); - assertEquals(2, objects.size()); - - assertEquals("location_dark_forest", objects.get(0).tag()); - assertEquals(List.of("location_forest", "location_shadow_woods"), objects.get(0).aliases()); - assertEquals(25, objects.get(0).importance()); - } - - @Test - public void testNoListSeparator() { - final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler( - TestObject.class, "_nolist_", ";", null); - final AgentContext context = new AgentContext(); - context.setConversation(Conversation.create("test")); - - final String csvResponse = """ - item1;tag1,tag2,tag3;42 - item2;ignored;100 - """; - - final Completion completion = new Completion(csvResponse, null, null, null); - final boolean result = handler.handle(completion, context); - - assertTrue(result); - - final List objects = context.getValues("_nolist_", TestObject.class); - assertEquals(2, objects.size()); - - assertEquals("item1", objects.get(0).name()); - assertTrue(objects.get(0).tags().isEmpty()); - assertEquals(42, objects.get(0).value()); - - assertEquals("item2", objects.get(1).name()); - assertTrue(objects.get(1).tags().isEmpty()); - assertEquals(100, objects.get(1).value()); - } + + public static record TestObject(String name, List tags, Integer value) { + } + + @Test + public void testSimpleCsvParsing() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + item1;tag1,tag2;42 + item2;tag3;100 + item3;;50 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(3, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertEquals(List.of("tag1", "tag2"), objects.get(0).tags()); + assertEquals(42, objects.get(0).value()); + + assertEquals("item2", objects.get(1).name()); + assertEquals(List.of("tag3"), objects.get(1).tags()); + assertEquals(100, objects.get(1).value()); + + assertEquals("item3", objects.get(2).name()); + assertTrue(objects.get(2).tags().isEmpty()); + assertEquals(50, objects.get(2).value()); + } + + @Test + public void testCsvWithHeader() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + name;tags;value + item1;tag1,tag2;42 + item2;tag3;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertEquals("item2", objects.get(1).name()); + } + + @Test + public void testCsvWithPreamble() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + Here is the result: + + item1;tag1,tag2;42 + item2;tag3;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + } + + @Test + public void testCsvWithHeaderAndPreamble() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + Here are the results in CSV format: + + name;tags;value + item1;tag1,tag2;42 + item2;tag3;100 + item3;;50 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(3, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertEquals(List.of("tag1", "tag2"), objects.get(0).tags()); + assertEquals(42, objects.get(0).value()); + } + + @Test + public void testCsvWithEmptyLines() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + + item1;tag1,tag2;42 + + item2;tag3;100 + + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + } + + @Test + public void testCsvWithNullValues() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + item1;tag1,tag2; + item2;;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertNotNull(objects.get(0).tags()); + assertEquals(null, objects.get(0).value()); + + assertEquals("item2", objects.get(1).name()); + assertTrue(objects.get(1).tags().isEmpty()); + assertEquals(100, objects.get(1).value()); + } + + public static record TagObject(String tag, List aliases, Integer importance) { + } + + @Test + public void testTagObjectExample() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler(TagObject.class, "_tag_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + location_dark_forest;location_forest,location_shadow_woods;25 + location_ancient_shrine;location_shrine,location_vine_shrine;65 + combat_dragon;combat_wyrm,combat_ancient_dragon;65 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_tag_", TagObject.class); + assertEquals(3, objects.size()); + + assertEquals("location_dark_forest", objects.get(0).tag()); + assertEquals(List.of("location_forest", "location_shadow_woods"), objects.get(0).aliases()); + assertEquals(25, objects.get(0).importance()); + + assertEquals("combat_dragon", objects.get(2).tag()); + assertEquals(List.of("combat_wyrm", "combat_ancient_dragon"), objects.get(2).aliases()); + assertEquals(65, objects.get(2).importance()); + } + + @Test + public void testCustomSeparators() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler( + TagObject.class, "_custom_", "|", ":"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + location_dark_forest|location_forest:location_shadow_woods|25 + location_ancient_shrine|location_shrine:location_vine_shrine|65 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_custom_", TagObject.class); + assertEquals(2, objects.size()); + + assertEquals("location_dark_forest", objects.get(0).tag()); + assertEquals(List.of("location_forest", "location_shadow_woods"), objects.get(0).aliases()); + assertEquals(25, objects.get(0).importance()); + } + + @Test + public void testNoListSeparator() { + final CsvUnmarshallingAgentResponseHandler handler = new CsvUnmarshallingAgentResponseHandler( + TestObject.class, "_nolist_", ";", null); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + item1;tag1,tag2,tag3;42 + item2;ignored;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_nolist_", TestObject.class); + assertEquals(2, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertTrue(objects.get(0).tags().isEmpty()); + assertEquals(42, objects.get(0).value()); + + assertEquals("item2", objects.get(1).name()); + assertTrue(objects.get(1).tags().isEmpty()); + assertEquals(100, objects.get(1).value()); + } } diff --git a/src/test/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandlerTest.java b/src/test/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandlerTest.java index 759e5eb..50ad7c2 100644 --- a/src/test/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandlerTest.java +++ b/src/test/java/de/extio/lmlib/agent/responsehandler/JsonAgentResponseHandlerTest.java @@ -204,7 +204,7 @@ public class JsonAgentResponseHandlerTest { assertTrue(value.contains("clown")); } - @Test + @Test public void testSanitizeUnescapedQuotesInString2() { final JsonAgentResponseHandler handler = new JsonAgentResponseHandler(); final AgentContext context = new AgentContext();