From 28a1fde3ef9a8380ab4646b37441f8f74e999cf2 Mon Sep 17 00:00:00 2001 From: extio-de Date: Mon, 8 Dec 2025 21:57:30 +0100 Subject: [PATCH] feat: Add CsvAgentResponseHandler and corresponding tests for CSV parsing functionality --- .../CsvAgentResponseHandler.java | 259 +++++++++++++++++ .../CsvAgentResponseHandlerTest.java | 263 ++++++++++++++++++ 2 files changed, 522 insertions(+) create mode 100644 src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java create mode 100644 src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java diff --git a/src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java b/src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java new file mode 100644 index 0000000..f0bfcef --- /dev/null +++ b/src/main/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandler.java @@ -0,0 +1,259 @@ +package de.extio.lmlib.agent.responsehandler; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.extio.lmlib.agent.AgentContext; +import de.extio.lmlib.client.Completion; +import de.extio.lmlib.client.Conversation; + +public class CsvAgentResponseHandler implements AgentResponseHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(CsvAgentResponseHandler.class); + + private final Class targetClass; + + private final String contextKey; + + private final String fieldSeparator; + + private final String listSeparator; + + private final Field[] fields; + + private final List fieldNames; + + private final Constructor constructor; + + private final String fieldSeparatorQuoted; + + private final String listSeparatorQuoted; + + private final String errorPromptSuffix; + + public CsvAgentResponseHandler(final Class targetClass, final String contextKey) { + this(targetClass, contextKey, ";", ","); + } + + public CsvAgentResponseHandler(final Class targetClass, final String contextKey, + final String fieldSeparator, final String listSeparator) { + this.targetClass = targetClass; + this.contextKey = contextKey; + this.fieldSeparator = fieldSeparator; + this.listSeparator = listSeparator; + + // Pre-compute static data + this.fields = targetClass.getDeclaredFields(); + this.fieldNames = Arrays.stream(this.fields) + .map(Field::getName) + .toList(); + this.constructor = this.findRecordConstructor(); + if (this.constructor == null) { + throw new IllegalStateException("No suitable constructor found for " + targetClass.getName()); + } + this.fieldSeparatorQuoted = Pattern.quote(fieldSeparator); + this.listSeparatorQuoted = (listSeparator != null && !listSeparator.isEmpty()) + ? Pattern.quote(listSeparator) + : null; + + // Pre-compute error message suffix + final String listInfo = (listSeparator != null && !listSeparator.isEmpty()) + ? " and '" + listSeparator + "' for list items" + : ""; + this.errorPromptSuffix = "\n\nThe previous response could not be fully processed or validated. " + + "Please format the response as CSV with '" + fieldSeparator + "' as field separator" + + listInfo + ". One object per line."; + } + + private Constructor findRecordConstructor() { + final Field[] targetFields = this.targetClass.getDeclaredFields(); + for (final Constructor ctor : this.targetClass.getDeclaredConstructors()) { + if (ctor.getParameterCount() == targetFields.length) { + return ctor; + } + } + return null; + } + + @Override + public boolean handle(final Completion completion, final AgentContext context) { + final String resp = completion.response(); + if (resp == null || resp.isBlank()) { + LOGGER.warn("Empty completion response for CSV parsing"); + return false; + } + + try { + final List parsedObjects = this.parseCsv(resp); + if (parsedObjects.isEmpty()) { + LOGGER.warn("No valid CSV data parsed from response"); + this.addCsvResponseErrorPrompt(context); + return false; + } + + context.setValues(this.contextKey, parsedObjects); + LOGGER.debug("Parsed {} CSV objects", parsedObjects.size()); + return true; + } + catch (final Exception ex) { + LOGGER.warn("Failed to parse CSV response", ex); + this.addCsvResponseErrorPrompt(context); + return false; + } + } + + private List parseCsv(final String csvText) throws Exception { + final List result = new ArrayList<>(); + final String[] lines = csvText.split("\n"); + + int startLine = 0; + + for (int i = 0; i < lines.length; i++) { + final String line = lines[i].trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.contains(this.fieldSeparator)) { + startLine = i + 1; + continue; + } + + final String lowerLine = line.toLowerCase(); + boolean isHeader = false; + for (final String fieldName : this.fieldNames) { + if (lowerLine.contains(fieldName.toLowerCase())) { + isHeader = true; + break; + } + } + + if (isHeader) { + startLine = i + 1; + continue; + } + + break; + } + + for (int i = startLine; i < lines.length; i++) { + final String line = lines[i].trim(); + + if (line.isEmpty()) { + continue; + } + + if (!line.contains(this.fieldSeparator)) { + continue; + } + + try { + final Object obj = this.parseLine(line); + if (obj != null) { + result.add(obj); + } + } + catch (final Exception ex) { + LOGGER.debug("Failed to parse line '{}': {}", line, ex.getMessage()); + } + } + + return result; + } + + private Object parseLine(final String line) throws Exception { + final String[] parts = line.split(this.fieldSeparatorQuoted, -1); + if (parts.length != this.fields.length) { + LOGGER.debug("Field count mismatch: expected {}, got {} for line '{}'", + this.fields.length, parts.length, line); + return null; + } + + final Object[] args = new Object[this.fields.length]; + + for (int i = 0; i < this.fields.length; i++) { + final String value = parts[i].trim(); + args[i] = this.parseValue(value, this.fields[i]); + } + + return this.constructor.newInstance(args); + } + + private Object parseValue(final String value, final Field field) { + final Class fieldType = field.getType(); + + if (value.isEmpty() || value.equalsIgnoreCase("null")) { + if (fieldType == List.class) { + return new ArrayList<>(); + } + return null; + } + + if (fieldType == String.class) { + return value; + } + + if (fieldType == Integer.class || fieldType == int.class) { + return Integer.parseInt(value); + } + + if (fieldType == Long.class || fieldType == long.class) { + return Long.parseLong(value); + } + + if (fieldType == Double.class || fieldType == double.class) { + return Double.parseDouble(value); + } + + if (fieldType == Boolean.class || fieldType == boolean.class) { + return Boolean.parseBoolean(value); + } + + if (fieldType == List.class) { + if (this.listSeparatorQuoted == null) { + return new ArrayList<>(); + } + + final Type genericType = field.getGenericType(); + if (genericType instanceof ParameterizedType) { + final ParameterizedType paramType = (ParameterizedType) genericType; + final Type[] typeArgs = paramType.getActualTypeArguments(); + + if (typeArgs.length > 0 && typeArgs[0] == String.class) { + if (value.isEmpty()) { + return new ArrayList(); + } + + final String[] items = value.split(this.listSeparatorQuoted); + final List list = new ArrayList<>(); + for (final String item : items) { + final String trimmed = item.trim(); + if (!trimmed.isEmpty()) { + list.add(trimmed); + } + } + return list; + } + } + } + + throw new IllegalArgumentException("Unsupported field type: " + fieldType.getName()); + } + + private void addCsvResponseErrorPrompt(final AgentContext context) { + final var turn = context.getConversation().getConversation().getLast(); + context.getConversation().replaceTurn(new Conversation.Turn( + turn.type(), + turn.text() + this.errorPromptSuffix)); + } +} diff --git a/src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java b/src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java new file mode 100644 index 0000000..46270fb --- /dev/null +++ b/src/test/java/de/extio/lmlib/agent/responsehandler/CsvAgentResponseHandlerTest.java @@ -0,0 +1,263 @@ +package de.extio.lmlib.agent.responsehandler; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import de.extio.lmlib.agent.AgentContext; +import de.extio.lmlib.client.Completion; +import de.extio.lmlib.client.Conversation; + +public class CsvAgentResponseHandlerTest { + + public static record TestObject(String name, List tags, Integer value) { + } + + @Test + public void testSimpleCsvParsing() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + item1;tag1,tag2;42 + item2;tag3;100 + item3;;50 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(3, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertEquals(List.of("tag1", "tag2"), objects.get(0).tags()); + assertEquals(42, objects.get(0).value()); + + assertEquals("item2", objects.get(1).name()); + assertEquals(List.of("tag3"), objects.get(1).tags()); + assertEquals(100, objects.get(1).value()); + + assertEquals("item3", objects.get(2).name()); + assertTrue(objects.get(2).tags().isEmpty()); + assertEquals(50, objects.get(2).value()); + } + + @Test + public void testCsvWithHeader() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + name;tags;value + item1;tag1,tag2;42 + item2;tag3;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertEquals("item2", objects.get(1).name()); + } + + @Test + public void testCsvWithPreamble() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + Here is the result: + + item1;tag1,tag2;42 + item2;tag3;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + } + + @Test + public void testCsvWithHeaderAndPreamble() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + Here are the results in CSV format: + + name;tags;value + item1;tag1,tag2;42 + item2;tag3;100 + item3;;50 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(3, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertEquals(List.of("tag1", "tag2"), objects.get(0).tags()); + assertEquals(42, objects.get(0).value()); + } + + @Test + public void testCsvWithEmptyLines() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + + item1;tag1,tag2;42 + + item2;tag3;100 + + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + } + + @Test + public void testCsvWithNullValues() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(TestObject.class, "_test_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + item1;tag1,tag2; + item2;;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_test_", TestObject.class); + assertEquals(2, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertNotNull(objects.get(0).tags()); + assertEquals(null, objects.get(0).value()); + + assertEquals("item2", objects.get(1).name()); + assertTrue(objects.get(1).tags().isEmpty()); + assertEquals(100, objects.get(1).value()); + } + + public static record TagObject(String tag, List aliases, Integer importance) { + } + + @Test + public void testTagObjectExample() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler(TagObject.class, "_tag_"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + location_dark_forest;location_forest,location_shadow_woods;25 + location_ancient_shrine;location_shrine,location_vine_shrine;65 + combat_dragon;combat_wyrm,combat_ancient_dragon;65 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_tag_", TagObject.class); + assertEquals(3, objects.size()); + + assertEquals("location_dark_forest", objects.get(0).tag()); + assertEquals(List.of("location_forest", "location_shadow_woods"), objects.get(0).aliases()); + assertEquals(25, objects.get(0).importance()); + + assertEquals("combat_dragon", objects.get(2).tag()); + assertEquals(List.of("combat_wyrm", "combat_ancient_dragon"), objects.get(2).aliases()); + assertEquals(65, objects.get(2).importance()); + } + + @Test + public void testCustomSeparators() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler( + TagObject.class, "_custom_", "|", ":"); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + location_dark_forest|location_forest:location_shadow_woods|25 + location_ancient_shrine|location_shrine:location_vine_shrine|65 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_custom_", TagObject.class); + assertEquals(2, objects.size()); + + assertEquals("location_dark_forest", objects.get(0).tag()); + assertEquals(List.of("location_forest", "location_shadow_woods"), objects.get(0).aliases()); + assertEquals(25, objects.get(0).importance()); + } + + @Test + public void testNoListSeparator() { + final CsvAgentResponseHandler handler = new CsvAgentResponseHandler( + TestObject.class, "_nolist_", ";", null); + final AgentContext context = new AgentContext(); + context.setConversation(Conversation.create("test")); + + final String csvResponse = """ + item1;tag1,tag2,tag3;42 + item2;ignored;100 + """; + + final Completion completion = new Completion(csvResponse, null, null, null); + final boolean result = handler.handle(completion, context); + + assertTrue(result); + + final List objects = context.getValues("_nolist_", TestObject.class); + assertEquals(2, objects.size()); + + assertEquals("item1", objects.get(0).name()); + assertTrue(objects.get(0).tags().isEmpty()); + assertEquals(42, objects.get(0).value()); + + assertEquals("item2", objects.get(1).name()); + assertTrue(objects.get(1).tags().isEmpty()); + assertEquals(100, objects.get(1).value()); + } +}