refactor: Update README to remove optional dependency on commons-lang3 and increment version to 1.7.1; enhance AzureAiClient with proxy settings and refactor conversation methods; add AnswerGrader class for answer evaluation; improve TextUtils with new methods for string manipulation and regex matching; introduce comprehensive tests for TextUtils functionality.
This commit is contained in:
extio-de
@@ -29,13 +29,6 @@ To keep this library lean and compatible with different dependency trees, all de
|
||||
|
||||
## Optional dependencies
|
||||
|
||||
TextUtils require commons-lang3
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</dependency>
|
||||
|
||||
Some services benefit from Spring cache framework, e.g. the model profile service:
|
||||
|
||||
<dependency>
|
||||
|
||||
10
pom.xml
10
pom.xml
@@ -4,7 +4,7 @@
|
||||
<artifactId>lm-lib</artifactId>
|
||||
<groupId>de.extio</groupId>
|
||||
<name>lm-lib</name>
|
||||
<version>1.7.0</version>
|
||||
<version>1.7.1</version>
|
||||
|
||||
<properties>
|
||||
<java.version>21</java.version>
|
||||
@@ -13,7 +13,6 @@
|
||||
<spring-boot.version>3.5.6</spring-boot.version>
|
||||
<azureai.version>1.0.0-beta.5</azureai.version>
|
||||
<jtokkit.version>1.1.0</jtokkit.version>
|
||||
<commons-lang3.version>3.12.0</commons-lang3.version>
|
||||
</properties>
|
||||
|
||||
<repositories>
|
||||
@@ -48,13 +47,6 @@
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>${commons-lang3.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-webflux</artifactId>
|
||||
|
||||
@@ -30,8 +30,9 @@ import com.azure.core.http.HttpClient;
|
||||
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
|
||||
import com.azure.core.http.policy.ExponentialBackoffOptions;
|
||||
import com.azure.core.http.policy.RetryOptions;
|
||||
import com.azure.core.http.policy.RetryPolicy;
|
||||
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
@@ -43,9 +44,7 @@ import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
import io.netty.channel.ChannelOption;
|
||||
import io.netty.handler.timeout.ReadTimeoutHandler;
|
||||
import de.extio.lmlib.token.Tokenizer;import io.netty.handler.timeout.ReadTimeoutHandler;
|
||||
import io.netty.handler.timeout.WriteTimeoutHandler;
|
||||
|
||||
public final class AzureAiClient implements Client {
|
||||
@@ -62,9 +61,24 @@ public final class AzureAiClient implements Client {
|
||||
|
||||
private final Tokenizer tokenizer;
|
||||
|
||||
AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
|
||||
private final boolean proxyEnabled;
|
||||
|
||||
private final String proxyHost;
|
||||
|
||||
private final int proxyPort;
|
||||
|
||||
private final String proxyUser;
|
||||
|
||||
private final String proxyPassword;
|
||||
|
||||
AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer, final boolean proxyEnabled, final String proxyHost, final int proxyPort, final String proxyUser, final String proxyPassword) {
|
||||
this.modelProfileService = modelProfileService;
|
||||
this.tokenizer = tokenizer;
|
||||
this.proxyEnabled = proxyEnabled;
|
||||
this.proxyHost = proxyHost;
|
||||
this.proxyPort = proxyPort;
|
||||
this.proxyUser = proxyUser;
|
||||
this.proxyPassword = proxyPassword;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -78,23 +92,13 @@ public final class AzureAiClient implements Client {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
final var chat = this.createChats(conversation, modelProfile);
|
||||
|
||||
return this.requestCompletions(chat, modelCategory, modelProfile);
|
||||
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
|
||||
return this.requestConversation(modelCategory, conversation, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
throw new UnsupportedOperationException("Not implemented yet");
|
||||
return this.requestConversation(modelCategory, conversation, chunkConsumer);
|
||||
}
|
||||
|
||||
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
@@ -168,40 +172,109 @@ public final class AzureAiClient implements Client {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private Completion requestConversation(ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
final var chat = this.createChats(conversation, modelProfile);
|
||||
|
||||
if (chunkConsumer == null) {
|
||||
return this.requestCompletions(chat, modelCategory, modelProfile);
|
||||
}
|
||||
else {
|
||||
return this.requestStreamCompletions(chat, modelCategory, modelProfile, chunkConsumer);
|
||||
}
|
||||
}
|
||||
|
||||
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var options = this.createChatCompletionsOptions(chat, modelProfile);
|
||||
final var client = this.createHttpClient(modelCategory, modelProfile);
|
||||
final var response = client.complete(options);
|
||||
|
||||
final var finishReason = this.mapFinishReason(response.getChoice().getFinishReason());
|
||||
final var statistics = this.createStatistics(start, response.getUsage().getPromptTokens(), response.getUsage().getCompletionTokens(), modelProfile);
|
||||
|
||||
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
|
||||
}
|
||||
|
||||
private Completion requestStreamCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
|
||||
LOGGER.debug("Requesting Streaming ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var options = this.createChatCompletionsOptions(chat, modelProfile);
|
||||
final var client = this.createHttpClient(modelCategory, modelProfile);
|
||||
final var completionsStream = client.completeStream(options);
|
||||
|
||||
final StringBuilder contentBuilder = new StringBuilder();
|
||||
final AtomicInteger promptTokens = new AtomicInteger(0);
|
||||
final AtomicInteger completionTokens = new AtomicInteger(0);
|
||||
final CompletionFinishReason[] finishReasonHolder = {CompletionFinishReason.DONE};
|
||||
|
||||
completionsStream.forEach(streamResponse -> {
|
||||
if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) {
|
||||
final var choice = streamResponse.getChoices().get(0);
|
||||
|
||||
if (choice.getFinishReason() != null) {
|
||||
finishReasonHolder[0] = this.mapFinishReason(choice.getFinishReason());
|
||||
}
|
||||
|
||||
if (choice.getDelta() != null && choice.getDelta().getContent() != null) {
|
||||
final String content = choice.getDelta().getContent();
|
||||
contentBuilder.append(content);
|
||||
chunkConsumer.accept(new Chunk(content, null));
|
||||
}
|
||||
}
|
||||
|
||||
if (streamResponse.getUsage() != null) {
|
||||
promptTokens.set(streamResponse.getUsage().getPromptTokens());
|
||||
completionTokens.set(streamResponse.getUsage().getCompletionTokens());
|
||||
}
|
||||
});
|
||||
|
||||
final var statistics = this.createStatistics(start, promptTokens.get(), completionTokens.get(), modelProfile);
|
||||
|
||||
return new Completion(contentBuilder.toString(), null, finishReasonHolder[0], statistics);
|
||||
}
|
||||
|
||||
private ChatCompletionsOptions createChatCompletionsOptions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile) {
|
||||
final var options = new ChatCompletionsOptions(chat);
|
||||
options.setModel(modelProfile.modelName());
|
||||
options.setMaxTokens(modelProfile.maxTokens());
|
||||
options.setTemperature(modelProfile.temperature());
|
||||
options.setTopP(modelProfile.topP());
|
||||
|
||||
final var client = this.createHttpClient(modelCategory, modelProfile);
|
||||
final var response = client.complete(options);
|
||||
|
||||
final CompletionFinishReason finishReason;
|
||||
if (CompletionsFinishReason.CONTENT_FILTERED.equals(response.getChoice().getFinishReason())) {
|
||||
finishReason = CompletionFinishReason.CONTENT_FILTERED;
|
||||
return options;
|
||||
}
|
||||
|
||||
private CompletionFinishReason mapFinishReason(final CompletionsFinishReason azureFinishReason) {
|
||||
if (CompletionsFinishReason.CONTENT_FILTERED.equals(azureFinishReason)) {
|
||||
return CompletionFinishReason.CONTENT_FILTERED;
|
||||
}
|
||||
else if (CompletionsFinishReason.TOKEN_LIMIT_REACHED.equals(response.getChoice().getFinishReason())) {
|
||||
finishReason = CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
else if (CompletionsFinishReason.TOKEN_LIMIT_REACHED.equals(azureFinishReason)) {
|
||||
return CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
}
|
||||
else {
|
||||
finishReason = CompletionFinishReason.DONE;
|
||||
return CompletionFinishReason.DONE;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private CompletionStatistics createStatistics(final LocalDateTime start, final int promptTokens, final int completionTokens, final ModelProfile modelProfile) {
|
||||
final var statistics = new CompletionStatistics(
|
||||
1,
|
||||
Duration.between(start, LocalDateTime.now()),
|
||||
response.getUsage().getPromptTokens(),
|
||||
response.getUsage().getCompletionTokens(),
|
||||
new BigDecimal(response.getUsage().getPromptTokens()).multiply(modelProfile.costInToken()).add(new BigDecimal(response.getUsage().getCompletionTokens()).multiply(modelProfile.costOutToken())),
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
new BigDecimal(promptTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(completionTokens).multiply(modelProfile.costOutToken())),
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
|
||||
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
|
||||
return statistics;
|
||||
}
|
||||
|
||||
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
@@ -224,22 +297,35 @@ public final class AzureAiClient implements Client {
|
||||
return false;
|
||||
});
|
||||
|
||||
final reactor.netty.http.client.HttpClient httpClient = reactor.netty.http.client.HttpClient.create()
|
||||
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
|
||||
reactor.netty.http.client.HttpClient nettyHttpClient = reactor.netty.http.client.HttpClient.create()
|
||||
.option(io.netty.channel.ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
|
||||
|
||||
final NettyAsyncHttpClientBuilder azureHttpClientBuilder = new NettyAsyncHttpClientBuilder(httpClient);
|
||||
final HttpClient azureHttpClient = azureHttpClientBuilder
|
||||
.readTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.writeTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.build();
|
||||
if (this.proxyEnabled) {
|
||||
LOGGER.info("Proxy is enabled. Configuring proxy settings for host: {} port: {} user: {}", this.proxyHost, this.proxyPort, this.proxyUser);
|
||||
nettyHttpClient = nettyHttpClient.proxy(proxy -> proxy
|
||||
.type(reactor.netty.transport.ProxyProvider.Proxy.HTTP)
|
||||
.host(this.proxyHost)
|
||||
.port(this.proxyPort)
|
||||
.username(this.proxyUser)
|
||||
.password(password -> this.proxyPassword));
|
||||
}
|
||||
else {
|
||||
LOGGER.info("Proxy is not enabled. Skipping proxy configuration.");
|
||||
}
|
||||
|
||||
final HttpClient azureHttpClient = new NettyAsyncHttpClientBuilder(nettyHttpClient)
|
||||
.connectTimeout(Duration.ofMillis(CONNECT_TIMEOUT))
|
||||
.readTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.writeTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.build();
|
||||
|
||||
final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
|
||||
.credential(new AzureKeyCredential(modelProfile.apiKey()))
|
||||
.endpoint(modelProfile.url())
|
||||
.retryOptions(retryOptions)
|
||||
.retryPolicy(new RetryPolicy(retryOptions))
|
||||
.httpClient(azureHttpClient)
|
||||
.buildClient();
|
||||
return chatCompletionsClient;
|
||||
@@ -247,7 +333,7 @@ public final class AzureAiClient implements Client {
|
||||
}
|
||||
|
||||
private boolean shouldRetryError(final int statusCode) {
|
||||
return statusCode == 408 || statusCode == 429 || (statusCode >= 500 && statusCode < 600);
|
||||
return statusCode == 408 || statusCode == 429 || statusCode >= 500 && statusCode < 600;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package de.extio.lmlib.client.azureai;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
@@ -11,9 +12,24 @@ import de.extio.lmlib.token.Tokenizer;
|
||||
@ConditionalOnClass(name = {"com.azure.ai.inference.ChatCompletionsClient" })
|
||||
public class AzureAiClientAutoConfiguration {
|
||||
|
||||
@Value("${extproxy.enabled:false}")
|
||||
private boolean proxyEnabled;
|
||||
|
||||
@Value("${extproxy.host:}")
|
||||
private String proxyHost;
|
||||
|
||||
@Value("${extproxy.port:3128}")
|
||||
private int proxyPort;
|
||||
|
||||
@Value("${extproxy.username:}")
|
||||
private String proxyUser;
|
||||
|
||||
@Value("${extproxy.password:}")
|
||||
private String proxyPassword;
|
||||
|
||||
@Bean
|
||||
AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
|
||||
return new AzureAiClient(modelProfileService, tokenizer);
|
||||
return new AzureAiClient(modelProfileService, tokenizer, this.proxyEnabled, this.proxyHost, this.proxyPort, this.proxyUser, this.proxyPassword);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
62
src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java
Executable file
62
src/main/java/de/extio/lmlib/grader/agent/AnswerGrader.java
Executable file
@@ -0,0 +1,62 @@
|
||||
package de.extio.lmlib.grader.agent;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import de.extio.lmlib.agent.Agent;
|
||||
import de.extio.lmlib.agent.AgentNext;
|
||||
import de.extio.lmlib.agent.AgentType;
|
||||
import de.extio.lmlib.agent.responsehandler.JsonAgentResponseHandler;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
|
||||
public class AnswerGrader {
|
||||
|
||||
public static Agent createAgent() {
|
||||
return new Agent(
|
||||
"AnswerGrader",
|
||||
AgentType.COMPLETION,
|
||||
ModelCategory.MEDIUM,
|
||||
"""
|
||||
You are a grader agent that evaluates the quality of an answer to a support question.
|
||||
Please provide a response on a scale between 0 and 100, where: 100 indicates that the answer directly addresses the question, is accurate, and provides a clear and concise solution or explanation; 90-99 indicates that the answer is mostly accurate and provides a good solution or explanation, but may lack some detail or clarity; 80-89 indicates that the answer is generally accurate but may require some additional information or clarification to fully understand the solution or explanation; 75 indicates that the answer describes how the system behaves to get to the subject of the question, but may not directly answer the question; 50-74 indicates that the answer is partially accurate or relevant, but may lack key information or provide an incomplete solution or explanation; 25-49 indicates that the answer does not directly answer the question or is speculative or it contains mainly instructions for changes or mainly refers to the Enterprise Integration team without providing substantial information, but may provide some relevant information or context; 0-24 indicates that the answer is not related to the question, is inaccurate, or provides no useful information;
|
||||
Please evaluate the answer and provide a rating between 0 and 100 based on its quality and relevance to the question.
|
||||
Provide no preamble and a short explanation. Return the response in JSON format with the following fields: {\"qaRating\": rating, \"qaExplanation\": \"explanation\"}
|
||||
""",
|
||||
"""
|
||||
The question is: {{questions}}
|
||||
The answer is: {{answer}}
|
||||
""",
|
||||
new JsonAgentResponseHandler(),
|
||||
null,
|
||||
context -> {
|
||||
if (!context.getContext().containsKey("qaRating")) {
|
||||
context.setStringValue("qaRating", "0");
|
||||
context.setStringValue("qaExplanation", "The grader agent did not provide an answer");
|
||||
}
|
||||
else if (!context.getContext().containsKey("qaExplanation")) {
|
||||
context.setStringValue("qaExplanation", "The grader agent did not provide an explanation");
|
||||
}
|
||||
|
||||
context.getContext().compute("qaIteration", (k, v) -> {
|
||||
if (v == null) {
|
||||
return List.of(1);
|
||||
}
|
||||
else {
|
||||
return List.of((Integer) v.get(0) + 1);
|
||||
}
|
||||
});
|
||||
},
|
||||
null,
|
||||
context -> {
|
||||
final var qaRatingEntry = context.getStringValue("qaRating");
|
||||
final var qaRating = qaRatingEntry != null ? Integer.parseInt(qaRatingEntry) : 0;
|
||||
final var qaPassed = qaRating > 60;
|
||||
context.getContext().put("qaPassed", List.of(qaPassed));
|
||||
if (!qaPassed && (int) context.getContext().get("qaIteration").get(0) < 2) {
|
||||
context.setStringValue("qaFeedback", "Feedback from the previous attempt to consider: " + context.getContext().get("qaExplanation").get(0).toString());
|
||||
return new AgentNext(context.getStringValue("qaFeedbackLoopDestination"), Boolean.FALSE);
|
||||
}
|
||||
|
||||
return new AgentNext(null, qaPassed);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,14 @@ package de.extio.lmlib.util;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
|
||||
public class TextUtils {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TextUtils.class);
|
||||
@@ -53,26 +56,55 @@ public class TextUtils {
|
||||
}
|
||||
|
||||
public static String normalizeModelResponse(final String response, final boolean removePreamble) {
|
||||
if (response == null) {
|
||||
if (response == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
var result = StringUtils.replace(response, "\r", "");
|
||||
|
||||
String result = response.replace("\r", "");
|
||||
|
||||
if (removePreamble) {
|
||||
final int colon = result.indexOf(':');
|
||||
if (colon > -1 && colon < result.length() - 2 && (colon < 50 || result.charAt(colon + 1) == '\n')) {
|
||||
final int dot = result.indexOf('.');
|
||||
int firstLineEnd = result.indexOf('\n');
|
||||
final boolean colonInFirstLine = firstLineEnd == -1 || colon != -1 && colon < firstLineEnd;
|
||||
final boolean dotInFirstLine = firstLineEnd == -1 || dot != -1 && dot < firstLineEnd;
|
||||
|
||||
if (colon > -1 && colon < result.length() - 2 && (colon < 50 || result.charAt(colon + 1) == '\n') && colonInFirstLine && !dotInFirstLine) {
|
||||
result = result.substring(colon + 1);
|
||||
}
|
||||
firstLineEnd = result.indexOf('\n');
|
||||
if (firstLineEnd != -1) {
|
||||
String firstLine = result.substring(0, firstLineEnd);
|
||||
firstLine = stripChars(firstLine.trim(), "*#´`'“”„‟«»\"\n");
|
||||
result = firstLine + result.substring(firstLineEnd);
|
||||
}
|
||||
else {
|
||||
result = stripChars(result, "*#´`'“”„‟«»\"\n");
|
||||
}
|
||||
}
|
||||
|
||||
result = StringUtils.trim(result);
|
||||
result = StringUtils.strip(result, "*#´`'“”„‟«»\"\n");
|
||||
result = StringUtils.trim(result);
|
||||
result = result.trim();
|
||||
if (!removePreamble) {
|
||||
result = stripChars(result, "´`'“”„‟«»\"\n");
|
||||
}
|
||||
result = result.trim();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String stripChars(final String str, final String chars) {
|
||||
if (str == null || str.isEmpty())
|
||||
return str;
|
||||
int start = 0, end = str.length();
|
||||
|
||||
while (start < end && chars.indexOf(str.charAt(start)) >= 0)
|
||||
start++;
|
||||
while (end > start && chars.indexOf(str.charAt(end - 1)) >= 0)
|
||||
end--;
|
||||
|
||||
return str.substring(start, end);
|
||||
}
|
||||
|
||||
public static List<String> splitParagraphs(final String text, final int chunks_norm, final int chunks_var, final boolean slidingWindows) {
|
||||
final int CHUNKS_MIN = chunks_norm - chunks_var;
|
||||
final int CHUNKS_MAX = chunks_norm + chunks_var;
|
||||
@@ -136,4 +168,27 @@ public class TextUtils {
|
||||
return splits;
|
||||
}
|
||||
|
||||
public static String capitalizeWords(final String input) {
|
||||
final String[] words = input.split(" ");
|
||||
final StringBuilder capitalizedWords = new StringBuilder();
|
||||
|
||||
for (final String word : words) {
|
||||
if (!word.isEmpty()) {
|
||||
capitalizedWords.append(Character.toUpperCase(word.charAt(0)))
|
||||
.append(word.substring(1).toLowerCase())
|
||||
.append(" ");
|
||||
}
|
||||
}
|
||||
return capitalizedWords.toString().trim();
|
||||
}
|
||||
|
||||
private static final LoadingCache<String, Pattern> wholeWordRegexCache = Caffeine.newBuilder()
|
||||
.maximumSize(1000)
|
||||
.build(regex -> Pattern.compile(regex, Pattern.CASE_INSENSITIVE));
|
||||
|
||||
public static boolean matchWholeWordCaseInsensitive(final String text, final String word) {
|
||||
final String regex = "\\b" + Pattern.quote(word) + "\\b";
|
||||
final Pattern pattern = wholeWordRegexCache.get(regex);
|
||||
return pattern.matcher(text).find();
|
||||
}
|
||||
}
|
||||
|
||||
275
src/test/java/de/extio/lmlib/util/TextUtilsTest.java
Executable file
275
src/test/java/de/extio/lmlib/util/TextUtilsTest.java
Executable file
@@ -0,0 +1,275 @@
|
||||
package de.extio.lmlib.util;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
|
||||
public class TextUtilsTest {
|
||||
|
||||
@Test
|
||||
void normalizeResponseTestRemovePremable() {
|
||||
final String textToNormalize = """
|
||||
Here is a clear and concise answer to the support question:
|
||||
|
||||
The `enrichFuelSurcharchesCronEndpoint` Camel route uses the `FuelSurchargeEnricher` bean, which is responsible for enriching transport data with fuel surcharge information. This bean performs the following tasks:
|
||||
|
||||
* Calculates the fuel surcharge per mile based on the current fuel rate and transport distance
|
||||
* Adds the calculated surcharge to the transport's price additional 1
|
||||
* Logs information about the enrichment process
|
||||
|
||||
The `FuelSurchargeEnricher` bean relies on other services, including `DataServiceFacade` for database data retrieval, `EiaFuelRatesApi` for current fuel rate data, and `ExchangeRouteLogger` for logging purposes. If you require further information or clarification on the implementation details, please contact the Enterprise Integration team.""";
|
||||
|
||||
final String expectedResponse = """
|
||||
The `enrichFuelSurcharchesCronEndpoint` Camel route uses the `FuelSurchargeEnricher` bean, which is responsible for enriching transport data with fuel surcharge information. This bean performs the following tasks:
|
||||
|
||||
* Calculates the fuel surcharge per mile based on the current fuel rate and transport distance
|
||||
* Adds the calculated surcharge to the transport's price additional 1
|
||||
* Logs information about the enrichment process
|
||||
|
||||
The `FuelSurchargeEnricher` bean relies on other services, including `DataServiceFacade` for database data retrieval, `EiaFuelRatesApi` for current fuel rate data, and `ExchangeRouteLogger` for logging purposes. If you require further information or clarification on the implementation details, please contact the Enterprise Integration team.""";
|
||||
|
||||
final String result = TextUtils.normalizeModelResponse(textToNormalize, true);
|
||||
assertEquals(expectedResponse, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizeResponseTestDontRemoveFirstLine1() {
|
||||
final String textToNormalize = """
|
||||
Camel Route Flow Description
|
||||
|
||||
#### Route Name: `bookingModified`
|
||||
|
||||
1. **Initiation**: The `bookingModified` route starts when a message is sent to the `direct:bookingModified` endpoint.
|
||||
2. **Setting Headers**: The route sets the following headers:
|
||||
- `TpExchangeDirection` to `IN`
|
||||
- `TpExchangeDescription` to `Booking modified`
|
||||
- `TpLogBody` to `true`
|
||||
3. **Filtering**: It calls the `isChangeRelevant` method on the `exportChecker` bean with the current and original booking as parameters. If the method returns `true`, the route proceeds.
|
||||
4. **Routing to `outboundIDOCEXP`**: If the filter condition is met, the message is sent to the `direct:outboundIDOCEXP` endpoint, initiating the `outboundIDOCEXP` route.
|
||||
|
||||
#### Route Name: `outboundIDOCEXP`
|
||||
|
||||
1. **Initiation**: This route starts when a message is sent to the `direct:outboundIDOCEXP` endpoint.
|
||||
2. **Setting Headers**: The route sets several headers:
|
||||
- `TpExchangeDirection` to `OUT`
|
||||
- `TpExchangeDescription` to a concatenated string containing the original description and `| IDOC ZTRP_EXP export`
|
||||
- `TpLogBody` to `true`
|
||||
- `CamelFileName` to a generated filename based on the current timestamp (`ZTRP_EXP_<timestamp>.txt`)
|
||||
3. **Determining Environment**: It checks the server environment using the `serverEnvironmentProvider` bean and sets the `_TpEnvironment` header:
|
||||
- Productive environment: `_TpEnvironment` is set to `SAPPRD`.
|
||||
- Customer testing environment: `_TpEnvironment` is set to `SAPQUA`.
|
||||
4. **Setting `TpTransportNumber` Header**: The `TpTransportNumber` header is set to the `transportNumber` property of the current booking.
|
||||
5. **Processing Dispatch Status**: The route calls the `process` method on the `dispatchStatusProvider` bean to extract dispatch status information from the booking and sets it as a header (`_TpStatusTimes`).
|
||||
6. **Templating**: It uses the `velocity` component with the `aquamarkleclerc-export-IDOC-EXP.vm` template to generate an output message.
|
||||
7. **Sending to Outbound Endpoint**: The generated output message is sent to the `outboundEndpoint`, configured to send data to the customer's system.
|
||||
|
||||
### Summary of the Flow
|
||||
|
||||
- The flow starts with the `bookingModified` route.
|
||||
- It checks if the change is relevant and proceeds if true.
|
||||
- It then initiates the `outboundIDOCEXP` route.
|
||||
- The `outboundIDOCEXP` route processes the message, sets necessary headers, and generates an output message based on a template.
|
||||
- The final output message, containing data extracted from the booking and dispatch status, is sent to the customer's system via the `outboundEndpoint`.
|
||||
|
||||
This description outlines the steps from the initiation of the `bookingModified` route to the end of the `outboundIDOCEXP` route, detailing the processing and transformations that occur along the way.""";
|
||||
|
||||
final String expectedResponse = """
|
||||
Camel Route Flow Description
|
||||
|
||||
#### Route Name: `bookingModified`
|
||||
|
||||
1. **Initiation**: The `bookingModified` route starts when a message is sent to the `direct:bookingModified` endpoint.
|
||||
2. **Setting Headers**: The route sets the following headers:
|
||||
- `TpExchangeDirection` to `IN`
|
||||
- `TpExchangeDescription` to `Booking modified`
|
||||
- `TpLogBody` to `true`
|
||||
3. **Filtering**: It calls the `isChangeRelevant` method on the `exportChecker` bean with the current and original booking as parameters. If the method returns `true`, the route proceeds.
|
||||
4. **Routing to `outboundIDOCEXP`**: If the filter condition is met, the message is sent to the `direct:outboundIDOCEXP` endpoint, initiating the `outboundIDOCEXP` route.
|
||||
|
||||
#### Route Name: `outboundIDOCEXP`
|
||||
|
||||
1. **Initiation**: This route starts when a message is sent to the `direct:outboundIDOCEXP` endpoint.
|
||||
2. **Setting Headers**: The route sets several headers:
|
||||
- `TpExchangeDirection` to `OUT`
|
||||
- `TpExchangeDescription` to a concatenated string containing the original description and `| IDOC ZTRP_EXP export`
|
||||
- `TpLogBody` to `true`
|
||||
- `CamelFileName` to a generated filename based on the current timestamp (`ZTRP_EXP_<timestamp>.txt`)
|
||||
3. **Determining Environment**: It checks the server environment using the `serverEnvironmentProvider` bean and sets the `_TpEnvironment` header:
|
||||
- Productive environment: `_TpEnvironment` is set to `SAPPRD`.
|
||||
- Customer testing environment: `_TpEnvironment` is set to `SAPQUA`.
|
||||
4. **Setting `TpTransportNumber` Header**: The `TpTransportNumber` header is set to the `transportNumber` property of the current booking.
|
||||
5. **Processing Dispatch Status**: The route calls the `process` method on the `dispatchStatusProvider` bean to extract dispatch status information from the booking and sets it as a header (`_TpStatusTimes`).
|
||||
6. **Templating**: It uses the `velocity` component with the `aquamarkleclerc-export-IDOC-EXP.vm` template to generate an output message.
|
||||
7. **Sending to Outbound Endpoint**: The generated output message is sent to the `outboundEndpoint`, configured to send data to the customer's system.
|
||||
|
||||
### Summary of the Flow
|
||||
|
||||
- The flow starts with the `bookingModified` route.
|
||||
- It checks if the change is relevant and proceeds if true.
|
||||
- It then initiates the `outboundIDOCEXP` route.
|
||||
- The `outboundIDOCEXP` route processes the message, sets necessary headers, and generates an output message based on a template.
|
||||
- The final output message, containing data extracted from the booking and dispatch status, is sent to the customer's system via the `outboundEndpoint`.
|
||||
|
||||
This description outlines the steps from the initiation of the `bookingModified` route to the end of the `outboundIDOCEXP` route, detailing the processing and transformations that occur along the way.""";
|
||||
|
||||
final String result = TextUtils.normalizeModelResponse(textToNormalize, true);
|
||||
assertEquals(expectedResponse, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizeResponseTestDontRemoveFirstLine2() {
|
||||
final String textToNormalize = """
|
||||
The output is an XML document that represents a transport assignment data in a specific format. It contains a `tisys` root element with a `Transport2Inhouse` child element, which includes details such as:
|
||||
|
||||
- `transport_number`
|
||||
- `status` (if deleted)
|
||||
- `assigned_carrier`
|
||||
- `price`
|
||||
- `currency`
|
||||
|
||||
The XML structure and values are determined based on the properties of the shipment data, specifically the `assignmentMode` and `deleted` status.
|
||||
|
||||
The resulting XML output will be sent to the `outboundEndpoint` as part of the Apache Camel route.""";
|
||||
|
||||
final String expectedResponse = """
|
||||
The output is an XML document that represents a transport assignment data in a specific format. It contains a `tisys` root element with a `Transport2Inhouse` child element, which includes details such as:
|
||||
|
||||
- `transport_number`
|
||||
- `status` (if deleted)
|
||||
- `assigned_carrier`
|
||||
- `price`
|
||||
- `currency`
|
||||
|
||||
The XML structure and values are determined based on the properties of the shipment data, specifically the `assignmentMode` and `deleted` status.
|
||||
|
||||
The resulting XML output will be sent to the `outboundEndpoint` as part of the Apache Camel route.""";
|
||||
|
||||
final String result = TextUtils.normalizeModelResponse(textToNormalize, true);
|
||||
assertEquals(expectedResponse, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizeResponseTestStripQuotesWithPreamble() {
|
||||
final String textToNormalize = """
|
||||
**Here is what he said:**
|
||||
|
||||
This is coming directly from the source.""";
|
||||
|
||||
final String expectedResponse = "This is coming directly from the source.";
|
||||
|
||||
final String result = TextUtils.normalizeModelResponse(textToNormalize, true);
|
||||
assertEquals(expectedResponse, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizeResponseTestStripQuotesNoPreamble() {
|
||||
final String textToNormalize = "«This is a quote directly from the source.»";
|
||||
final String expectedResponse = "This is a quote directly from the source.";
|
||||
|
||||
final String result = TextUtils.normalizeModelResponse(textToNormalize, false);
|
||||
assertEquals(expectedResponse, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitParagraphsTest() {
|
||||
final List<String> splittedText = TextUtils.splitParagraphs(generateText(), 1750, 200, false);
|
||||
|
||||
assertEquals(3, splittedText.size());
|
||||
for (final String chunk : splittedText) {
|
||||
assertTrue(chunk.length() <= 1950);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void matchWholeWordTest() {
|
||||
assertTrue(TextUtils.matchWholeWordCaseInsensitive("The quick brown fox jumps over the lazy dog", "fox"));
|
||||
assertTrue(TextUtils.matchWholeWordCaseInsensitive("The quick brown fox jumps over the lazy dog", "Fox"));
|
||||
assertTrue(TextUtils.matchWholeWordCaseInsensitive("The quick brown fox jumps over the lazy dog", "jumps"));
|
||||
assertFalse(TextUtils.matchWholeWordCaseInsensitive("The quick brown fox jumps over the lazy dog", "foxy"));
|
||||
assertFalse(TextUtils.matchWholeWordCaseInsensitive("The quick brown fox jumps over the lazy dog", "jump"));
|
||||
assertFalse(TextUtils.matchWholeWordCaseInsensitive("The quick brown fox jumps over the lazy dog", "row"));
|
||||
|
||||
assertTrue(TextUtils.matchWholeWordCaseInsensitive("RandomHelloService.doItNow()", "doItNow"));
|
||||
assertTrue(TextUtils.matchWholeWordCaseInsensitive("RandomHelloService.doItNow()", "RandomHelloService"));
|
||||
assertFalse(TextUtils.matchWholeWordCaseInsensitive("RandomHelloService.doItNow()", "HelloService"));
|
||||
assertFalse(TextUtils.matchWholeWordCaseInsensitive("RandomHelloService.doItNow()", "RandomHello"));
|
||||
assertFalse(TextUtils.matchWholeWordCaseInsensitive("RandomHelloService.doItNow()", "doIt"));
|
||||
assertFalse(TextUtils.matchWholeWordCaseInsensitive("RandomHelloService.doItNow()", "now"));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@CsvSource({
|
||||
"'This is a test with special characters', 'test', true",
|
||||
"'This is a test with special characters', 'test*', false",
|
||||
"'This is a test with special characters', 'test+', false",
|
||||
"'This is a test with special characters', 'test?', false",
|
||||
"'This is a test with special characters', 'test$', false",
|
||||
"'This is a test with special characters', 'test${exchange}', false",
|
||||
"'This is a test with special characters', 'test{2}', false",
|
||||
"'This is a test with special characters', 'test[abc]', false",
|
||||
|
||||
"'bean:exchangeSenderProducer?method=sendToNamedConnection(${exchange}, \\'HttpsOutbound\\')', 'exchangeSenderProducer', true",
|
||||
|
||||
"'Service.process(${param}, [1,2,3])', 'Service', true",
|
||||
"'Service.process(${param}, [1,2,3])', 'process', true",
|
||||
"'Service.process(${param}, [1,2,3])', 'param', true",
|
||||
|
||||
"'test${variable}test', 'test', true",
|
||||
"'test(test)test', 'test', true",
|
||||
"'test[array]test', 'test', true",
|
||||
"'test*star+plus?question', 'test', true",
|
||||
"'test^caret$dollar|pipe', 'test', true"
|
||||
})
|
||||
void matchWholeWordWithRegexSpecialCharactersTest(final String text, final String word, final boolean expectedResult) {
|
||||
try {
|
||||
final boolean result = TextUtils.matchWholeWordCaseInsensitive(text, word);
|
||||
assertEquals(expectedResult, result);
|
||||
}
|
||||
catch (final Exception e) {
|
||||
fail(String.format("Exception thrown for '%s' in '%s': %s", word, text, e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
private static String generateText() {
|
||||
return "The *flow* of the Camel `route` named `outboundConfirmationMessage` from `start` to finish is as `follows`:\r\n"
|
||||
+ "1. The `route` starts with a from *endpoint* that *listens* on a direct *endpoint* named `direct:outboundConfirmationMessage`.\r\n"
|
||||
+ "2. The `setHeader` processor sets the `TpExchangeLogReference` header.\r\n"
|
||||
+ "3. The `first` filter `processor` checks if the *isTPM* method of the `eventProducerFilter` bean returns *true*. If it does, the *route* *continues*. Otherwise, it *stops*.\r\n"
|
||||
+ "4. If the first filter `passes`, the `setHeader` processor sets the `TpLogMessage` header.\r\n"
|
||||
+ "5. The *second* filter *processor* checks if the *creationMode* of the message body is equal to 'AUTOMATIC'. If it is, the *route* *continues*. Otherwise, it *stops*.\r\n"
|
||||
+ "6. If the *second* filter passes, the bean processor calls the **forOutboundConfirmationMessage** method of the **outboundTransmissionFactory** bean, *passing* in the transport object, the *eventQualifier* header, and the exchange object.\r\n"
|
||||
+ "7. The bean processor calls the send method of the *outboundTransmissionServiceClient* bean, `passing` in the *transmission* object and the *exchange* object.\r\n"
|
||||
+ "8. The *outboundTransmissionServiceClient* bean sends the *transmission* to the **Webservice** client using the publish *method*.\r\n"
|
||||
+ "9. The **Webservice** client returns a **PublishResponse** object, which is *logged* by the `ExchangeRouteLogger` bean.\r\n"
|
||||
+ "Note *that* the route *stops* if either of the *filters* returns false.\r\n"
|
||||
+ "The *flow* of the Camel `route` named `outboundConfirmationMessage` from `start` to finish is as `follows`:\r\n"
|
||||
+ "1. The `route` starts with a from *endpoint* that *listens* on a direct *endpoint* named `direct:outboundConfirmationMessage`.\r\n"
|
||||
+ "2. The `setHeader` processor sets the `TpExchangeLogReference` header.\r\n"
|
||||
+ "3. The `first` filter `processor` checks if the *isTPM* method of the `eventProducerFilter` bean returns *true*. If it does, the *route* *continues*. Otherwise, it *stops*.\r\n"
|
||||
+ "4. If the first filter `passes`, the `setHeader` processor sets the `TpLogMessage` header.\r\n"
|
||||
+ "5. The *second* filter *processor* checks if the *creationMode* of the message body is equal to 'AUTOMATIC'. If it is, the *route* *continues*. Otherwise, it *stops*.\r\n"
|
||||
+ "6. If the *second* filter passes, the bean processor calls the **forOutboundConfirmationMessage** method of the **outboundTransmissionFactory** bean, *passing* in the transport object, the *eventQualifier* header, and the exchange object.\r\n"
|
||||
+ "7. The bean processor calls the send method of the *outboundTransmissionServiceClient* bean, `passing` in the *transmission* object and the *exchange* object.\r\n"
|
||||
+ "8. The *outboundTransmissionServiceClient* bean sends the *transmission* to the **Webservice** client using the publish *method*.\r\n"
|
||||
+ "9. The **Webservice** client returns a **PublishResponse** object, which is *logged* by the `ExchangeRouteLogger` bean.\r\n"
|
||||
+ "Note *that* the route *stops* if either of the *filters* returns false.\r\n"
|
||||
+ "The *flow* of the Camel `route` named `outboundConfirmationMessage` from `start` to finish is as `follows`:\r\n"
|
||||
+ "1. The `route` starts with a from *endpoint* that *listens* on a direct *endpoint* named `direct:outboundConfirmationMessage`.\r\n"
|
||||
+ "2. The `setHeader` processor sets the `TpExchangeLogReference` header.\r\n"
|
||||
+ "3. The `first` filter `processor` checks if the *isTPM* method of the `eventProducerFilter` bean returns *true*. If it does, the *route* *continues*. Otherwise, it *stops*.\r\n"
|
||||
+ "4. If the first filter `passes`, the `setHeader` processor sets the `TpLogMessage` header.\r\n"
|
||||
+ "5. The *second* filter *processor* checks if the *creationMode* of the message body is equal to 'AUTOMATIC'. If it is, the *route* *continues*. Otherwise, it *stops*.\r\n"
|
||||
+ "6. If the *second* filter passes, the bean processor calls the **forOutboundConfirmationMessage** method of the **outboundTransmissionFactory** bean, *passing* in the transport object, the *eventQualifier* header, and the exchange object.\r\n"
|
||||
+ "7. The bean processor calls the send method of the *outboundTransmissionServiceClient* bean, `passing` in the *transmission* object and the *exchange* object.\r\n"
|
||||
+ "8. The *outboundTransmissionServiceClient* bean sends the *transmission* to the **Webservice** client using the publish *method*.\r\n"
|
||||
+ "9. The **Webservice** client returns a **PublishResponse** object, which is *logged* by the `ExchangeRouteLogger` bean.\r\n"
|
||||
+ "Note *that* the route *stops* if either of the *filters* returns false.";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user