refactor: Update README to remove optional dependency on commons-lang3 and increment version to 1.7.1; enhance AzureAiClient with proxy settings and refactor conversation methods; add AnswerGrader class for answer evaluation; improve TextUtils with new methods for string manipulation and regex matching; introduce comprehensive tests for TextUtils functionality.
This commit is contained in:
extio-de
@@ -30,8 +30,9 @@ import com.azure.core.http.HttpClient;
|
||||
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
|
||||
import com.azure.core.http.policy.ExponentialBackoffOptions;
|
||||
import com.azure.core.http.policy.RetryOptions;
|
||||
import com.azure.core.http.policy.RetryPolicy;
|
||||
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
@@ -43,9 +44,7 @@ import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
import io.netty.channel.ChannelOption;
|
||||
import io.netty.handler.timeout.ReadTimeoutHandler;
|
||||
import de.extio.lmlib.token.Tokenizer;import io.netty.handler.timeout.ReadTimeoutHandler;
|
||||
import io.netty.handler.timeout.WriteTimeoutHandler;
|
||||
|
||||
public final class AzureAiClient implements Client {
|
||||
@@ -62,9 +61,24 @@ public final class AzureAiClient implements Client {
|
||||
|
||||
private final Tokenizer tokenizer;
|
||||
|
||||
AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
|
||||
private final boolean proxyEnabled;
|
||||
|
||||
private final String proxyHost;
|
||||
|
||||
private final int proxyPort;
|
||||
|
||||
private final String proxyUser;
|
||||
|
||||
private final String proxyPassword;
|
||||
|
||||
AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer, final boolean proxyEnabled, final String proxyHost, final int proxyPort, final String proxyUser, final String proxyPassword) {
|
||||
this.modelProfileService = modelProfileService;
|
||||
this.tokenizer = tokenizer;
|
||||
this.proxyEnabled = proxyEnabled;
|
||||
this.proxyHost = proxyHost;
|
||||
this.proxyPort = proxyPort;
|
||||
this.proxyUser = proxyUser;
|
||||
this.proxyPassword = proxyPassword;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -78,23 +92,13 @@ public final class AzureAiClient implements Client {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
final var chat = this.createChats(conversation, modelProfile);
|
||||
|
||||
return this.requestCompletions(chat, modelCategory, modelProfile);
|
||||
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
|
||||
return this.requestConversation(modelCategory, conversation, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
throw new UnsupportedOperationException("Not implemented yet");
|
||||
return this.requestConversation(modelCategory, conversation, chunkConsumer);
|
||||
}
|
||||
|
||||
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
@@ -168,40 +172,109 @@ public final class AzureAiClient implements Client {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private Completion requestConversation(ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.AZURE_AI) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
final var chat = this.createChats(conversation, modelProfile);
|
||||
|
||||
if (chunkConsumer == null) {
|
||||
return this.requestCompletions(chat, modelCategory, modelProfile);
|
||||
}
|
||||
else {
|
||||
return this.requestStreamCompletions(chat, modelCategory, modelProfile, chunkConsumer);
|
||||
}
|
||||
}
|
||||
|
||||
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var options = this.createChatCompletionsOptions(chat, modelProfile);
|
||||
final var client = this.createHttpClient(modelCategory, modelProfile);
|
||||
final var response = client.complete(options);
|
||||
|
||||
final var finishReason = this.mapFinishReason(response.getChoice().getFinishReason());
|
||||
final var statistics = this.createStatistics(start, response.getUsage().getPromptTokens(), response.getUsage().getCompletionTokens(), modelProfile);
|
||||
|
||||
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
|
||||
}
|
||||
|
||||
private Completion requestStreamCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
|
||||
LOGGER.debug("Requesting Streaming ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var options = this.createChatCompletionsOptions(chat, modelProfile);
|
||||
final var client = this.createHttpClient(modelCategory, modelProfile);
|
||||
final var completionsStream = client.completeStream(options);
|
||||
|
||||
final StringBuilder contentBuilder = new StringBuilder();
|
||||
final AtomicInteger promptTokens = new AtomicInteger(0);
|
||||
final AtomicInteger completionTokens = new AtomicInteger(0);
|
||||
final CompletionFinishReason[] finishReasonHolder = {CompletionFinishReason.DONE};
|
||||
|
||||
completionsStream.forEach(streamResponse -> {
|
||||
if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) {
|
||||
final var choice = streamResponse.getChoices().get(0);
|
||||
|
||||
if (choice.getFinishReason() != null) {
|
||||
finishReasonHolder[0] = this.mapFinishReason(choice.getFinishReason());
|
||||
}
|
||||
|
||||
if (choice.getDelta() != null && choice.getDelta().getContent() != null) {
|
||||
final String content = choice.getDelta().getContent();
|
||||
contentBuilder.append(content);
|
||||
chunkConsumer.accept(new Chunk(content, null));
|
||||
}
|
||||
}
|
||||
|
||||
if (streamResponse.getUsage() != null) {
|
||||
promptTokens.set(streamResponse.getUsage().getPromptTokens());
|
||||
completionTokens.set(streamResponse.getUsage().getCompletionTokens());
|
||||
}
|
||||
});
|
||||
|
||||
final var statistics = this.createStatistics(start, promptTokens.get(), completionTokens.get(), modelProfile);
|
||||
|
||||
return new Completion(contentBuilder.toString(), null, finishReasonHolder[0], statistics);
|
||||
}
|
||||
|
||||
private ChatCompletionsOptions createChatCompletionsOptions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile) {
|
||||
final var options = new ChatCompletionsOptions(chat);
|
||||
options.setModel(modelProfile.modelName());
|
||||
options.setMaxTokens(modelProfile.maxTokens());
|
||||
options.setTemperature(modelProfile.temperature());
|
||||
options.setTopP(modelProfile.topP());
|
||||
|
||||
final var client = this.createHttpClient(modelCategory, modelProfile);
|
||||
final var response = client.complete(options);
|
||||
|
||||
final CompletionFinishReason finishReason;
|
||||
if (CompletionsFinishReason.CONTENT_FILTERED.equals(response.getChoice().getFinishReason())) {
|
||||
finishReason = CompletionFinishReason.CONTENT_FILTERED;
|
||||
return options;
|
||||
}
|
||||
|
||||
private CompletionFinishReason mapFinishReason(final CompletionsFinishReason azureFinishReason) {
|
||||
if (CompletionsFinishReason.CONTENT_FILTERED.equals(azureFinishReason)) {
|
||||
return CompletionFinishReason.CONTENT_FILTERED;
|
||||
}
|
||||
else if (CompletionsFinishReason.TOKEN_LIMIT_REACHED.equals(response.getChoice().getFinishReason())) {
|
||||
finishReason = CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
else if (CompletionsFinishReason.TOKEN_LIMIT_REACHED.equals(azureFinishReason)) {
|
||||
return CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
}
|
||||
else {
|
||||
finishReason = CompletionFinishReason.DONE;
|
||||
return CompletionFinishReason.DONE;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private CompletionStatistics createStatistics(final LocalDateTime start, final int promptTokens, final int completionTokens, final ModelProfile modelProfile) {
|
||||
final var statistics = new CompletionStatistics(
|
||||
1,
|
||||
Duration.between(start, LocalDateTime.now()),
|
||||
response.getUsage().getPromptTokens(),
|
||||
response.getUsage().getCompletionTokens(),
|
||||
new BigDecimal(response.getUsage().getPromptTokens()).multiply(modelProfile.costInToken()).add(new BigDecimal(response.getUsage().getCompletionTokens()).multiply(modelProfile.costOutToken())),
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
new BigDecimal(promptTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(completionTokens).multiply(modelProfile.costOutToken())),
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
|
||||
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
|
||||
return statistics;
|
||||
}
|
||||
|
||||
private ChatCompletionsClient createHttpClient(final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
@@ -224,22 +297,35 @@ public final class AzureAiClient implements Client {
|
||||
return false;
|
||||
});
|
||||
|
||||
final reactor.netty.http.client.HttpClient httpClient = reactor.netty.http.client.HttpClient.create()
|
||||
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
|
||||
reactor.netty.http.client.HttpClient nettyHttpClient = reactor.netty.http.client.HttpClient.create()
|
||||
.option(io.netty.channel.ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
|
||||
|
||||
final NettyAsyncHttpClientBuilder azureHttpClientBuilder = new NettyAsyncHttpClientBuilder(httpClient);
|
||||
final HttpClient azureHttpClient = azureHttpClientBuilder
|
||||
.readTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.writeTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.build();
|
||||
if (this.proxyEnabled) {
|
||||
LOGGER.info("Proxy is enabled. Configuring proxy settings for host: {} port: {} user: {}", this.proxyHost, this.proxyPort, this.proxyUser);
|
||||
nettyHttpClient = nettyHttpClient.proxy(proxy -> proxy
|
||||
.type(reactor.netty.transport.ProxyProvider.Proxy.HTTP)
|
||||
.host(this.proxyHost)
|
||||
.port(this.proxyPort)
|
||||
.username(this.proxyUser)
|
||||
.password(password -> this.proxyPassword));
|
||||
}
|
||||
else {
|
||||
LOGGER.info("Proxy is not enabled. Skipping proxy configuration.");
|
||||
}
|
||||
|
||||
final HttpClient azureHttpClient = new NettyAsyncHttpClientBuilder(nettyHttpClient)
|
||||
.connectTimeout(Duration.ofMillis(CONNECT_TIMEOUT))
|
||||
.readTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.writeTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.build();
|
||||
|
||||
final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
|
||||
.credential(new AzureKeyCredential(modelProfile.apiKey()))
|
||||
.endpoint(modelProfile.url())
|
||||
.retryOptions(retryOptions)
|
||||
.retryPolicy(new RetryPolicy(retryOptions))
|
||||
.httpClient(azureHttpClient)
|
||||
.buildClient();
|
||||
return chatCompletionsClient;
|
||||
@@ -247,7 +333,7 @@ public final class AzureAiClient implements Client {
|
||||
}
|
||||
|
||||
private boolean shouldRetryError(final int statusCode) {
|
||||
return statusCode == 408 || statusCode == 429 || (statusCode >= 500 && statusCode < 600);
|
||||
return statusCode == 408 || statusCode == 429 || statusCode >= 500 && statusCode < 600;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package de.extio.lmlib.client.azureai;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
@@ -11,9 +12,24 @@ import de.extio.lmlib.token.Tokenizer;
|
||||
@ConditionalOnClass(name = {"com.azure.ai.inference.ChatCompletionsClient" })
|
||||
public class AzureAiClientAutoConfiguration {
|
||||
|
||||
@Value("${extproxy.enabled:false}")
|
||||
private boolean proxyEnabled;
|
||||
|
||||
@Value("${extproxy.host:}")
|
||||
private String proxyHost;
|
||||
|
||||
@Value("${extproxy.port:3128}")
|
||||
private int proxyPort;
|
||||
|
||||
@Value("${extproxy.username:}")
|
||||
private String proxyUser;
|
||||
|
||||
@Value("${extproxy.password:}")
|
||||
private String proxyPassword;
|
||||
|
||||
@Bean
|
||||
AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
|
||||
return new AzureAiClient(modelProfileService, tokenizer);
|
||||
return new AzureAiClient(modelProfileService, tokenizer, this.proxyEnabled, this.proxyHost, this.proxyPort, this.proxyUser, this.proxyPassword);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user