refactor: Remove Azure AI client implementation and related configuration
This commit is contained in:
extio-de
@@ -1,361 +0,0 @@
|
||||
package de.extio.lmlib.client.azureai;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.azure.ai.inference.ChatCompletionsClient;
|
||||
import com.azure.ai.inference.ChatCompletionsClientBuilder;
|
||||
import com.azure.ai.inference.models.ChatCompletionsOptions;
|
||||
import com.azure.ai.inference.models.ChatRequestAssistantMessage;
|
||||
import com.azure.ai.inference.models.ChatRequestMessage;
|
||||
import com.azure.ai.inference.models.ChatRequestSystemMessage;
|
||||
import com.azure.ai.inference.models.ChatRequestUserMessage;
|
||||
import com.azure.ai.inference.models.CompletionsFinishReason;
|
||||
import com.azure.core.credential.AzureKeyCredential;
|
||||
import com.azure.core.http.HttpClient;
|
||||
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
|
||||
import com.azure.core.http.policy.ExponentialBackoffOptions;
|
||||
import com.azure.core.http.policy.RetryOptions;
|
||||
import com.azure.core.http.policy.RetryPolicy;
|
||||
|
||||
|
||||
import de.extio.lmlib.client.Chunk;
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.CompletionStatistics;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.token.Tokenizer;import io.netty.handler.timeout.ReadTimeoutHandler;
|
||||
import io.netty.handler.timeout.WriteTimeoutHandler;
|
||||
|
||||
public final class AzureAiClient implements Client {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClient.class);
|
||||
|
||||
private static final int TIMEOUT = 300000;
|
||||
|
||||
private static final int CONNECT_TIMEOUT = 10000;
|
||||
|
||||
private final ConcurrentMap<String, ChatCompletionsClient> chatCompletionsClients = new ConcurrentHashMap<>();
|
||||
|
||||
private final ModelProfileService modelProfileService;
|
||||
|
||||
private final Tokenizer tokenizer;
|
||||
|
||||
private final boolean proxyEnabled;
|
||||
|
||||
private final String proxyHost;
|
||||
|
||||
private final int proxyPort;
|
||||
|
||||
private final String proxyUser;
|
||||
|
||||
private final String proxyPassword;
|
||||
|
||||
AzureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer, final boolean proxyEnabled, final String proxyHost, final int proxyPort, final String proxyUser, final String proxyPassword) {
|
||||
this.modelProfileService = modelProfileService;
|
||||
this.tokenizer = tokenizer;
|
||||
this.proxyEnabled = proxyEnabled;
|
||||
this.proxyHost = proxyHost;
|
||||
this.proxyPort = proxyPort;
|
||||
this.proxyUser = proxyUser;
|
||||
this.proxyPassword = proxyPassword;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ModelProvider getModelProvider() {
|
||||
return ModelProvider.AZURE_AI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
|
||||
return this.conversation(modelCategory, Conversation.create(system, text));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(final ModelCategory modelCategory, final Conversation conversation) {
|
||||
return this.requestConversation(modelCategory, null, conversation, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(final ModelProfile modelProfile, final Conversation conversation) {
|
||||
return this.requestConversation(null, modelProfile, conversation, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelCategory modelCategory, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
return this.requestConversation(modelCategory, null, conversation, chunkConsumer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion streamConversation(final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
return this.requestConversation(null, modelProfile, conversation, chunkConsumer);
|
||||
}
|
||||
|
||||
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
final List<ChatRequestMessage> chat = new ArrayList<>();
|
||||
|
||||
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
|
||||
Turn systemTurn = null;
|
||||
Turn userTurn = null;
|
||||
Turn assistantTurn = null;
|
||||
boolean first = true;
|
||||
|
||||
final AtomicInteger tokens = new AtomicInteger();
|
||||
|
||||
while (!turns.isEmpty()) {
|
||||
final Turn turn = turns.pop();
|
||||
switch (turn.type()) {
|
||||
case SYSTEM:
|
||||
systemTurn = turn;
|
||||
break;
|
||||
case USER:
|
||||
userTurn = turn;
|
||||
break;
|
||||
case ASSISTANT:
|
||||
assistantTurn = turn;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
|
||||
}
|
||||
|
||||
if (first && userTurn != null) {
|
||||
if (systemTurn != null) {
|
||||
chat.add(new ChatRequestSystemMessage(systemTurn.text()));
|
||||
}
|
||||
if (this.addChatRequestMessage(userTurn.text(), tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
|
||||
break;
|
||||
}
|
||||
userTurn = null;
|
||||
first = false;
|
||||
}
|
||||
else if (!first && assistantTurn != null && userTurn != null) {
|
||||
if (this.addChatRequestMessage(assistantTurn.text(), tokens, chat, modelProfile, ChatRequestAssistantMessage::new)) {
|
||||
break;
|
||||
}
|
||||
if (this.addChatRequestMessage(userTurn.text(), tokens, chat, modelProfile, ChatRequestUserMessage::new)) {
|
||||
break;
|
||||
}
|
||||
userTurn = null;
|
||||
assistantTurn = null;
|
||||
}
|
||||
}
|
||||
|
||||
return chat;
|
||||
}
|
||||
|
||||
private boolean addChatRequestMessage(final String text, final AtomicInteger tokens, final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Function<String, ChatRequestMessage> messageFactory) {
|
||||
var prompt = text;
|
||||
var trimmed = false;
|
||||
|
||||
final var textTokens = this.tokenizer.count(prompt, modelProfile);
|
||||
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
|
||||
if (tokens.addAndGet(textTokens) > maxLength) {
|
||||
var tokenized = this.tokenizer.tokenize(prompt, modelProfile);
|
||||
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
|
||||
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
|
||||
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
|
||||
trimmed = true;
|
||||
}
|
||||
|
||||
chat.add(messageFactory.apply(prompt));
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private Completion requestConversation(ModelCategory modelCategory, final ModelProfile modelProfile, final Conversation conversation, final Consumer<Chunk> chunkConsumer) {
|
||||
final ModelProfile profile;
|
||||
if (modelProfile != null) {
|
||||
profile = modelProfile;
|
||||
if (modelCategory == null) {
|
||||
modelCategory = new ModelCategory(modelProfile.modelName(), modelProfile.category());
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
profile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile(), modelCategory);
|
||||
}
|
||||
|
||||
if (profile == null || profile.modelProvider() != ModelProvider.AZURE_AI) {
|
||||
throw new IllegalArgumentException("Invalid ModelProfile");
|
||||
}
|
||||
|
||||
final var chat = this.createChats(conversation, profile);
|
||||
|
||||
if (chunkConsumer == null) {
|
||||
return this.requestCompletions(chat, profile);
|
||||
}
|
||||
else {
|
||||
return this.requestStreamCompletions(chat, profile, chunkConsumer);
|
||||
}
|
||||
}
|
||||
|
||||
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile) {
|
||||
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var options = this.createChatCompletionsOptions(chat, modelProfile);
|
||||
final var client = this.createHttpClient(modelProfile);
|
||||
final var response = client.complete(options);
|
||||
|
||||
final var finishReason = this.mapFinishReason(response.getChoice().getFinishReason());
|
||||
final var statistics = this.createStatistics(start, response.getUsage().getPromptTokens(), response.getUsage().getCompletionTokens(), modelProfile);
|
||||
|
||||
return new Completion(response.getChoice().getMessage().getContent(), null, finishReason, statistics);
|
||||
}
|
||||
|
||||
private Completion requestStreamCompletions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer) {
|
||||
LOGGER.debug("Requesting Streaming ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var options = this.createChatCompletionsOptions(chat, modelProfile);
|
||||
final var client = this.createHttpClient(modelProfile);
|
||||
final var completionsStream = client.completeStream(options);
|
||||
|
||||
final StringBuilder contentBuilder = new StringBuilder();
|
||||
final AtomicInteger promptTokens = new AtomicInteger(0);
|
||||
final AtomicInteger completionTokens = new AtomicInteger(0);
|
||||
final CompletionFinishReason[] finishReasonHolder = {CompletionFinishReason.DONE};
|
||||
|
||||
completionsStream.forEach(streamResponse -> {
|
||||
if (streamResponse.getChoices() != null && !streamResponse.getChoices().isEmpty()) {
|
||||
final var choice = streamResponse.getChoices().get(0);
|
||||
|
||||
if (choice.getFinishReason() != null) {
|
||||
finishReasonHolder[0] = this.mapFinishReason(choice.getFinishReason());
|
||||
}
|
||||
|
||||
if (choice.getDelta() != null && choice.getDelta().getContent() != null) {
|
||||
final String content = choice.getDelta().getContent();
|
||||
contentBuilder.append(content);
|
||||
chunkConsumer.accept(new Chunk(content, null));
|
||||
}
|
||||
}
|
||||
|
||||
if (streamResponse.getUsage() != null) {
|
||||
promptTokens.set(streamResponse.getUsage().getPromptTokens());
|
||||
completionTokens.set(streamResponse.getUsage().getCompletionTokens());
|
||||
}
|
||||
});
|
||||
|
||||
final var statistics = this.createStatistics(start, promptTokens.get(), completionTokens.get(), modelProfile);
|
||||
|
||||
return new Completion(contentBuilder.toString(), null, finishReasonHolder[0], statistics);
|
||||
}
|
||||
|
||||
private ChatCompletionsOptions createChatCompletionsOptions(final List<ChatRequestMessage> chat, final ModelProfile modelProfile) {
|
||||
final var options = new ChatCompletionsOptions(chat);
|
||||
options.setModel(modelProfile.modelName());
|
||||
options.setMaxTokens(modelProfile.maxTokens());
|
||||
options.setTemperature(modelProfile.temperature());
|
||||
options.setTopP(modelProfile.topP());
|
||||
return options;
|
||||
}
|
||||
|
||||
private CompletionFinishReason mapFinishReason(final CompletionsFinishReason azureFinishReason) {
|
||||
if (CompletionsFinishReason.CONTENT_FILTERED.equals(azureFinishReason)) {
|
||||
return CompletionFinishReason.CONTENT_FILTERED;
|
||||
}
|
||||
else if (CompletionsFinishReason.TOKEN_LIMIT_REACHED.equals(azureFinishReason)) {
|
||||
return CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
}
|
||||
else {
|
||||
return CompletionFinishReason.DONE;
|
||||
}
|
||||
}
|
||||
|
||||
private CompletionStatistics createStatistics(final LocalDateTime start, final int promptTokens, final int completionTokens, final ModelProfile modelProfile) {
|
||||
final var statistics = new CompletionStatistics(
|
||||
1,
|
||||
Duration.between(start, LocalDateTime.now()),
|
||||
promptTokens,
|
||||
0,
|
||||
completionTokens,
|
||||
0,
|
||||
new BigDecimal(promptTokens).multiply(modelProfile.costPerInToken()).add(new BigDecimal(completionTokens).multiply(modelProfile.costPerOutToken())),
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
return statistics;
|
||||
}
|
||||
|
||||
private ChatCompletionsClient createHttpClient(final ModelProfile modelProfile) {
|
||||
return this.chatCompletionsClients.computeIfAbsent(modelProfile.category(), key -> {
|
||||
final ExponentialBackoffOptions exponentialOptions = new ExponentialBackoffOptions()
|
||||
.setMaxRetries(99)
|
||||
.setBaseDelay(Duration.ofSeconds(1))
|
||||
.setMaxDelay(Duration.ofSeconds(10));
|
||||
|
||||
final RetryOptions retryOptions = new RetryOptions(exponentialOptions)
|
||||
.setShouldRetryCondition(httpResponse -> {
|
||||
if (httpResponse != null) {
|
||||
if (httpResponse.getThrowable() != null) {
|
||||
return true;
|
||||
}
|
||||
if (httpResponse.getResponse() != null) {
|
||||
return this.shouldRetryError(httpResponse.getResponse().getStatusCode());
|
||||
}
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
reactor.netty.http.client.HttpClient nettyHttpClient = reactor.netty.http.client.HttpClient.create()
|
||||
.option(io.netty.channel.ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT)
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)).addHandlerLast(new WriteTimeoutHandler(TIMEOUT, TimeUnit.MILLISECONDS)));
|
||||
|
||||
if (this.proxyEnabled) {
|
||||
LOGGER.info("Proxy is enabled. Configuring proxy settings for host: {} port: {} user: {}", this.proxyHost, this.proxyPort, this.proxyUser);
|
||||
nettyHttpClient = nettyHttpClient.proxy(proxy -> proxy
|
||||
.type(reactor.netty.transport.ProxyProvider.Proxy.HTTP)
|
||||
.host(this.proxyHost)
|
||||
.port(this.proxyPort)
|
||||
.username(this.proxyUser)
|
||||
.password(password -> this.proxyPassword));
|
||||
}
|
||||
else {
|
||||
LOGGER.info("Proxy is not enabled. Skipping proxy configuration.");
|
||||
}
|
||||
|
||||
final HttpClient azureHttpClient = new NettyAsyncHttpClientBuilder(nettyHttpClient)
|
||||
.connectTimeout(Duration.ofMillis(CONNECT_TIMEOUT))
|
||||
.readTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.writeTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.responseTimeout(Duration.ofMillis(TIMEOUT))
|
||||
.build();
|
||||
|
||||
final ChatCompletionsClient chatCompletionsClient = new ChatCompletionsClientBuilder()
|
||||
.credential(new AzureKeyCredential(modelProfile.apiKey()))
|
||||
.endpoint(modelProfile.url())
|
||||
.retryPolicy(new RetryPolicy(retryOptions))
|
||||
.httpClient(azureHttpClient)
|
||||
.buildClient();
|
||||
return chatCompletionsClient;
|
||||
});
|
||||
}
|
||||
|
||||
private boolean shouldRetryError(final int statusCode) {
|
||||
return statusCode == 408 || statusCode == 429 || statusCode >= 500 && statusCode < 600;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package de.extio.lmlib.client.azureai;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
|
||||
@AutoConfiguration
|
||||
@ConditionalOnClass(name = {"com.azure.ai.inference.ChatCompletionsClient" })
|
||||
public class AzureAiClientAutoConfiguration {
|
||||
|
||||
@Value("${extproxy.enabled:false}")
|
||||
private boolean proxyEnabled;
|
||||
|
||||
@Value("${extproxy.host:}")
|
||||
private String proxyHost;
|
||||
|
||||
@Value("${extproxy.port:3128}")
|
||||
private int proxyPort;
|
||||
|
||||
@Value("${extproxy.username:}")
|
||||
private String proxyUser;
|
||||
|
||||
@Value("${extproxy.password:}")
|
||||
private String proxyPassword;
|
||||
|
||||
@Bean
|
||||
AzureAiClient azureAiClient(final ModelProfileService modelProfileService, final Tokenizer tokenizer) {
|
||||
return new AzureAiClient(modelProfileService, tokenizer, this.proxyEnabled, this.proxyHost, this.proxyPort, this.proxyUser, this.proxyPassword);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -46,6 +46,6 @@ public record ModelProfile(
|
||||
}
|
||||
|
||||
public static enum ModelProvider {
|
||||
AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION;
|
||||
OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ import io.netty.util.internal.ThreadLocalRandom;
|
||||
/**
|
||||
* These tests have been verified with Llama 3.1 8B, Gemma2 27B, Gemma3 27B and GPT-OSS models
|
||||
*/
|
||||
@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
|
||||
@Disabled("This test requires a running Llama server or a cloud subscription")
|
||||
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
|
||||
@SpringBootConfiguration
|
||||
@ComponentScan(basePackages = "de.extio.lmlib")
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
package de.extio.lmlib.client.azureai;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.PropertySource;
|
||||
|
||||
import de.extio.lmlib.client.ClientService;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.Conversation.TurnType;
|
||||
import de.extio.lmlib.grader.Grader2;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
|
||||
@Disabled("This test requires an Azure subscription (setup azure key in model profile)")
|
||||
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
|
||||
@SpringBootConfiguration
|
||||
@ComponentScan(basePackages = "de.extio.lmlib")
|
||||
@PropertySource("classpath:/application-test.properties")
|
||||
public class AzureAiClientTest {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClientTest.class);
|
||||
|
||||
@Autowired
|
||||
private AzureAiClient client;
|
||||
|
||||
@Autowired
|
||||
private ModelProfileService modelProfileService;
|
||||
|
||||
@Autowired
|
||||
private ClientService clientService;
|
||||
|
||||
@Test
|
||||
void completion() throws Exception {
|
||||
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
|
||||
final var completion = this.client.conversation(
|
||||
modelProfile,
|
||||
Conversation.create("You are a clever story teller", "Continue this story:\nOnce upon a time, there was a spaceship travelling to a black hole. The crew were cats!"));
|
||||
LOGGER.info(completion.response());
|
||||
|
||||
assertTrue(Grader2.assessScoreBinary("Does the story continue about a spaceship with cat crew?", completion.response(), modelProfile, this.clientService));
|
||||
}
|
||||
|
||||
@Test
|
||||
void completionLarge() throws Exception {
|
||||
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
|
||||
final var completion = this.client.conversation(
|
||||
modelProfile,
|
||||
Conversation.create("You are a clever story teller", "Continue this story:\nOnce upon a time, there was a spaceship travelling to a black hole. The crew were cats!"));
|
||||
LOGGER.info(completion.response());
|
||||
|
||||
assertTrue(Grader2.assessScoreBinary("Does the story continue about a spaceship with cat crew?", completion.response(), modelProfile, this.clientService));
|
||||
}
|
||||
|
||||
@Test
|
||||
void chat() throws Exception {
|
||||
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
|
||||
final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Berlin.");
|
||||
conversation.addTurn(new Turn(TurnType.ASSISTANT,
|
||||
"T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius."));
|
||||
conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?"));
|
||||
|
||||
final var completion = this.client.conversation(modelProfile, conversation);
|
||||
LOGGER.info(completion.response());
|
||||
|
||||
assertTrue(Grader2.assessScoreBinary("Does the response mention summer weather?", completion.response(), modelProfile, this.clientService));
|
||||
}
|
||||
|
||||
@Test
|
||||
void longPrompt() throws Exception {
|
||||
final var modelProfile = this.modelProfileService.getModelProfile("profile.model.azure");
|
||||
final var completion = this.client.conversation(
|
||||
modelProfile,
|
||||
Conversation.create("You are a helpful assistant", "Count the numbers: 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9"));
|
||||
LOGGER.info(completion.response());
|
||||
|
||||
assertTrue(Grader2.assessScoreBinary("Did the assistant count the numbers?", completion.response(), modelProfile, this.clientService));
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.Conversation.TurnType;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
|
||||
@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
|
||||
@Disabled("This test requires a running Llama server or a cloud subscription (setup key in model profile)")
|
||||
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
|
||||
@SpringBootConfiguration
|
||||
@ComponentScan(basePackages = "de.extio.lmlib")
|
||||
|
||||
@@ -41,7 +41,7 @@ import de.extio.lmlib.grader.agent.AnswerGrader2;
|
||||
import de.extio.lmlib.grader.agent.AnswerGraderBinary;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
|
||||
@Disabled("This test requires a running Llama server or an Azure subscription (setup azure key in model profile)")
|
||||
@Disabled("This test requires a running Llama server or a cloud subscription (setup key in model profile)")
|
||||
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
|
||||
@SpringBootConfiguration
|
||||
@ComponentScan(basePackages = "de.extio.lmlib")
|
||||
|
||||
@@ -6,7 +6,7 @@ profile.model.cold=local-chatcompletion
|
||||
|
||||
profile.model.textcompletion=local-completion
|
||||
profile.model.chatcompletion=local-chatcompletion
|
||||
profile.model.azure=llama3.3-70b-azure
|
||||
#profile.model.azure=llama3.3-70b-azure
|
||||
|
||||
agent.threads=2
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
category=AZURE
|
||||
prompts=
|
||||
tokenEncoding=cl100k_base
|
||||
maxTokens=1500
|
||||
maxContextLength=32767
|
||||
temperature=0.2
|
||||
topP=1.0
|
||||
modelProvider=AZURE_AI
|
||||
modelName=Llama-3.3-70B-Instruct
|
||||
url=XXX
|
||||
apiKey=XXX
|
||||
cost1MInTokens=0.71
|
||||
cost1MCachedInTokens=0.71
|
||||
cost1MOutTokens=0.71
|
||||
cost1MReasoningOutTokens=0
|
||||
Reference in New Issue
Block a user