From 2350740136ef8fa152cbf539369864bf54728aab Mon Sep 17 00:00:00 2001 From: extio-de Date: Fri, 27 Jun 2025 23:39:17 +0200 Subject: [PATCH] Introduced OpenAI chat completion client; --- README.md | 10 +- .../lmlib/client/azureai/AzureAiClient.java | 4 +- .../lmlib/client/oai/ModelNameSupplier.java | 2 +- .../completion/AbstractCompletionClient.java | 158 ++++++++++ .../oai/completion/TextCompletionClient.java | 276 ------------------ .../lmlib/client/oai/completion/Usage.java | 2 +- .../oai/completion/chat/ChatChoice.java | 40 +++ .../completion/chat/ChatCompletionClient.java | 148 ++++++++++ .../chat/ChatCompletionRequest.java | 87 ++++++ .../chat/ChatCompletionResponse.java | 82 ++++++ .../oai/completion/chat/ChatMessage.java | 37 +++ .../oai/completion/{ => text}/Choice.java | 2 +- .../{ => text}/CompletionRequest.java | 2 +- .../{ => text}/CompletionResponse.java | 4 +- .../completion/text/TextCompletionClient.java | 149 ++++++++++ .../de/extio/lmlib/profile/ModelProfile.java | 2 +- .../lmlib/profile/ModelProfileService.java | 8 +- ...entTest.java => CompletionClientTest.java} | 48 ++- .../resources/local-completion.properties | 4 +- 19 files changed, 757 insertions(+), 308 deletions(-) create mode 100755 src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java delete mode 100755 src/main/java/de/extio/lmlib/client/oai/completion/TextCompletionClient.java create mode 100755 src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java create mode 100755 src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java create mode 100755 src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java create mode 100755 src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java create mode 100644 src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java rename src/main/java/de/extio/lmlib/client/oai/completion/{ => text}/Choice.java (95%) rename src/main/java/de/extio/lmlib/client/oai/completion/{ => text}/CompletionRequest.java (96%) rename src/main/java/de/extio/lmlib/client/oai/completion/{ => text}/CompletionResponse.java (95%) create mode 100755 src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java rename src/test/java/de/extio/lmlib/client/oai/{TextCompletionClientTest.java => CompletionClientTest.java} (71%) diff --git a/README.md b/README.md index dc3019d..34e6164 100644 --- a/README.md +++ b/README.md @@ -70,13 +70,17 @@ If you use jtokkit tokenizer: jtokkit -It is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context). +The library supports both chat completions and (legacy) text completions. Choose one of these options in your model profile: -Set the following property in your model profile: + modelProvider=OAI_CHAT_COMPLETION + modelProvider=OAI_TEXT_COMPLETION + +Chat completions are recommended for most use cases and easy to use, since the library only needs to send the actual conversation to the server, whereas for text completions, the library needs to create a raw prompt that includes the correct instruction formatting (e.g. special tokens). Text completions are still preferred if you need more control over the raw prompt or if you don't use an instruct model. For text completions, it is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context). +To activate a prompt template, set the following property in your model profile: prompts=llama3 -Example profile: src/test/resources/llama3-local.properties +Example profile: src/test/resources/xxx-local.properties ## Azure AI client diff --git a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java index c59c1f8..e750abf 100755 --- a/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java +++ b/src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java @@ -106,7 +106,7 @@ public final class AzureAiClient implements Client { final var chat = this.createChats(conversation, modelProfile); - return this.requestCompletionsContinuations(chat, modelCategory, modelProfile); + return this.requestCompletions(chat, modelCategory, modelProfile); } private List createChats(final Conversation conversation, final ModelProfile modelProfile) { @@ -180,7 +180,7 @@ public final class AzureAiClient implements Client { return trimmed; } - private Completion requestCompletionsContinuations(final List chat, final ModelCategory modelCategory, final ModelProfile modelProfile) { + private Completion requestCompletions(final List chat, final ModelCategory modelCategory, final ModelProfile modelProfile) { LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName()); final LocalDateTime start = LocalDateTime.now(); diff --git a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java b/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java index 8702f69..876f51b 100755 --- a/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java +++ b/src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java @@ -30,7 +30,7 @@ public class ModelNameSupplier implements InitializingBean { @Override public void afterPropertiesSet() throws Exception { - final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION); + final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION, ModelProvider.OAI_CHAT_COMPLETION); for (final var url : urls) { this.countDownLatches.put(url, new CountDownLatch(1)); this.fetchModelName(url); diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java new file mode 100755 index 0000000..1b27649 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -0,0 +1,158 @@ +package de.extio.lmlib.client.oai.completion; + +import java.math.BigDecimal; +import java.text.DecimalFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.reactive.function.client.WebClient; + +import de.extio.lmlib.client.Client; +import de.extio.lmlib.client.Completion; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.oai.ModelNameSupplier; +import de.extio.lmlib.profile.ModelCategory; +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; +import de.extio.lmlib.profile.ModelProfileService; +import de.extio.lmlib.token.Tokenizer; + +public abstract class AbstractCompletionClient implements Client, DisposableBean { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class); + + @Autowired + @Qualifier("lmLibWebClientBuilder") + protected WebClient.Builder webClientBuilder; + + @Autowired + protected ModelNameSupplier modelNameSupplier; + + @Autowired + protected Tokenizer tokenizer; + + @Autowired + protected ModelProfileService modelProfileService; + + @Value("${client.collectStatistics:false}") + protected boolean collectStatistics; + + protected final CompletionStatistics totalStatistics = new CompletionStatistics(); + + @Override + public Completion completion(final ModelCategory modelCategory, final String system, final String text) { + return this.conversation(modelCategory, Conversation.create(system, text)); + } + + @Override + public Completion conversation(ModelCategory modelCategory, final Conversation conversation) { + if (modelCategory == null) { + modelCategory = ModelCategory.MEDIUM; + } + final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile()); + if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) { + throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile()); + } + + return this.requestCompletion(conversation, modelCategory, modelProfile); + } + + protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile); + + protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) { + int inTokens = 0; + int outTokens = 0; + if (usage != null && usage.getPromptTokens() > 0) { + inTokens = usage.getPromptTokens(); + outTokens = usage.getCompletionTokens(); + } + else { + if (prompt != null) { + inTokens = this.tokenizer.count(prompt, modelProfile); + } + if (response != null) { + outTokens = this.tokenizer.count(response, modelProfile); + } + } + + final var statistics = new de.extio.lmlib.client.CompletionStatistics( + 1, + Duration.between(start, LocalDateTime.now()), + inTokens, + outTokens, + new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())), + false); + LOGGER.debug("{}", statistics); + if (this.collectStatistics) { + this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost()); + } + return statistics; + } + + @Override + public void destroy() throws Exception { + if (this.collectStatistics && this.totalStatistics.requests > 0) { + LOGGER.info("Client Statistics: {}", this.totalStatistics); + } + } + + static class CompletionStatistics { + + private final Instant start = Instant.now(); + + private int requests; + + private Duration duration = Duration.ofMillis(0l); + + private long inTokens; + + private long outTokens; + + private BigDecimal cost = BigDecimal.ZERO; + + synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) { + this.requests++; + this.duration = this.duration.plus(duration); + this.inTokens += inTokens; + this.outTokens += outTokens; + this.cost = this.cost.add(cost); + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("CompletionStatistics [requests="); + builder.append(this.requests); + builder.append(", duration="); + builder.append(this.duration); + builder.append(", inTokens="); + builder.append(this.inTokens); + builder.append(", outTokens="); + builder.append(this.outTokens); + builder.append(", averageTps="); + builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests))); + builder.append(", averageOutTps="); + builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0))); + builder.append(", effectiveDuration="); + final var effectiveDuration = Duration.between(this.start, Instant.now()); + builder.append(effectiveDuration); + builder.append(", effectiveTps="); + builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests))); + builder.append(", effectiveOutTps="); + builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0))); + builder.append(", cost="); + builder.append(new DecimalFormat("0.###").format(cost.doubleValue())); + builder.append("]"); + return builder.toString(); + } + + } + +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/TextCompletionClient.java deleted file mode 100755 index c00d94e..0000000 --- a/src/main/java/de/extio/lmlib/client/oai/completion/TextCompletionClient.java +++ /dev/null @@ -1,276 +0,0 @@ -package de.extio.lmlib.client.oai.completion; - -import java.math.BigDecimal; -import java.text.DecimalFormat; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpMethod; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; - -import de.extio.lmlib.client.Client; -import de.extio.lmlib.client.Completion; -import de.extio.lmlib.client.CompletionFinishReason; -import de.extio.lmlib.client.Conversation; -import de.extio.lmlib.client.Conversation.Turn; -import de.extio.lmlib.client.oai.ModelNameSupplier; -import de.extio.lmlib.profile.ModelCategory; -import de.extio.lmlib.profile.ModelProfile; -import de.extio.lmlib.profile.ModelProfile.ModelProvider; -import de.extio.lmlib.profile.ModelProfileService; -import de.extio.lmlib.prompt.PromptStrategy; -import de.extio.lmlib.prompt.PromptStrategyFactory; -import de.extio.lmlib.token.Tokenizer; -import reactor.util.retry.Retry; - -@Component -public class TextCompletionClient implements Client, DisposableBean { - - private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class); - - @Autowired - @Qualifier("lmLibWebClientBuilder") - private WebClient.Builder webClientBuilder; - - @Autowired - private PromptStrategyFactory promptStrategyFactory; - - @Autowired - private ModelNameSupplier modelNameSupplier; - - @Autowired - private Tokenizer tokenizer; - - @Autowired - private ModelProfileService modelProfileService; - - @Value("${client.collectStatistics:false}") - private boolean collectStatistics; - - private final CompletionStatistics totalStatistics = new CompletionStatistics(); - - @Override - public ModelProvider getModelProvider() { - return ModelProvider.OAI_TEXT_COMPLETION; - } - - @Override - public Completion completion(final ModelCategory modelCategory, final String system, final String text) { - return this.conversation(modelCategory, Conversation.create(system, text)); - } - - @Override - public Completion conversation(ModelCategory modelCategory, final Conversation conversation) { - if (modelCategory == null) { - modelCategory = ModelCategory.MEDIUM; - } - final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile()); - if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION) { - throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile()); - } - - final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt()); - if (promptStrategy == null) { - throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt()); - } - - final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy); - - return this.requestCompletion(prompt, modelProfile, promptStrategy); - } - - private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) { - StringBuilder prompt = this.formatConversation(conversation, promptStrategy); - - List tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile); - final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens - if (tokenized.size() > maxLength) { - LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength); - tokenized = tokenized.subList(0, maxLength); - prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile)); - } - - LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size()); - - return prompt.toString(); - } - - private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) { - final StringBuilder prompt = new StringBuilder(); - - final Deque turns = new ArrayDeque<>(conversation.getConversation()); - Turn systemTurn = null; - Turn userTurn = null; - Turn assistantTurn = null; - boolean first = true; - - while (!turns.isEmpty()) { - final Turn turn = turns.pop(); - switch (turn.type()) { - case SYSTEM: - systemTurn = turn; - break; - case USER: - userTurn = turn; - break; - case ASSISTANT: - assistantTurn = turn; - break; - default: - throw new IllegalArgumentException("Unknown turn type: " + turn.type()); - } - - if (first && userTurn != null) { - prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text())); - userTurn = null; - first = false; - } - else if (!first && assistantTurn != null && userTurn != null) { - promptStrategy.next(prompt, assistantTurn.text(), userTurn.text()); - userTurn = null; - assistantTurn = null; - } - } - - return prompt; - } - - private Completion requestCompletion(final String prompt, final ModelProfile modelProfile, final PromptStrategy promptStrategy) { - LOGGER.debug("Requesting completion at {}", modelProfile.url()); - final LocalDateTime start = LocalDateTime.now(); - - final var request = new CompletionRequest(); - request.setModel(this.modelNameSupplier.getModelName(modelProfile.url())); - request.setPrompt(prompt); - request.setMaxTokens(modelProfile.maxTokens()); - request.setTemperature(modelProfile.temperature()); - request.setTopP(modelProfile.topP()); - request.setStream(false); - - final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); - final var response = webClient - .method(HttpMethod.POST) - .uri(uriBuilder -> uriBuilder.path("/v1/completions").build()) - .header("Content-Type", "application/json") - .bodyValue(request) - .retrieve() - .bodyToMono(CompletionResponse.class) - .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) - .block(); - - String content; - CompletionFinishReason finishReason; - if (response.getChoices() != null && !response.getChoices().isEmpty()) { - final var choice = response.getChoices().getFirst(); - content = promptStrategy.removeEOT(choice.getText()); - finishReason = switch (choice.getFinishReason()) { - case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED; - case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED; - default -> CompletionFinishReason.DONE; - }; - } - else if (response.getContent() != null) { - content = promptStrategy.removeEOT(response.getContent()); - finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED; - } - else { - throw new IllegalStateException("No parseable response content from server"); - } - - int inTokens; - int outTokens; - if (response.getUsage() != null && response.getUsage().getPromptTokens() > 0) { - inTokens = response.getUsage().getPromptTokens(); - outTokens = response.getUsage().getCompletionTokens(); - } - else { - inTokens = this.tokenizer.count(prompt, modelProfile); - outTokens = this.tokenizer.count(content, modelProfile); - } - - final var statistics = new de.extio.lmlib.client.CompletionStatistics( - 1, - Duration.between(start, LocalDateTime.now()), - inTokens, - outTokens, - new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())), - false); - LOGGER.debug("{}", statistics); - if (this.collectStatistics) { - this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost()); - } - - return new Completion(content, finishReason, statistics); - } - - @Override - public void destroy() throws Exception { - if (this.collectStatistics && this.totalStatistics.requests > 0) { - LOGGER.info("Client Statistics: {}", this.totalStatistics); - } - } - - static class CompletionStatistics { - - private final Instant start = Instant.now(); - - private int requests; - - private Duration duration = Duration.ofMillis(0l); - - private long inTokens; - - private long outTokens; - - private BigDecimal cost = BigDecimal.ZERO; - - synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) { - this.requests++; - this.duration = this.duration.plus(duration); - this.inTokens += inTokens; - this.outTokens += outTokens; - this.cost = this.cost.add(cost); - } - - @Override - public String toString() { - final StringBuilder builder = new StringBuilder(); - builder.append("CompletionStatistics [requests="); - builder.append(this.requests); - builder.append(", duration="); - builder.append(this.duration); - builder.append(", inTokens="); - builder.append(this.inTokens); - builder.append(", outTokens="); - builder.append(this.outTokens); - builder.append(", averageTps="); - builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests))); - builder.append(", averageOutTps="); - builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0))); - builder.append(", effectiveDuration="); - final var effectiveDuration = Duration.between(this.start, Instant.now()); - builder.append(effectiveDuration); - builder.append(", effectiveTps="); - builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests))); - builder.append(", effectiveOutTps="); - builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0))); - builder.append(", cost="); - builder.append(new DecimalFormat("0.###").format(cost.doubleValue())); - builder.append("]"); - return builder.toString(); - } - - } - -} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java index 0aa4b65..e0d3f62 100644 --- a/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/Usage.java @@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion; import com.fasterxml.jackson.annotation.JsonProperty; -final class Usage { +public final class Usage { private int completionTokens; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java new file mode 100755 index 0000000..0a62ee3 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java @@ -0,0 +1,40 @@ +package de.extio.lmlib.client.oai.completion.chat; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class ChatChoice { + + public static final String FINISH_REASON_STOP = "stop"; + + public static final String FINISH_REASON_LENGTH = "length"; + + public static final String FINISH_REASON_CONTENT_FILTER = "content_filter"; + + private ChatMessage message; + + private String finishReason; + + @JsonProperty("message") + public ChatMessage getMessage() { + return message; + } + + public void setMessage(final ChatMessage message) { + this.message = message; + } + + @JsonProperty("finish_reason") + public String getFinishReason() { + return this.finishReason; + } + + public void setFinishReason(final String finishReason) { + this.finishReason = finishReason; + } + + @Override + public String toString() { + return "ChatChoice [message=" + message + ", finishReason=" + finishReason + "]"; + } + +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java new file mode 100755 index 0000000..265229d --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -0,0 +1,148 @@ +package de.extio.lmlib.client.oai.completion.chat; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpMethod; +import org.springframework.stereotype.Component; + +import de.extio.lmlib.client.Completion; +import de.extio.lmlib.client.CompletionFinishReason; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.Conversation.Turn; +import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; +import de.extio.lmlib.profile.ModelCategory; +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; +import reactor.util.retry.Retry; + +@Component +public class ChatCompletionClient extends AbstractCompletionClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(ChatCompletionClient.class); + + @Override + public ModelProvider getModelProvider() { + return ModelProvider.OAI_CHAT_COMPLETION; + } + + @Override + protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) { + final var chat = createChats(conversation, modelProfile); + + final var request = new ChatCompletionRequest(); + request.setModel(this.modelNameSupplier.getModelName(modelProfile.url())); + request.setMessages(chat); + request.setMaxTokens(modelProfile.maxTokens()); + request.setMaxCompletionTokens(modelProfile.maxTokens()); + request.setTemperature(modelProfile.temperature()); + request.setTopP(modelProfile.topP()); + request.setStream(false); + + LOGGER.debug("Requesting chat completion at {}", modelProfile.url()); + final LocalDateTime start = LocalDateTime.now(); + final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); + final var response = webClient + .method(HttpMethod.POST) + .uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build()) + .header("Content-Type", "application/json") + .bodyValue(request) + .retrieve() + .bodyToMono(ChatCompletionResponse.class) + .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) + .block(); + + if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) { + throw new IllegalStateException("No parseable response content from server"); + } + + final var choice = response.getChoices().getFirst(); + final var content = response.getChoices().getFirst().getMessage().getContent(); + final var finishReason = switch (choice.getFinishReason()) { + case ChatChoice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED; + case ChatChoice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED; + default -> CompletionFinishReason.DONE; + }; + final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content); + return new Completion(content, finishReason, statistics); + } + + private List createChats(final Conversation conversation, final ModelProfile modelProfile) { + final List chat = new ArrayList<>(); + + final Deque turns = new ArrayDeque<>(conversation.getConversation()); + Turn systemTurn = null; + Turn userTurn = null; + Turn assistantTurn = null; + boolean first = true; + + final AtomicInteger tokens = new AtomicInteger(); + + while (!turns.isEmpty()) { + final Turn turn = turns.pop(); + switch (turn.type()) { + case SYSTEM: + systemTurn = turn; + break; + case USER: + userTurn = turn; + break; + case ASSISTANT: + assistantTurn = turn; + break; + default: + throw new IllegalArgumentException("Unknown turn type: " + turn.type()); + } + + if (first && userTurn != null) { + if (systemTurn != null) { + chat.add(new ChatMessage("system", systemTurn.text())); + } + if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) { + break; + } + userTurn = null; + first = false; + } + else if (!first && assistantTurn != null && userTurn != null) { + if (this.addChatRequestMessage("assistant", assistantTurn.text(), tokens, chat, modelProfile)) { + break; + } + if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) { + break; + } + userTurn = null; + assistantTurn = null; + } + } + + return chat; + } + + private boolean addChatRequestMessage(final String role, final String text, final AtomicInteger tokens, final List chat, final ModelProfile modelProfile) { + var prompt = text; + var trimmed = false; + + final var textTokens = this.tokenizer.count(prompt, modelProfile); + final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens + if (tokens.addAndGet(textTokens) > maxLength) { + var tokenized = this.tokenizer.tokenize(prompt, modelProfile); + LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength); + tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength)); + prompt = this.tokenizer.detokenize(tokenized, modelProfile); + trimmed = true; + } + + chat.add(new ChatMessage(role, prompt)); + + return trimmed; + } + +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java new file mode 100755 index 0000000..cc842f6 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java @@ -0,0 +1,87 @@ +package de.extio.lmlib.client.oai.completion.chat; + +import java.util.List; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class ChatCompletionRequest { + + private String model; + + private List messages; + + private Integer maxTokens; + + private Integer maxCompletionTokens; + + private Double temperature; + + private Double topP; + + private Boolean stream; + + @JsonProperty("model") + public String getModel() { + return this.model; + } + + public void setModel(final String model) { + this.model = model; + } + + @JsonProperty("messages") + public List getMessages() { + return this.messages; + } + + public void setMessages(final List messages) { + this.messages = messages; + } + + @JsonProperty("max_tokens") + public Integer getMaxTokens() { + return this.maxTokens; + } + + public void setMaxTokens(final Integer maxTokens) { + this.maxTokens = maxTokens; + } + + @JsonProperty("max_completion_tokens") + public Integer getMaxCompletionTokens() { + return this.maxCompletionTokens; + } + + public void setMaxCompletionTokens(final Integer maxCompletionTokens) { + this.maxCompletionTokens = maxCompletionTokens; + } + + @JsonProperty("temperature") + public Double getTemperature() { + return this.temperature; + } + + public void setTemperature(final Double temperature) { + this.temperature = temperature; + } + + @JsonProperty("top_p") + public Double getTopP() { + return this.topP; + } + + public void setTopP(final Double topP) { + this.topP = topP; + } + + @JsonProperty("stream") + public Boolean isStream() { + return this.stream; + } + + public void setStream(final Boolean stream) { + this.stream = stream; + } +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java new file mode 100755 index 0000000..12bcd0a --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java @@ -0,0 +1,82 @@ +package de.extio.lmlib.client.oai.completion.chat; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import de.extio.lmlib.client.oai.completion.Usage; + +final class ChatCompletionResponse { + + private String id; + + private String object; + + private Integer created; + + private List choices; + + private Usage usage; + + @JsonProperty("id") + public String getId() { + return this.id; + } + + public void setId(final String id) { + this.id = id; + } + + @JsonProperty("object") + public String getObject() { + return this.object; + } + + public void setObject(final String object) { + this.object = object; + } + + @JsonProperty("created") + public Integer getCreated() { + return this.created; + } + + public void setCreated(final Integer created) { + this.created = created; + } + + @JsonProperty("choices") + public List getChoices() { + return this.choices; + } + + public void setChoices(final List choices) { + this.choices = choices; + } + + @JsonProperty("usage") + public Usage getUsage() { + return usage; + } + + public void setUsage(final Usage usage) { + this.usage = usage; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("CompletionResponse [id="); + builder.append(this.id); + builder.append(", object="); + builder.append(this.object); + builder.append(", created="); + builder.append(this.created); + builder.append(", choices="); + builder.append(this.choices); + builder.append(", usage="); + builder.append(this.usage); + builder.append("]"); + return builder.toString(); + } +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java new file mode 100644 index 0000000..af8fdd8 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java @@ -0,0 +1,37 @@ +package de.extio.lmlib.client.oai.completion.chat; + +import com.fasterxml.jackson.annotation.JsonProperty; + +final class ChatMessage { + + private String role; + + private String content; + + public ChatMessage() { + } + + public ChatMessage(final String role, final String content) { + this.role = role; + this.content = content; + } + + @JsonProperty("role") + public String getRole() { + return role; + } + + public void setRole(final String role) { + this.role = role; + } + + @JsonProperty("content") + public String getContent() { + return content; + } + + public void setContent(final String content) { + this.content = content; + } + +} diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/Choice.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java similarity index 95% rename from src/main/java/de/extio/lmlib/client/oai/completion/Choice.java rename to src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java index 0683ac2..87e67ab 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/Choice.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java @@ -1,4 +1,4 @@ -package de.extio.lmlib.client.oai.completion; +package de.extio.lmlib.client.oai.completion.text; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/CompletionRequest.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java similarity index 96% rename from src/main/java/de/extio/lmlib/client/oai/completion/CompletionRequest.java rename to src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java index b9fd80d..01acb14 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/CompletionRequest.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java @@ -1,4 +1,4 @@ -package de.extio.lmlib.client.oai.completion; +package de.extio.lmlib.client.oai.completion.text; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/CompletionResponse.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java similarity index 95% rename from src/main/java/de/extio/lmlib/client/oai/completion/CompletionResponse.java rename to src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java index c8f2683..25d7010 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/CompletionResponse.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java @@ -1,9 +1,11 @@ -package de.extio.lmlib.client.oai.completion; +package de.extio.lmlib.client.oai.completion.text; import java.util.List; import com.fasterxml.jackson.annotation.JsonProperty; +import de.extio.lmlib.client.oai.completion.Usage; + final class CompletionResponse { private String id; diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java new file mode 100755 index 0000000..fe8b35e --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -0,0 +1,149 @@ +package de.extio.lmlib.client.oai.completion.text; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpMethod; +import org.springframework.stereotype.Component; + +import de.extio.lmlib.client.Completion; +import de.extio.lmlib.client.CompletionFinishReason; +import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.Conversation.Turn; +import de.extio.lmlib.client.oai.completion.AbstractCompletionClient; +import de.extio.lmlib.profile.ModelCategory; +import de.extio.lmlib.profile.ModelProfile; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; +import de.extio.lmlib.prompt.PromptStrategy; +import de.extio.lmlib.prompt.PromptStrategyFactory; +import reactor.util.retry.Retry; + +@Component +public class TextCompletionClient extends AbstractCompletionClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class); + + @Autowired + private PromptStrategyFactory promptStrategyFactory; + + @Override + public ModelProvider getModelProvider() { + return ModelProvider.OAI_TEXT_COMPLETION; + } + + @Override + protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) { + final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt()); + if (promptStrategy == null) { + throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt()); + } + + final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy); + + final var request = new CompletionRequest(); + request.setModel(this.modelNameSupplier.getModelName(modelProfile.url())); + request.setPrompt(prompt); + request.setMaxTokens(modelProfile.maxTokens()); + request.setTemperature(modelProfile.temperature()); + request.setTopP(modelProfile.topP()); + request.setStream(false); + + LOGGER.debug("Requesting completion at {}", modelProfile.url()); + final LocalDateTime start = LocalDateTime.now(); + final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build(); + final var response = webClient + .method(HttpMethod.POST) + .uri(uriBuilder -> uriBuilder.path("/v1/completions").build()) + .header("Content-Type", "application/json") + .bodyValue(request) + .retrieve() + .bodyToMono(CompletionResponse.class) + .retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage()))) + .block(); + + String content; + CompletionFinishReason finishReason; + if (response.getChoices() != null && !response.getChoices().isEmpty()) { + final var choice = response.getChoices().getFirst(); + content = promptStrategy.removeEOT(choice.getText()); + finishReason = switch (choice.getFinishReason()) { + case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED; + case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED; + default -> CompletionFinishReason.DONE; + }; + } + else if (response.getContent() != null) { + content = promptStrategy.removeEOT(response.getContent()); + finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED; + } + else { + throw new IllegalStateException("No parseable response content from server"); + } + + final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent()); + return new Completion(content, finishReason, statistics); + } + + private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) { + StringBuilder prompt = this.formatConversation(conversation, promptStrategy); + + List tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile); + final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens + if (tokenized.size() > maxLength) { + LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength); + tokenized = tokenized.subList(0, maxLength); + prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile)); + } + + LOGGER.debug("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size()); + + return prompt.toString(); + } + + private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) { + final StringBuilder prompt = new StringBuilder(); + + final Deque turns = new ArrayDeque<>(conversation.getConversation()); + Turn systemTurn = null; + Turn userTurn = null; + Turn assistantTurn = null; + boolean first = true; + + while (!turns.isEmpty()) { + final Turn turn = turns.pop(); + switch (turn.type()) { + case SYSTEM: + systemTurn = turn; + break; + case USER: + userTurn = turn; + break; + case ASSISTANT: + assistantTurn = turn; + break; + default: + throw new IllegalArgumentException("Unknown turn type: " + turn.type()); + } + + if (first && userTurn != null) { + prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text())); + userTurn = null; + first = false; + } + else if (!first && assistantTurn != null && userTurn != null) { + promptStrategy.next(prompt, assistantTurn.text(), userTurn.text()); + userTurn = null; + assistantTurn = null; + } + } + + return prompt; + } + +} diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfile.java b/src/main/java/de/extio/lmlib/profile/ModelProfile.java index 03d926c..75fd19d 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfile.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfile.java @@ -17,6 +17,6 @@ public record ModelProfile( BigDecimal costOutToken) { public static enum ModelProvider { - AZURE_AI, OAI_TEXT_COMPLETION; + AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION; } } diff --git a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java index c131458..bb324b8 100755 --- a/src/main/java/de/extio/lmlib/profile/ModelProfileService.java +++ b/src/main/java/de/extio/lmlib/profile/ModelProfileService.java @@ -105,7 +105,7 @@ public class ModelProfileService { } } - public List getModelProfileUrls(final ModelProvider modelProvider) { + public List getModelProfileUrls(final ModelProvider... modelProviders) { final var modelProfiles = Binder.get(this.environment) .bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class)) .orElse(Collections.emptyMap()); @@ -115,8 +115,10 @@ public class ModelProfileService { .map(model -> { try { final var resource = new ResourcePropertySource("classpath:" + model + ".properties"); - if (modelProvider.name().equals(resource.getProperty("modelProvider"))) { - return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null; + for (final ModelProvider modelProvider : modelProviders) { + if (modelProvider.name().equals(resource.getProperty("modelProvider"))) { + return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null; + } } return null; } diff --git a/src/test/java/de/extio/lmlib/client/oai/TextCompletionClientTest.java b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java similarity index 71% rename from src/test/java/de/extio/lmlib/client/oai/TextCompletionClientTest.java rename to src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java index 41bafa6..06cc82c 100755 --- a/src/test/java/de/extio/lmlib/client/oai/TextCompletionClientTest.java +++ b/src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java @@ -21,46 +21,62 @@ import org.springframework.context.annotation.PropertySource; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.Conversation.TurnType; -import de.extio.lmlib.client.oai.completion.TextCompletionClient; +import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient; +import de.extio.lmlib.client.oai.completion.text.TextCompletionClient; import de.extio.lmlib.grader.Grader; import de.extio.lmlib.profile.ModelCategory; -//@Disabled("This test requires a running Llama server") +@Disabled("This test requires a running Llama server") @SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootConfiguration @ComponentScan(basePackages = "de.extio.lmlib") @PropertySource("classpath:/application-test.properties") -public class TextCompletionClientTest { +public class CompletionClientTest { - private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClientTest.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CompletionClientTest.class); @Autowired - private TextCompletionClient client; + private TextCompletionClient textCompletionClient; + @Autowired + private ChatCompletionClient chatCompletionclient; + @Test void completion() throws Exception { - final var completion = this.client.completion( + final var completion = this.textCompletionClient.completion( null, "You are a helpful assistant", "Say the color is green"); LOGGER.info(completion.response()); - assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.client)); + assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.textCompletionClient)); } @Test - void chat() throws Exception { + void chatTextCompletion() throws Exception { final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color."); conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?")); conversation.addTurn(new Turn(TurnType.USER, "You choose!")); - final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation); + final var completion = this.textCompletionClient.conversation(ModelCategory.MEDIUM, conversation); LOGGER.info(completion.response()); - assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.client)); + assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient)); } - + + @Test + void chatCompletion() throws Exception { + final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color."); + conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?")); + conversation.addTurn(new Turn(TurnType.USER, "You choose!")); + final var completion = this.chatCompletionclient.conversation(ModelCategory.MEDIUM, conversation); + + LOGGER.info(completion.response()); + + assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient)); + } + @Disabled @Test void hugePrompt() throws Exception { @@ -71,7 +87,7 @@ public class TextCompletionClientTest { } sb.delete(sb.length() - 2, sb.length()); - final var completion = this.client.completion( + final var completion = this.textCompletionClient.completion( null, "You are counting machine", "How many numbers do you count?\n" + sb.toString()); @@ -119,7 +135,7 @@ public class TextCompletionClientTest { private Tasks createTasks() { return new Tasks(() -> { try { - final var completion = this.client.completion( + final var completion = this.textCompletionClient.completion( null, "You are a helpful assistant", "How do I calculate the annual profit margin?"); @@ -130,7 +146,7 @@ public class TextCompletionClientTest { } }, () -> { try { - final var completion = this.client.completion( + final var completion = this.textCompletionClient.completion( null, "You are a helpful assistant", "How do I calculate the operating profit ratio?"); @@ -141,7 +157,7 @@ public class TextCompletionClientTest { } }, () -> { try { - final var completion = this.client.completion( + final var completion = this.textCompletionClient.completion( null, "You are a helpful assistant", "How do I calculate the return of investment?"); @@ -152,7 +168,7 @@ public class TextCompletionClientTest { } }, () -> { try { - final var completion = this.client.completion( + final var completion = this.textCompletionClient.completion( null, "You are a helpful assistant", "How do I calculate the return on net worth?"); diff --git a/src/test/resources/local-completion.properties b/src/test/resources/local-completion.properties index 05cb96e..7b0f810 100755 --- a/src/test/resources/local-completion.properties +++ b/src/test/resources/local-completion.properties @@ -1,10 +1,10 @@ -prompts=gemma2 +prompts= tokenEncoding=cl100k_base maxTokens=2000 maxContextLength=8000 temperature=0.4 topP=1.0 -modelProvider=OAI_TEXT_COMPLETION +modelProvider=OAI_CHAT_COMPLETION modelName= url=http://localhost:5001 apiKey=