Introduced OpenAI chat completion client;
This commit is contained in:
extio-de
10
README.md
10
README.md
@@ -70,13 +70,17 @@ If you use jtokkit tokenizer:
|
||||
<artifactId>jtokkit</artifactId>
|
||||
</dependency>
|
||||
|
||||
It is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context).
|
||||
The library supports both chat completions and (legacy) text completions. Choose one of these options in your model profile:
|
||||
|
||||
Set the following property in your model profile:
|
||||
modelProvider=OAI_CHAT_COMPLETION
|
||||
modelProvider=OAI_TEXT_COMPLETION
|
||||
|
||||
Chat completions are recommended for most use cases and easy to use, since the library only needs to send the actual conversation to the server, whereas for text completions, the library needs to create a raw prompt that includes the correct instruction formatting (e.g. special tokens). Text completions are still preferred if you need more control over the raw prompt or if you don't use an instruct model. For text completions, it is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context).
|
||||
To activate a prompt template, set the following property in your model profile:
|
||||
|
||||
prompts=llama3
|
||||
|
||||
Example profile: src/test/resources/llama3-local.properties
|
||||
Example profile: src/test/resources/xxx-local.properties
|
||||
|
||||
## Azure AI client
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ public final class AzureAiClient implements Client {
|
||||
|
||||
final var chat = this.createChats(conversation, modelProfile);
|
||||
|
||||
return this.requestCompletionsContinuations(chat, modelCategory, modelProfile);
|
||||
return this.requestCompletions(chat, modelCategory, modelProfile);
|
||||
}
|
||||
|
||||
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
@@ -180,7 +180,7 @@ public final class AzureAiClient implements Client {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private Completion requestCompletionsContinuations(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ public class ModelNameSupplier implements InitializingBean {
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() throws Exception {
|
||||
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION);
|
||||
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION, ModelProvider.OAI_CHAT_COMPLETION);
|
||||
for (final var url : urls) {
|
||||
this.countDownLatches.put(url, new CountDownLatch(1));
|
||||
this.fetchModelName(url);
|
||||
|
||||
158
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
Executable file
158
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
Executable file
@@ -0,0 +1,158 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.text.DecimalFormat;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.oai.ModelNameSupplier;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
|
||||
public abstract class AbstractCompletionClient implements Client, DisposableBean {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
|
||||
|
||||
@Autowired
|
||||
@Qualifier("lmLibWebClientBuilder")
|
||||
protected WebClient.Builder webClientBuilder;
|
||||
|
||||
@Autowired
|
||||
protected ModelNameSupplier modelNameSupplier;
|
||||
|
||||
@Autowired
|
||||
protected Tokenizer tokenizer;
|
||||
|
||||
@Autowired
|
||||
protected ModelProfileService modelProfileService;
|
||||
|
||||
@Value("${client.collectStatistics:false}")
|
||||
protected boolean collectStatistics;
|
||||
|
||||
protected final CompletionStatistics totalStatistics = new CompletionStatistics();
|
||||
|
||||
@Override
|
||||
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
|
||||
return this.conversation(modelCategory, Conversation.create(system, text));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
return this.requestCompletion(conversation, modelCategory, modelProfile);
|
||||
}
|
||||
|
||||
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile);
|
||||
|
||||
protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) {
|
||||
int inTokens = 0;
|
||||
int outTokens = 0;
|
||||
if (usage != null && usage.getPromptTokens() > 0) {
|
||||
inTokens = usage.getPromptTokens();
|
||||
outTokens = usage.getCompletionTokens();
|
||||
}
|
||||
else {
|
||||
if (prompt != null) {
|
||||
inTokens = this.tokenizer.count(prompt, modelProfile);
|
||||
}
|
||||
if (response != null) {
|
||||
outTokens = this.tokenizer.count(response, modelProfile);
|
||||
}
|
||||
}
|
||||
|
||||
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
|
||||
1,
|
||||
Duration.between(start, LocalDateTime.now()),
|
||||
inTokens,
|
||||
outTokens,
|
||||
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
if (this.collectStatistics) {
|
||||
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
|
||||
}
|
||||
return statistics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (this.collectStatistics && this.totalStatistics.requests > 0) {
|
||||
LOGGER.info("Client Statistics: {}", this.totalStatistics);
|
||||
}
|
||||
}
|
||||
|
||||
static class CompletionStatistics {
|
||||
|
||||
private final Instant start = Instant.now();
|
||||
|
||||
private int requests;
|
||||
|
||||
private Duration duration = Duration.ofMillis(0l);
|
||||
|
||||
private long inTokens;
|
||||
|
||||
private long outTokens;
|
||||
|
||||
private BigDecimal cost = BigDecimal.ZERO;
|
||||
|
||||
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
|
||||
this.requests++;
|
||||
this.duration = this.duration.plus(duration);
|
||||
this.inTokens += inTokens;
|
||||
this.outTokens += outTokens;
|
||||
this.cost = this.cost.add(cost);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("CompletionStatistics [requests=");
|
||||
builder.append(this.requests);
|
||||
builder.append(", duration=");
|
||||
builder.append(this.duration);
|
||||
builder.append(", inTokens=");
|
||||
builder.append(this.inTokens);
|
||||
builder.append(", outTokens=");
|
||||
builder.append(this.outTokens);
|
||||
builder.append(", averageTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", averageOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
|
||||
builder.append(", effectiveDuration=");
|
||||
final var effectiveDuration = Duration.between(this.start, Instant.now());
|
||||
builder.append(effectiveDuration);
|
||||
builder.append(", effectiveTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", effectiveOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
|
||||
builder.append(", cost=");
|
||||
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.text.DecimalFormat;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.ModelNameSupplier;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.prompt.PromptStrategy;
|
||||
import de.extio.lmlib.prompt.PromptStrategyFactory;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class TextCompletionClient implements Client, DisposableBean {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
|
||||
|
||||
@Autowired
|
||||
@Qualifier("lmLibWebClientBuilder")
|
||||
private WebClient.Builder webClientBuilder;
|
||||
|
||||
@Autowired
|
||||
private PromptStrategyFactory promptStrategyFactory;
|
||||
|
||||
@Autowired
|
||||
private ModelNameSupplier modelNameSupplier;
|
||||
|
||||
@Autowired
|
||||
private Tokenizer tokenizer;
|
||||
|
||||
@Autowired
|
||||
private ModelProfileService modelProfileService;
|
||||
|
||||
@Value("${client.collectStatistics:false}")
|
||||
private boolean collectStatistics;
|
||||
|
||||
private final CompletionStatistics totalStatistics = new CompletionStatistics();
|
||||
|
||||
@Override
|
||||
public ModelProvider getModelProvider() {
|
||||
return ModelProvider.OAI_TEXT_COMPLETION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
|
||||
return this.conversation(modelCategory, Conversation.create(system, text));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
|
||||
if (promptStrategy == null) {
|
||||
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
|
||||
}
|
||||
|
||||
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
|
||||
|
||||
return this.requestCompletion(prompt, modelProfile, promptStrategy);
|
||||
}
|
||||
|
||||
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
|
||||
|
||||
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
|
||||
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
|
||||
if (tokenized.size() > maxLength) {
|
||||
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
|
||||
tokenized = tokenized.subList(0, maxLength);
|
||||
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
|
||||
}
|
||||
|
||||
LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
|
||||
|
||||
return prompt.toString();
|
||||
}
|
||||
|
||||
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
|
||||
final StringBuilder prompt = new StringBuilder();
|
||||
|
||||
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
|
||||
Turn systemTurn = null;
|
||||
Turn userTurn = null;
|
||||
Turn assistantTurn = null;
|
||||
boolean first = true;
|
||||
|
||||
while (!turns.isEmpty()) {
|
||||
final Turn turn = turns.pop();
|
||||
switch (turn.type()) {
|
||||
case SYSTEM:
|
||||
systemTurn = turn;
|
||||
break;
|
||||
case USER:
|
||||
userTurn = turn;
|
||||
break;
|
||||
case ASSISTANT:
|
||||
assistantTurn = turn;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
|
||||
}
|
||||
|
||||
if (first && userTurn != null) {
|
||||
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
|
||||
userTurn = null;
|
||||
first = false;
|
||||
}
|
||||
else if (!first && assistantTurn != null && userTurn != null) {
|
||||
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
|
||||
userTurn = null;
|
||||
assistantTurn = null;
|
||||
}
|
||||
}
|
||||
|
||||
return prompt;
|
||||
}
|
||||
|
||||
private Completion requestCompletion(final String prompt, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
LOGGER.debug("Requesting completion at {}", modelProfile.url());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var request = new CompletionRequest();
|
||||
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
|
||||
request.setPrompt(prompt);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(false);
|
||||
|
||||
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
|
||||
final var response = webClient
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.retrieve()
|
||||
.bodyToMono(CompletionResponse.class)
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.block();
|
||||
|
||||
String content;
|
||||
CompletionFinishReason finishReason;
|
||||
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
|
||||
final var choice = response.getChoices().getFirst();
|
||||
content = promptStrategy.removeEOT(choice.getText());
|
||||
finishReason = switch (choice.getFinishReason()) {
|
||||
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
}
|
||||
else if (response.getContent() != null) {
|
||||
content = promptStrategy.removeEOT(response.getContent());
|
||||
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
|
||||
int inTokens;
|
||||
int outTokens;
|
||||
if (response.getUsage() != null && response.getUsage().getPromptTokens() > 0) {
|
||||
inTokens = response.getUsage().getPromptTokens();
|
||||
outTokens = response.getUsage().getCompletionTokens();
|
||||
}
|
||||
else {
|
||||
inTokens = this.tokenizer.count(prompt, modelProfile);
|
||||
outTokens = this.tokenizer.count(content, modelProfile);
|
||||
}
|
||||
|
||||
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
|
||||
1,
|
||||
Duration.between(start, LocalDateTime.now()),
|
||||
inTokens,
|
||||
outTokens,
|
||||
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
if (this.collectStatistics) {
|
||||
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
|
||||
}
|
||||
|
||||
return new Completion(content, finishReason, statistics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (this.collectStatistics && this.totalStatistics.requests > 0) {
|
||||
LOGGER.info("Client Statistics: {}", this.totalStatistics);
|
||||
}
|
||||
}
|
||||
|
||||
static class CompletionStatistics {
|
||||
|
||||
private final Instant start = Instant.now();
|
||||
|
||||
private int requests;
|
||||
|
||||
private Duration duration = Duration.ofMillis(0l);
|
||||
|
||||
private long inTokens;
|
||||
|
||||
private long outTokens;
|
||||
|
||||
private BigDecimal cost = BigDecimal.ZERO;
|
||||
|
||||
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
|
||||
this.requests++;
|
||||
this.duration = this.duration.plus(duration);
|
||||
this.inTokens += inTokens;
|
||||
this.outTokens += outTokens;
|
||||
this.cost = this.cost.add(cost);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("CompletionStatistics [requests=");
|
||||
builder.append(this.requests);
|
||||
builder.append(", duration=");
|
||||
builder.append(this.duration);
|
||||
builder.append(", inTokens=");
|
||||
builder.append(this.inTokens);
|
||||
builder.append(", outTokens=");
|
||||
builder.append(this.outTokens);
|
||||
builder.append(", averageTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", averageOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
|
||||
builder.append(", effectiveDuration=");
|
||||
final var effectiveDuration = Duration.between(this.start, Instant.now());
|
||||
builder.append(effectiveDuration);
|
||||
builder.append(", effectiveTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", effectiveOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
|
||||
builder.append(", cost=");
|
||||
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class Usage {
|
||||
public final class Usage {
|
||||
|
||||
private int completionTokens;
|
||||
|
||||
|
||||
40
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java
Executable file
40
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java
Executable file
@@ -0,0 +1,40 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class ChatChoice {
|
||||
|
||||
public static final String FINISH_REASON_STOP = "stop";
|
||||
|
||||
public static final String FINISH_REASON_LENGTH = "length";
|
||||
|
||||
public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
|
||||
|
||||
private ChatMessage message;
|
||||
|
||||
private String finishReason;
|
||||
|
||||
@JsonProperty("message")
|
||||
public ChatMessage getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(final ChatMessage message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
@JsonProperty("finish_reason")
|
||||
public String getFinishReason() {
|
||||
return this.finishReason;
|
||||
}
|
||||
|
||||
public void setFinishReason(final String finishReason) {
|
||||
this.finishReason = finishReason;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ChatChoice [message=" + message + ", finishReason=" + finishReason + "]";
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ChatCompletionClient.class);
|
||||
|
||||
@Override
|
||||
public ModelProvider getModelProvider() {
|
||||
return ModelProvider.OAI_CHAT_COMPLETION;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
final var chat = createChats(conversation, modelProfile);
|
||||
|
||||
final var request = new ChatCompletionRequest();
|
||||
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
|
||||
request.setMessages(chat);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(false);
|
||||
|
||||
LOGGER.debug("Requesting chat completion at {}", modelProfile.url());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
|
||||
final var response = webClient
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.retrieve()
|
||||
.bodyToMono(ChatCompletionResponse.class)
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.block();
|
||||
|
||||
if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
|
||||
final var choice = response.getChoices().getFirst();
|
||||
final var content = response.getChoices().getFirst().getMessage().getContent();
|
||||
final var finishReason = switch (choice.getFinishReason()) {
|
||||
case ChatChoice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case ChatChoice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content);
|
||||
return new Completion(content, finishReason, statistics);
|
||||
}
|
||||
|
||||
private List<ChatMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
final List<ChatMessage> chat = new ArrayList<>();
|
||||
|
||||
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
|
||||
Turn systemTurn = null;
|
||||
Turn userTurn = null;
|
||||
Turn assistantTurn = null;
|
||||
boolean first = true;
|
||||
|
||||
final AtomicInteger tokens = new AtomicInteger();
|
||||
|
||||
while (!turns.isEmpty()) {
|
||||
final Turn turn = turns.pop();
|
||||
switch (turn.type()) {
|
||||
case SYSTEM:
|
||||
systemTurn = turn;
|
||||
break;
|
||||
case USER:
|
||||
userTurn = turn;
|
||||
break;
|
||||
case ASSISTANT:
|
||||
assistantTurn = turn;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
|
||||
}
|
||||
|
||||
if (first && userTurn != null) {
|
||||
if (systemTurn != null) {
|
||||
chat.add(new ChatMessage("system", systemTurn.text()));
|
||||
}
|
||||
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
|
||||
break;
|
||||
}
|
||||
userTurn = null;
|
||||
first = false;
|
||||
}
|
||||
else if (!first && assistantTurn != null && userTurn != null) {
|
||||
if (this.addChatRequestMessage("assistant", assistantTurn.text(), tokens, chat, modelProfile)) {
|
||||
break;
|
||||
}
|
||||
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
|
||||
break;
|
||||
}
|
||||
userTurn = null;
|
||||
assistantTurn = null;
|
||||
}
|
||||
}
|
||||
|
||||
return chat;
|
||||
}
|
||||
|
||||
private boolean addChatRequestMessage(final String role, final String text, final AtomicInteger tokens, final List<ChatMessage> chat, final ModelProfile modelProfile) {
|
||||
var prompt = text;
|
||||
var trimmed = false;
|
||||
|
||||
final var textTokens = this.tokenizer.count(prompt, modelProfile);
|
||||
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
|
||||
if (tokens.addAndGet(textTokens) > maxLength) {
|
||||
var tokenized = this.tokenizer.tokenize(prompt, modelProfile);
|
||||
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
|
||||
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
|
||||
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
|
||||
trimmed = true;
|
||||
}
|
||||
|
||||
chat.add(new ChatMessage(role, prompt));
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class ChatCompletionRequest {
|
||||
|
||||
private String model;
|
||||
|
||||
private List<ChatMessage> messages;
|
||||
|
||||
private Integer maxTokens;
|
||||
|
||||
private Integer maxCompletionTokens;
|
||||
|
||||
private Double temperature;
|
||||
|
||||
private Double topP;
|
||||
|
||||
private Boolean stream;
|
||||
|
||||
@JsonProperty("model")
|
||||
public String getModel() {
|
||||
return this.model;
|
||||
}
|
||||
|
||||
public void setModel(final String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
@JsonProperty("messages")
|
||||
public List<ChatMessage> getMessages() {
|
||||
return this.messages;
|
||||
}
|
||||
|
||||
public void setMessages(final List<ChatMessage> messages) {
|
||||
this.messages = messages;
|
||||
}
|
||||
|
||||
@JsonProperty("max_tokens")
|
||||
public Integer getMaxTokens() {
|
||||
return this.maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokens(final Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("max_completion_tokens")
|
||||
public Integer getMaxCompletionTokens() {
|
||||
return this.maxCompletionTokens;
|
||||
}
|
||||
|
||||
public void setMaxCompletionTokens(final Integer maxCompletionTokens) {
|
||||
this.maxCompletionTokens = maxCompletionTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("temperature")
|
||||
public Double getTemperature() {
|
||||
return this.temperature;
|
||||
}
|
||||
|
||||
public void setTemperature(final Double temperature) {
|
||||
this.temperature = temperature;
|
||||
}
|
||||
|
||||
@JsonProperty("top_p")
|
||||
public Double getTopP() {
|
||||
return this.topP;
|
||||
}
|
||||
|
||||
public void setTopP(final Double topP) {
|
||||
this.topP = topP;
|
||||
}
|
||||
|
||||
@JsonProperty("stream")
|
||||
public Boolean isStream() {
|
||||
return this.stream;
|
||||
}
|
||||
|
||||
public void setStream(final Boolean stream) {
|
||||
this.stream = stream;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import de.extio.lmlib.client.oai.completion.Usage;
|
||||
|
||||
final class ChatCompletionResponse {
|
||||
|
||||
private String id;
|
||||
|
||||
private String object;
|
||||
|
||||
private Integer created;
|
||||
|
||||
private List<ChatChoice> choices;
|
||||
|
||||
private Usage usage;
|
||||
|
||||
@JsonProperty("id")
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public void setId(final String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@JsonProperty("object")
|
||||
public String getObject() {
|
||||
return this.object;
|
||||
}
|
||||
|
||||
public void setObject(final String object) {
|
||||
this.object = object;
|
||||
}
|
||||
|
||||
@JsonProperty("created")
|
||||
public Integer getCreated() {
|
||||
return this.created;
|
||||
}
|
||||
|
||||
public void setCreated(final Integer created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
@JsonProperty("choices")
|
||||
public List<ChatChoice> getChoices() {
|
||||
return this.choices;
|
||||
}
|
||||
|
||||
public void setChoices(final List<ChatChoice> choices) {
|
||||
this.choices = choices;
|
||||
}
|
||||
|
||||
@JsonProperty("usage")
|
||||
public Usage getUsage() {
|
||||
return usage;
|
||||
}
|
||||
|
||||
public void setUsage(final Usage usage) {
|
||||
this.usage = usage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("CompletionResponse [id=");
|
||||
builder.append(this.id);
|
||||
builder.append(", object=");
|
||||
builder.append(this.object);
|
||||
builder.append(", created=");
|
||||
builder.append(this.created);
|
||||
builder.append(", choices=");
|
||||
builder.append(this.choices);
|
||||
builder.append(", usage=");
|
||||
builder.append(this.usage);
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class ChatMessage {
|
||||
|
||||
private String role;
|
||||
|
||||
private String content;
|
||||
|
||||
public ChatMessage() {
|
||||
}
|
||||
|
||||
public ChatMessage(final String role, final String content) {
|
||||
this.role = role;
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
@JsonProperty("role")
|
||||
public String getRole() {
|
||||
return role;
|
||||
}
|
||||
|
||||
public void setRole(final String role) {
|
||||
this.role = role;
|
||||
}
|
||||
|
||||
@JsonProperty("content")
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public void setContent(final String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import de.extio.lmlib.client.oai.completion.Usage;
|
||||
|
||||
final class CompletionResponse {
|
||||
|
||||
private String id;
|
||||
@@ -0,0 +1,149 @@
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.prompt.PromptStrategy;
|
||||
import de.extio.lmlib.prompt.PromptStrategyFactory;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class TextCompletionClient extends AbstractCompletionClient {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
|
||||
|
||||
@Autowired
|
||||
private PromptStrategyFactory promptStrategyFactory;
|
||||
|
||||
@Override
|
||||
public ModelProvider getModelProvider() {
|
||||
return ModelProvider.OAI_TEXT_COMPLETION;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
|
||||
if (promptStrategy == null) {
|
||||
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
|
||||
}
|
||||
|
||||
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
|
||||
|
||||
final var request = new CompletionRequest();
|
||||
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
|
||||
request.setPrompt(prompt);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(false);
|
||||
|
||||
LOGGER.debug("Requesting completion at {}", modelProfile.url());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
|
||||
final var response = webClient
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.retrieve()
|
||||
.bodyToMono(CompletionResponse.class)
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.block();
|
||||
|
||||
String content;
|
||||
CompletionFinishReason finishReason;
|
||||
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
|
||||
final var choice = response.getChoices().getFirst();
|
||||
content = promptStrategy.removeEOT(choice.getText());
|
||||
finishReason = switch (choice.getFinishReason()) {
|
||||
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
}
|
||||
else if (response.getContent() != null) {
|
||||
content = promptStrategy.removeEOT(response.getContent());
|
||||
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
|
||||
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent());
|
||||
return new Completion(content, finishReason, statistics);
|
||||
}
|
||||
|
||||
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
|
||||
|
||||
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
|
||||
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
|
||||
if (tokenized.size() > maxLength) {
|
||||
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
|
||||
tokenized = tokenized.subList(0, maxLength);
|
||||
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
|
||||
}
|
||||
|
||||
LOGGER.debug("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
|
||||
|
||||
return prompt.toString();
|
||||
}
|
||||
|
||||
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
|
||||
final StringBuilder prompt = new StringBuilder();
|
||||
|
||||
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
|
||||
Turn systemTurn = null;
|
||||
Turn userTurn = null;
|
||||
Turn assistantTurn = null;
|
||||
boolean first = true;
|
||||
|
||||
while (!turns.isEmpty()) {
|
||||
final Turn turn = turns.pop();
|
||||
switch (turn.type()) {
|
||||
case SYSTEM:
|
||||
systemTurn = turn;
|
||||
break;
|
||||
case USER:
|
||||
userTurn = turn;
|
||||
break;
|
||||
case ASSISTANT:
|
||||
assistantTurn = turn;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
|
||||
}
|
||||
|
||||
if (first && userTurn != null) {
|
||||
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
|
||||
userTurn = null;
|
||||
first = false;
|
||||
}
|
||||
else if (!first && assistantTurn != null && userTurn != null) {
|
||||
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
|
||||
userTurn = null;
|
||||
assistantTurn = null;
|
||||
}
|
||||
}
|
||||
|
||||
return prompt;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -17,6 +17,6 @@ public record ModelProfile(
|
||||
BigDecimal costOutToken) {
|
||||
|
||||
public static enum ModelProvider {
|
||||
AZURE_AI, OAI_TEXT_COMPLETION;
|
||||
AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ public class ModelProfileService {
|
||||
}
|
||||
}
|
||||
|
||||
public List<String> getModelProfileUrls(final ModelProvider modelProvider) {
|
||||
public List<String> getModelProfileUrls(final ModelProvider... modelProviders) {
|
||||
final var modelProfiles = Binder.get(this.environment)
|
||||
.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
|
||||
.orElse(Collections.emptyMap());
|
||||
@@ -115,8 +115,10 @@ public class ModelProfileService {
|
||||
.map(model -> {
|
||||
try {
|
||||
final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
|
||||
if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
|
||||
return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
|
||||
for (final ModelProvider modelProvider : modelProviders) {
|
||||
if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
|
||||
return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -21,44 +21,60 @@ import org.springframework.context.annotation.PropertySource;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.Conversation.TurnType;
|
||||
import de.extio.lmlib.client.oai.completion.TextCompletionClient;
|
||||
import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient;
|
||||
import de.extio.lmlib.client.oai.completion.text.TextCompletionClient;
|
||||
import de.extio.lmlib.grader.Grader;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
|
||||
//@Disabled("This test requires a running Llama server")
|
||||
@Disabled("This test requires a running Llama server")
|
||||
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
|
||||
@SpringBootConfiguration
|
||||
@ComponentScan(basePackages = "de.extio.lmlib")
|
||||
@PropertySource("classpath:/application-test.properties")
|
||||
public class TextCompletionClientTest {
|
||||
public class CompletionClientTest {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClientTest.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(CompletionClientTest.class);
|
||||
|
||||
@Autowired
|
||||
private TextCompletionClient client;
|
||||
private TextCompletionClient textCompletionClient;
|
||||
|
||||
@Autowired
|
||||
private ChatCompletionClient chatCompletionclient;
|
||||
|
||||
@Test
|
||||
void completion() throws Exception {
|
||||
final var completion = this.client.completion(
|
||||
final var completion = this.textCompletionClient.completion(
|
||||
null,
|
||||
"You are a helpful assistant",
|
||||
"Say the color is green");
|
||||
|
||||
LOGGER.info(completion.response());
|
||||
|
||||
assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.client));
|
||||
assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.textCompletionClient));
|
||||
}
|
||||
|
||||
@Test
|
||||
void chat() throws Exception {
|
||||
void chatTextCompletion() throws Exception {
|
||||
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
|
||||
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
|
||||
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
|
||||
final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation);
|
||||
final var completion = this.textCompletionClient.conversation(ModelCategory.MEDIUM, conversation);
|
||||
|
||||
LOGGER.info(completion.response());
|
||||
|
||||
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.client));
|
||||
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient));
|
||||
}
|
||||
|
||||
@Test
|
||||
void chatCompletion() throws Exception {
|
||||
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
|
||||
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
|
||||
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
|
||||
final var completion = this.chatCompletionclient.conversation(ModelCategory.MEDIUM, conversation);
|
||||
|
||||
LOGGER.info(completion.response());
|
||||
|
||||
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient));
|
||||
}
|
||||
|
||||
@Disabled
|
||||
@@ -71,7 +87,7 @@ public class TextCompletionClientTest {
|
||||
}
|
||||
sb.delete(sb.length() - 2, sb.length());
|
||||
|
||||
final var completion = this.client.completion(
|
||||
final var completion = this.textCompletionClient.completion(
|
||||
null,
|
||||
"You are counting machine",
|
||||
"How many numbers do you count?\n" + sb.toString());
|
||||
@@ -119,7 +135,7 @@ public class TextCompletionClientTest {
|
||||
private Tasks createTasks() {
|
||||
return new Tasks(() -> {
|
||||
try {
|
||||
final var completion = this.client.completion(
|
||||
final var completion = this.textCompletionClient.completion(
|
||||
null,
|
||||
"You are a helpful assistant",
|
||||
"How do I calculate the annual profit margin?");
|
||||
@@ -130,7 +146,7 @@ public class TextCompletionClientTest {
|
||||
}
|
||||
}, () -> {
|
||||
try {
|
||||
final var completion = this.client.completion(
|
||||
final var completion = this.textCompletionClient.completion(
|
||||
null,
|
||||
"You are a helpful assistant",
|
||||
"How do I calculate the operating profit ratio?");
|
||||
@@ -141,7 +157,7 @@ public class TextCompletionClientTest {
|
||||
}
|
||||
}, () -> {
|
||||
try {
|
||||
final var completion = this.client.completion(
|
||||
final var completion = this.textCompletionClient.completion(
|
||||
null,
|
||||
"You are a helpful assistant",
|
||||
"How do I calculate the return of investment?");
|
||||
@@ -152,7 +168,7 @@ public class TextCompletionClientTest {
|
||||
}
|
||||
}, () -> {
|
||||
try {
|
||||
final var completion = this.client.completion(
|
||||
final var completion = this.textCompletionClient.completion(
|
||||
null,
|
||||
"You are a helpful assistant",
|
||||
"How do I calculate the return on net worth?");
|
||||
@@ -1,10 +1,10 @@
|
||||
prompts=gemma2
|
||||
prompts=
|
||||
tokenEncoding=cl100k_base
|
||||
maxTokens=2000
|
||||
maxContextLength=8000
|
||||
temperature=0.4
|
||||
topP=1.0
|
||||
modelProvider=OAI_TEXT_COMPLETION
|
||||
modelProvider=OAI_CHAT_COMPLETION
|
||||
modelName=
|
||||
url=http://localhost:5001
|
||||
apiKey=
|
||||
|
||||
Reference in New Issue
Block a user