Introduced OpenAI chat completion client;
This commit is contained in:
extio-de
@@ -106,7 +106,7 @@ public final class AzureAiClient implements Client {
|
||||
|
||||
final var chat = this.createChats(conversation, modelProfile);
|
||||
|
||||
return this.requestCompletionsContinuations(chat, modelCategory, modelProfile);
|
||||
return this.requestCompletions(chat, modelCategory, modelProfile);
|
||||
}
|
||||
|
||||
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
@@ -180,7 +180,7 @@ public final class AzureAiClient implements Client {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private Completion requestCompletionsContinuations(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ public class ModelNameSupplier implements InitializingBean {
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() throws Exception {
|
||||
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION);
|
||||
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION, ModelProvider.OAI_CHAT_COMPLETION);
|
||||
for (final var url : urls) {
|
||||
this.countDownLatches.put(url, new CountDownLatch(1));
|
||||
this.fetchModelName(url);
|
||||
|
||||
158
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
Executable file
158
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java
Executable file
@@ -0,0 +1,158 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.text.DecimalFormat;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.oai.ModelNameSupplier;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
|
||||
public abstract class AbstractCompletionClient implements Client, DisposableBean {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
|
||||
|
||||
@Autowired
|
||||
@Qualifier("lmLibWebClientBuilder")
|
||||
protected WebClient.Builder webClientBuilder;
|
||||
|
||||
@Autowired
|
||||
protected ModelNameSupplier modelNameSupplier;
|
||||
|
||||
@Autowired
|
||||
protected Tokenizer tokenizer;
|
||||
|
||||
@Autowired
|
||||
protected ModelProfileService modelProfileService;
|
||||
|
||||
@Value("${client.collectStatistics:false}")
|
||||
protected boolean collectStatistics;
|
||||
|
||||
protected final CompletionStatistics totalStatistics = new CompletionStatistics();
|
||||
|
||||
@Override
|
||||
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
|
||||
return this.conversation(modelCategory, Conversation.create(system, text));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
return this.requestCompletion(conversation, modelCategory, modelProfile);
|
||||
}
|
||||
|
||||
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile);
|
||||
|
||||
protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) {
|
||||
int inTokens = 0;
|
||||
int outTokens = 0;
|
||||
if (usage != null && usage.getPromptTokens() > 0) {
|
||||
inTokens = usage.getPromptTokens();
|
||||
outTokens = usage.getCompletionTokens();
|
||||
}
|
||||
else {
|
||||
if (prompt != null) {
|
||||
inTokens = this.tokenizer.count(prompt, modelProfile);
|
||||
}
|
||||
if (response != null) {
|
||||
outTokens = this.tokenizer.count(response, modelProfile);
|
||||
}
|
||||
}
|
||||
|
||||
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
|
||||
1,
|
||||
Duration.between(start, LocalDateTime.now()),
|
||||
inTokens,
|
||||
outTokens,
|
||||
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
if (this.collectStatistics) {
|
||||
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
|
||||
}
|
||||
return statistics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (this.collectStatistics && this.totalStatistics.requests > 0) {
|
||||
LOGGER.info("Client Statistics: {}", this.totalStatistics);
|
||||
}
|
||||
}
|
||||
|
||||
static class CompletionStatistics {
|
||||
|
||||
private final Instant start = Instant.now();
|
||||
|
||||
private int requests;
|
||||
|
||||
private Duration duration = Duration.ofMillis(0l);
|
||||
|
||||
private long inTokens;
|
||||
|
||||
private long outTokens;
|
||||
|
||||
private BigDecimal cost = BigDecimal.ZERO;
|
||||
|
||||
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
|
||||
this.requests++;
|
||||
this.duration = this.duration.plus(duration);
|
||||
this.inTokens += inTokens;
|
||||
this.outTokens += outTokens;
|
||||
this.cost = this.cost.add(cost);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("CompletionStatistics [requests=");
|
||||
builder.append(this.requests);
|
||||
builder.append(", duration=");
|
||||
builder.append(this.duration);
|
||||
builder.append(", inTokens=");
|
||||
builder.append(this.inTokens);
|
||||
builder.append(", outTokens=");
|
||||
builder.append(this.outTokens);
|
||||
builder.append(", averageTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", averageOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
|
||||
builder.append(", effectiveDuration=");
|
||||
final var effectiveDuration = Duration.between(this.start, Instant.now());
|
||||
builder.append(effectiveDuration);
|
||||
builder.append(", effectiveTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", effectiveOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
|
||||
builder.append(", cost=");
|
||||
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.text.DecimalFormat;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
import de.extio.lmlib.client.Client;
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.ModelNameSupplier;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.profile.ModelProfileService;
|
||||
import de.extio.lmlib.prompt.PromptStrategy;
|
||||
import de.extio.lmlib.prompt.PromptStrategyFactory;
|
||||
import de.extio.lmlib.token.Tokenizer;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class TextCompletionClient implements Client, DisposableBean {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
|
||||
|
||||
@Autowired
|
||||
@Qualifier("lmLibWebClientBuilder")
|
||||
private WebClient.Builder webClientBuilder;
|
||||
|
||||
@Autowired
|
||||
private PromptStrategyFactory promptStrategyFactory;
|
||||
|
||||
@Autowired
|
||||
private ModelNameSupplier modelNameSupplier;
|
||||
|
||||
@Autowired
|
||||
private Tokenizer tokenizer;
|
||||
|
||||
@Autowired
|
||||
private ModelProfileService modelProfileService;
|
||||
|
||||
@Value("${client.collectStatistics:false}")
|
||||
private boolean collectStatistics;
|
||||
|
||||
private final CompletionStatistics totalStatistics = new CompletionStatistics();
|
||||
|
||||
@Override
|
||||
public ModelProvider getModelProvider() {
|
||||
return ModelProvider.OAI_TEXT_COMPLETION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
|
||||
return this.conversation(modelCategory, Conversation.create(system, text));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
|
||||
if (modelCategory == null) {
|
||||
modelCategory = ModelCategory.MEDIUM;
|
||||
}
|
||||
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
|
||||
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION) {
|
||||
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
|
||||
}
|
||||
|
||||
final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
|
||||
if (promptStrategy == null) {
|
||||
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
|
||||
}
|
||||
|
||||
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
|
||||
|
||||
return this.requestCompletion(prompt, modelProfile, promptStrategy);
|
||||
}
|
||||
|
||||
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
|
||||
|
||||
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
|
||||
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
|
||||
if (tokenized.size() > maxLength) {
|
||||
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
|
||||
tokenized = tokenized.subList(0, maxLength);
|
||||
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
|
||||
}
|
||||
|
||||
LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
|
||||
|
||||
return prompt.toString();
|
||||
}
|
||||
|
||||
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
|
||||
final StringBuilder prompt = new StringBuilder();
|
||||
|
||||
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
|
||||
Turn systemTurn = null;
|
||||
Turn userTurn = null;
|
||||
Turn assistantTurn = null;
|
||||
boolean first = true;
|
||||
|
||||
while (!turns.isEmpty()) {
|
||||
final Turn turn = turns.pop();
|
||||
switch (turn.type()) {
|
||||
case SYSTEM:
|
||||
systemTurn = turn;
|
||||
break;
|
||||
case USER:
|
||||
userTurn = turn;
|
||||
break;
|
||||
case ASSISTANT:
|
||||
assistantTurn = turn;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
|
||||
}
|
||||
|
||||
if (first && userTurn != null) {
|
||||
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
|
||||
userTurn = null;
|
||||
first = false;
|
||||
}
|
||||
else if (!first && assistantTurn != null && userTurn != null) {
|
||||
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
|
||||
userTurn = null;
|
||||
assistantTurn = null;
|
||||
}
|
||||
}
|
||||
|
||||
return prompt;
|
||||
}
|
||||
|
||||
private Completion requestCompletion(final String prompt, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
LOGGER.debug("Requesting completion at {}", modelProfile.url());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
|
||||
final var request = new CompletionRequest();
|
||||
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
|
||||
request.setPrompt(prompt);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(false);
|
||||
|
||||
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
|
||||
final var response = webClient
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.retrieve()
|
||||
.bodyToMono(CompletionResponse.class)
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.block();
|
||||
|
||||
String content;
|
||||
CompletionFinishReason finishReason;
|
||||
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
|
||||
final var choice = response.getChoices().getFirst();
|
||||
content = promptStrategy.removeEOT(choice.getText());
|
||||
finishReason = switch (choice.getFinishReason()) {
|
||||
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
}
|
||||
else if (response.getContent() != null) {
|
||||
content = promptStrategy.removeEOT(response.getContent());
|
||||
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
|
||||
int inTokens;
|
||||
int outTokens;
|
||||
if (response.getUsage() != null && response.getUsage().getPromptTokens() > 0) {
|
||||
inTokens = response.getUsage().getPromptTokens();
|
||||
outTokens = response.getUsage().getCompletionTokens();
|
||||
}
|
||||
else {
|
||||
inTokens = this.tokenizer.count(prompt, modelProfile);
|
||||
outTokens = this.tokenizer.count(content, modelProfile);
|
||||
}
|
||||
|
||||
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
|
||||
1,
|
||||
Duration.between(start, LocalDateTime.now()),
|
||||
inTokens,
|
||||
outTokens,
|
||||
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
|
||||
false);
|
||||
LOGGER.debug("{}", statistics);
|
||||
if (this.collectStatistics) {
|
||||
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
|
||||
}
|
||||
|
||||
return new Completion(content, finishReason, statistics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (this.collectStatistics && this.totalStatistics.requests > 0) {
|
||||
LOGGER.info("Client Statistics: {}", this.totalStatistics);
|
||||
}
|
||||
}
|
||||
|
||||
static class CompletionStatistics {
|
||||
|
||||
private final Instant start = Instant.now();
|
||||
|
||||
private int requests;
|
||||
|
||||
private Duration duration = Duration.ofMillis(0l);
|
||||
|
||||
private long inTokens;
|
||||
|
||||
private long outTokens;
|
||||
|
||||
private BigDecimal cost = BigDecimal.ZERO;
|
||||
|
||||
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
|
||||
this.requests++;
|
||||
this.duration = this.duration.plus(duration);
|
||||
this.inTokens += inTokens;
|
||||
this.outTokens += outTokens;
|
||||
this.cost = this.cost.add(cost);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("CompletionStatistics [requests=");
|
||||
builder.append(this.requests);
|
||||
builder.append(", duration=");
|
||||
builder.append(this.duration);
|
||||
builder.append(", inTokens=");
|
||||
builder.append(this.inTokens);
|
||||
builder.append(", outTokens=");
|
||||
builder.append(this.outTokens);
|
||||
builder.append(", averageTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", averageOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
|
||||
builder.append(", effectiveDuration=");
|
||||
final var effectiveDuration = Duration.between(this.start, Instant.now());
|
||||
builder.append(effectiveDuration);
|
||||
builder.append(", effectiveTps=");
|
||||
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
|
||||
builder.append(", effectiveOutTps=");
|
||||
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
|
||||
builder.append(", cost=");
|
||||
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class Usage {
|
||||
public final class Usage {
|
||||
|
||||
private int completionTokens;
|
||||
|
||||
|
||||
40
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java
Executable file
40
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java
Executable file
@@ -0,0 +1,40 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class ChatChoice {
|
||||
|
||||
public static final String FINISH_REASON_STOP = "stop";
|
||||
|
||||
public static final String FINISH_REASON_LENGTH = "length";
|
||||
|
||||
public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
|
||||
|
||||
private ChatMessage message;
|
||||
|
||||
private String finishReason;
|
||||
|
||||
@JsonProperty("message")
|
||||
public ChatMessage getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(final ChatMessage message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
@JsonProperty("finish_reason")
|
||||
public String getFinishReason() {
|
||||
return this.finishReason;
|
||||
}
|
||||
|
||||
public void setFinishReason(final String finishReason) {
|
||||
this.finishReason = finishReason;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ChatChoice [message=" + message + ", finishReason=" + finishReason + "]";
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ChatCompletionClient.class);
|
||||
|
||||
@Override
|
||||
public ModelProvider getModelProvider() {
|
||||
return ModelProvider.OAI_CHAT_COMPLETION;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
final var chat = createChats(conversation, modelProfile);
|
||||
|
||||
final var request = new ChatCompletionRequest();
|
||||
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
|
||||
request.setMessages(chat);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(false);
|
||||
|
||||
LOGGER.debug("Requesting chat completion at {}", modelProfile.url());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
|
||||
final var response = webClient
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.retrieve()
|
||||
.bodyToMono(ChatCompletionResponse.class)
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.block();
|
||||
|
||||
if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
|
||||
final var choice = response.getChoices().getFirst();
|
||||
final var content = response.getChoices().getFirst().getMessage().getContent();
|
||||
final var finishReason = switch (choice.getFinishReason()) {
|
||||
case ChatChoice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case ChatChoice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content);
|
||||
return new Completion(content, finishReason, statistics);
|
||||
}
|
||||
|
||||
private List<ChatMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
|
||||
final List<ChatMessage> chat = new ArrayList<>();
|
||||
|
||||
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
|
||||
Turn systemTurn = null;
|
||||
Turn userTurn = null;
|
||||
Turn assistantTurn = null;
|
||||
boolean first = true;
|
||||
|
||||
final AtomicInteger tokens = new AtomicInteger();
|
||||
|
||||
while (!turns.isEmpty()) {
|
||||
final Turn turn = turns.pop();
|
||||
switch (turn.type()) {
|
||||
case SYSTEM:
|
||||
systemTurn = turn;
|
||||
break;
|
||||
case USER:
|
||||
userTurn = turn;
|
||||
break;
|
||||
case ASSISTANT:
|
||||
assistantTurn = turn;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
|
||||
}
|
||||
|
||||
if (first && userTurn != null) {
|
||||
if (systemTurn != null) {
|
||||
chat.add(new ChatMessage("system", systemTurn.text()));
|
||||
}
|
||||
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
|
||||
break;
|
||||
}
|
||||
userTurn = null;
|
||||
first = false;
|
||||
}
|
||||
else if (!first && assistantTurn != null && userTurn != null) {
|
||||
if (this.addChatRequestMessage("assistant", assistantTurn.text(), tokens, chat, modelProfile)) {
|
||||
break;
|
||||
}
|
||||
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
|
||||
break;
|
||||
}
|
||||
userTurn = null;
|
||||
assistantTurn = null;
|
||||
}
|
||||
}
|
||||
|
||||
return chat;
|
||||
}
|
||||
|
||||
private boolean addChatRequestMessage(final String role, final String text, final AtomicInteger tokens, final List<ChatMessage> chat, final ModelProfile modelProfile) {
|
||||
var prompt = text;
|
||||
var trimmed = false;
|
||||
|
||||
final var textTokens = this.tokenizer.count(prompt, modelProfile);
|
||||
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
|
||||
if (tokens.addAndGet(textTokens) > maxLength) {
|
||||
var tokenized = this.tokenizer.tokenize(prompt, modelProfile);
|
||||
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
|
||||
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
|
||||
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
|
||||
trimmed = true;
|
||||
}
|
||||
|
||||
chat.add(new ChatMessage(role, prompt));
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class ChatCompletionRequest {
|
||||
|
||||
private String model;
|
||||
|
||||
private List<ChatMessage> messages;
|
||||
|
||||
private Integer maxTokens;
|
||||
|
||||
private Integer maxCompletionTokens;
|
||||
|
||||
private Double temperature;
|
||||
|
||||
private Double topP;
|
||||
|
||||
private Boolean stream;
|
||||
|
||||
@JsonProperty("model")
|
||||
public String getModel() {
|
||||
return this.model;
|
||||
}
|
||||
|
||||
public void setModel(final String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
@JsonProperty("messages")
|
||||
public List<ChatMessage> getMessages() {
|
||||
return this.messages;
|
||||
}
|
||||
|
||||
public void setMessages(final List<ChatMessage> messages) {
|
||||
this.messages = messages;
|
||||
}
|
||||
|
||||
@JsonProperty("max_tokens")
|
||||
public Integer getMaxTokens() {
|
||||
return this.maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokens(final Integer maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("max_completion_tokens")
|
||||
public Integer getMaxCompletionTokens() {
|
||||
return this.maxCompletionTokens;
|
||||
}
|
||||
|
||||
public void setMaxCompletionTokens(final Integer maxCompletionTokens) {
|
||||
this.maxCompletionTokens = maxCompletionTokens;
|
||||
}
|
||||
|
||||
@JsonProperty("temperature")
|
||||
public Double getTemperature() {
|
||||
return this.temperature;
|
||||
}
|
||||
|
||||
public void setTemperature(final Double temperature) {
|
||||
this.temperature = temperature;
|
||||
}
|
||||
|
||||
@JsonProperty("top_p")
|
||||
public Double getTopP() {
|
||||
return this.topP;
|
||||
}
|
||||
|
||||
public void setTopP(final Double topP) {
|
||||
this.topP = topP;
|
||||
}
|
||||
|
||||
@JsonProperty("stream")
|
||||
public Boolean isStream() {
|
||||
return this.stream;
|
||||
}
|
||||
|
||||
public void setStream(final Boolean stream) {
|
||||
this.stream = stream;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import de.extio.lmlib.client.oai.completion.Usage;
|
||||
|
||||
final class ChatCompletionResponse {
|
||||
|
||||
private String id;
|
||||
|
||||
private String object;
|
||||
|
||||
private Integer created;
|
||||
|
||||
private List<ChatChoice> choices;
|
||||
|
||||
private Usage usage;
|
||||
|
||||
@JsonProperty("id")
|
||||
public String getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public void setId(final String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@JsonProperty("object")
|
||||
public String getObject() {
|
||||
return this.object;
|
||||
}
|
||||
|
||||
public void setObject(final String object) {
|
||||
this.object = object;
|
||||
}
|
||||
|
||||
@JsonProperty("created")
|
||||
public Integer getCreated() {
|
||||
return this.created;
|
||||
}
|
||||
|
||||
public void setCreated(final Integer created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
@JsonProperty("choices")
|
||||
public List<ChatChoice> getChoices() {
|
||||
return this.choices;
|
||||
}
|
||||
|
||||
public void setChoices(final List<ChatChoice> choices) {
|
||||
this.choices = choices;
|
||||
}
|
||||
|
||||
@JsonProperty("usage")
|
||||
public Usage getUsage() {
|
||||
return usage;
|
||||
}
|
||||
|
||||
public void setUsage(final Usage usage) {
|
||||
this.usage = usage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
builder.append("CompletionResponse [id=");
|
||||
builder.append(this.id);
|
||||
builder.append(", object=");
|
||||
builder.append(this.object);
|
||||
builder.append(", created=");
|
||||
builder.append(this.created);
|
||||
builder.append(", choices=");
|
||||
builder.append(this.choices);
|
||||
builder.append(", usage=");
|
||||
builder.append(this.usage);
|
||||
builder.append("]");
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
final class ChatMessage {
|
||||
|
||||
private String role;
|
||||
|
||||
private String content;
|
||||
|
||||
public ChatMessage() {
|
||||
}
|
||||
|
||||
public ChatMessage(final String role, final String content) {
|
||||
this.role = role;
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
@JsonProperty("role")
|
||||
public String getRole() {
|
||||
return role;
|
||||
}
|
||||
|
||||
public void setRole(final String role) {
|
||||
this.role = role;
|
||||
}
|
||||
|
||||
@JsonProperty("content")
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public void setContent(final String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package de.extio.lmlib.client.oai.completion;
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import de.extio.lmlib.client.oai.completion.Usage;
|
||||
|
||||
final class CompletionResponse {
|
||||
|
||||
private String id;
|
||||
@@ -0,0 +1,149 @@
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import de.extio.lmlib.client.Completion;
|
||||
import de.extio.lmlib.client.CompletionFinishReason;
|
||||
import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.Conversation.Turn;
|
||||
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
import de.extio.lmlib.prompt.PromptStrategy;
|
||||
import de.extio.lmlib.prompt.PromptStrategyFactory;
|
||||
import reactor.util.retry.Retry;
|
||||
|
||||
@Component
|
||||
public class TextCompletionClient extends AbstractCompletionClient {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
|
||||
|
||||
@Autowired
|
||||
private PromptStrategyFactory promptStrategyFactory;
|
||||
|
||||
@Override
|
||||
public ModelProvider getModelProvider() {
|
||||
return ModelProvider.OAI_TEXT_COMPLETION;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
|
||||
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
|
||||
if (promptStrategy == null) {
|
||||
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
|
||||
}
|
||||
|
||||
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
|
||||
|
||||
final var request = new CompletionRequest();
|
||||
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
|
||||
request.setPrompt(prompt);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(false);
|
||||
|
||||
LOGGER.debug("Requesting completion at {}", modelProfile.url());
|
||||
final LocalDateTime start = LocalDateTime.now();
|
||||
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
|
||||
final var response = webClient
|
||||
.method(HttpMethod.POST)
|
||||
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
|
||||
.header("Content-Type", "application/json")
|
||||
.bodyValue(request)
|
||||
.retrieve()
|
||||
.bodyToMono(CompletionResponse.class)
|
||||
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
|
||||
.block();
|
||||
|
||||
String content;
|
||||
CompletionFinishReason finishReason;
|
||||
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
|
||||
final var choice = response.getChoices().getFirst();
|
||||
content = promptStrategy.removeEOT(choice.getText());
|
||||
finishReason = switch (choice.getFinishReason()) {
|
||||
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
|
||||
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
default -> CompletionFinishReason.DONE;
|
||||
};
|
||||
}
|
||||
else if (response.getContent() != null) {
|
||||
content = promptStrategy.removeEOT(response.getContent());
|
||||
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException("No parseable response content from server");
|
||||
}
|
||||
|
||||
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent());
|
||||
return new Completion(content, finishReason, statistics);
|
||||
}
|
||||
|
||||
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
|
||||
|
||||
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
|
||||
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
|
||||
if (tokenized.size() > maxLength) {
|
||||
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
|
||||
tokenized = tokenized.subList(0, maxLength);
|
||||
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
|
||||
}
|
||||
|
||||
LOGGER.debug("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
|
||||
|
||||
return prompt.toString();
|
||||
}
|
||||
|
||||
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
|
||||
final StringBuilder prompt = new StringBuilder();
|
||||
|
||||
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
|
||||
Turn systemTurn = null;
|
||||
Turn userTurn = null;
|
||||
Turn assistantTurn = null;
|
||||
boolean first = true;
|
||||
|
||||
while (!turns.isEmpty()) {
|
||||
final Turn turn = turns.pop();
|
||||
switch (turn.type()) {
|
||||
case SYSTEM:
|
||||
systemTurn = turn;
|
||||
break;
|
||||
case USER:
|
||||
userTurn = turn;
|
||||
break;
|
||||
case ASSISTANT:
|
||||
assistantTurn = turn;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
|
||||
}
|
||||
|
||||
if (first && userTurn != null) {
|
||||
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
|
||||
userTurn = null;
|
||||
first = false;
|
||||
}
|
||||
else if (!first && assistantTurn != null && userTurn != null) {
|
||||
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
|
||||
userTurn = null;
|
||||
assistantTurn = null;
|
||||
}
|
||||
}
|
||||
|
||||
return prompt;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user