exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduced OpenAI chat completion client;

Browse Source
This commit is contained in:
extio-de
2025-06-27 23:39:17 +02:00
parent eee37a5648
commit 2350740136
19 changed files with 757 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -106,7 +106,7 @@ public final class AzureAiClient implements Client {
final var chat = this.createChats(conversation, modelProfile);
return this.requestCompletionsContinuations(chat, modelCategory, modelProfile);
return this.requestCompletions(chat, modelCategory, modelProfile);
}
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
@@ -180,7 +180,7 @@ public final class AzureAiClient implements Client {
return trimmed;
}
private Completion requestCompletionsContinuations(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
final LocalDateTime start = LocalDateTime.now();

2
src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java
View File

@@ -30,7 +30,7 @@ public class ModelNameSupplier implements InitializingBean {
@Override
public void afterPropertiesSet() throws Exception {
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION);
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION, ModelProvider.OAI_CHAT_COMPLETION);
for (final var url : urls) {
this.countDownLatches.put(url, new CountDownLatch(1));
this.fetchModelName(url);

158
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java Executable file
View File

@@ -0,0 +1,158 @@
package de.extio.lmlib.client.oai.completion;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.oai.ModelNameSupplier;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
public abstract class AbstractCompletionClient implements Client, DisposableBean {
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
@Autowired
@Qualifier("lmLibWebClientBuilder")
protected WebClient.Builder webClientBuilder;
@Autowired
protected ModelNameSupplier modelNameSupplier;
@Autowired
protected Tokenizer tokenizer;
@Autowired
protected ModelProfileService modelProfileService;
@Value("${client.collectStatistics:false}")
protected boolean collectStatistics;
protected final CompletionStatistics totalStatistics = new CompletionStatistics();
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.conversation(modelCategory, Conversation.create(system, text));
}
@Override
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
return this.requestCompletion(conversation, modelCategory, modelProfile);
}
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile);
protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) {
int inTokens = 0;
int outTokens = 0;
if (usage != null && usage.getPromptTokens() > 0) {
inTokens = usage.getPromptTokens();
outTokens = usage.getCompletionTokens();
}
else {
if (prompt != null) {
inTokens = this.tokenizer.count(prompt, modelProfile);
}
if (response != null) {
outTokens = this.tokenizer.count(response, modelProfile);
}
}
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
1,
Duration.between(start, LocalDateTime.now()),
inTokens,
outTokens,
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
false);
LOGGER.debug("{}", statistics);
if (this.collectStatistics) {
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
}
return statistics;
}
@Override
public void destroy() throws Exception {
if (this.collectStatistics && this.totalStatistics.requests > 0) {
LOGGER.info("Client Statistics: {}", this.totalStatistics);
}
}
static class CompletionStatistics {
private final Instant start = Instant.now();
private int requests;
private Duration duration = Duration.ofMillis(0l);
private long inTokens;
private long outTokens;
private BigDecimal cost = BigDecimal.ZERO;
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
this.cost = this.cost.add(cost);
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("CompletionStatistics [requests=");
builder.append(this.requests);
builder.append(", duration=");
builder.append(this.duration);
builder.append(", inTokens=");
builder.append(this.inTokens);
builder.append(", outTokens=");
builder.append(this.outTokens);
builder.append(", averageTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(", cost=");
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
builder.append("]");
return builder.toString();
}
}
}

276
src/main/java/de/extio/lmlib/client/oai/completion/TextCompletionClient.java
View File

@@ -1,276 +0,0 @@
package de.extio.lmlib.client.oai.completion;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.ModelNameSupplier;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import de.extio.lmlib.token.Tokenizer;
import reactor.util.retry.Retry;
@Component
public class TextCompletionClient implements Client, DisposableBean {
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
@Autowired
@Qualifier("lmLibWebClientBuilder")
private WebClient.Builder webClientBuilder;
@Autowired
private PromptStrategyFactory promptStrategyFactory;
@Autowired
private ModelNameSupplier modelNameSupplier;
@Autowired
private Tokenizer tokenizer;
@Autowired
private ModelProfileService modelProfileService;
@Value("${client.collectStatistics:false}")
private boolean collectStatistics;
private final CompletionStatistics totalStatistics = new CompletionStatistics();
@Override
public ModelProvider getModelProvider() {
return ModelProvider.OAI_TEXT_COMPLETION;
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.conversation(modelCategory, Conversation.create(system, text));
}
@Override
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
if (promptStrategy == null) {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
}
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
return this.requestCompletion(prompt, modelProfile, promptStrategy);
}
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokenized.size() > maxLength) {
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, maxLength);
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
}
LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
return prompt.toString();
}
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
final StringBuilder prompt = new StringBuilder();
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
Turn systemTurn = null;
Turn userTurn = null;
Turn assistantTurn = null;
boolean first = true;
while (!turns.isEmpty()) {
final Turn turn = turns.pop();
switch (turn.type()) {
case SYSTEM:
systemTurn = turn;
break;
case USER:
userTurn = turn;
break;
case ASSISTANT:
assistantTurn = turn;
break;
default:
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
}
if (first && userTurn != null) {
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
userTurn = null;
first = false;
}
else if (!first && assistantTurn != null && userTurn != null) {
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
userTurn = null;
assistantTurn = null;
}
}
return prompt;
}
private Completion requestCompletion(final String prompt, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
LOGGER.debug("Requesting completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
request.setPrompt(prompt);
request.setMaxTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
request.setStream(false);
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.retrieve()
.bodyToMono(CompletionResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
String content;
CompletionFinishReason finishReason;
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
final var choice = response.getChoices().getFirst();
content = promptStrategy.removeEOT(choice.getText());
finishReason = switch (choice.getFinishReason()) {
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
}
else if (response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
}
else {
throw new IllegalStateException("No parseable response content from server");
}
int inTokens;
int outTokens;
if (response.getUsage() != null && response.getUsage().getPromptTokens() > 0) {
inTokens = response.getUsage().getPromptTokens();
outTokens = response.getUsage().getCompletionTokens();
}
else {
inTokens = this.tokenizer.count(prompt, modelProfile);
outTokens = this.tokenizer.count(content, modelProfile);
}
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
1,
Duration.between(start, LocalDateTime.now()),
inTokens,
outTokens,
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
false);
LOGGER.debug("{}", statistics);
if (this.collectStatistics) {
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
}
return new Completion(content, finishReason, statistics);
}
@Override
public void destroy() throws Exception {
if (this.collectStatistics && this.totalStatistics.requests > 0) {
LOGGER.info("Client Statistics: {}", this.totalStatistics);
}
}
static class CompletionStatistics {
private final Instant start = Instant.now();
private int requests;
private Duration duration = Duration.ofMillis(0l);
private long inTokens;
private long outTokens;
private BigDecimal cost = BigDecimal.ZERO;
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
this.cost = this.cost.add(cost);
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("CompletionStatistics [requests=");
builder.append(this.requests);
builder.append(", duration=");
builder.append(this.duration);
builder.append(", inTokens=");
builder.append(this.inTokens);
builder.append(", outTokens=");
builder.append(this.outTokens);
builder.append(", averageTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(", cost=");
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
builder.append("]");
return builder.toString();
}
}
}

2
src/main/java/de/extio/lmlib/client/oai/completion/Usage.java
View File

@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty;
final class Usage {
public final class Usage {
private int completionTokens;

40
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java Executable file
View File

@@ -0,0 +1,40 @@
package de.extio.lmlib.client.oai.completion.chat;
import com.fasterxml.jackson.annotation.JsonProperty;
final class ChatChoice {
public static final String FINISH_REASON_STOP = "stop";
public static final String FINISH_REASON_LENGTH = "length";
public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
private ChatMessage message;
private String finishReason;
@JsonProperty("message")
public ChatMessage getMessage() {
return message;
}
public void setMessage(final ChatMessage message) {
this.message = message;
}
@JsonProperty("finish_reason")
public String getFinishReason() {
return this.finishReason;
}
public void setFinishReason(final String finishReason) {
this.finishReason = finishReason;
}
@Override
public String toString() {
return "ChatChoice [message=" + message + ", finishReason=" + finishReason + "]";
}
}

148
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java Executable file
View File

@@ -0,0 +1,148 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import reactor.util.retry.Retry;
@Component
public class ChatCompletionClient extends AbstractCompletionClient {
private static final Logger LOGGER = LoggerFactory.getLogger(ChatCompletionClient.class);
@Override
public ModelProvider getModelProvider() {
return ModelProvider.OAI_CHAT_COMPLETION;
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
final var chat = createChats(conversation, modelProfile);
final var request = new ChatCompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
request.setMessages(chat);
request.setMaxTokens(modelProfile.maxTokens());
request.setMaxCompletionTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
request.setStream(false);
LOGGER.debug("Requesting chat completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.retrieve()
.bodyToMono(ChatCompletionResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) {
throw new IllegalStateException("No parseable response content from server");
}
final var choice = response.getChoices().getFirst();
final var content = response.getChoices().getFirst().getMessage().getContent();
final var finishReason = switch (choice.getFinishReason()) {
case ChatChoice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case ChatChoice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content);
return new Completion(content, finishReason, statistics);
}
private List<ChatMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
final List<ChatMessage> chat = new ArrayList<>();
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
Turn systemTurn = null;
Turn userTurn = null;
Turn assistantTurn = null;
boolean first = true;
final AtomicInteger tokens = new AtomicInteger();
while (!turns.isEmpty()) {
final Turn turn = turns.pop();
switch (turn.type()) {
case SYSTEM:
systemTurn = turn;
break;
case USER:
userTurn = turn;
break;
case ASSISTANT:
assistantTurn = turn;
break;
default:
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
}
if (first && userTurn != null) {
if (systemTurn != null) {
chat.add(new ChatMessage("system", systemTurn.text()));
}
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
break;
}
userTurn = null;
first = false;
}
else if (!first && assistantTurn != null && userTurn != null) {
if (this.addChatRequestMessage("assistant", assistantTurn.text(), tokens, chat, modelProfile)) {
break;
}
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
break;
}
userTurn = null;
assistantTurn = null;
}
}
return chat;
}
private boolean addChatRequestMessage(final String role, final String text, final AtomicInteger tokens, final List<ChatMessage> chat, final ModelProfile modelProfile) {
var prompt = text;
var trimmed = false;
final var textTokens = this.tokenizer.count(prompt, modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokens.addAndGet(textTokens) > maxLength) {
var tokenized = this.tokenizer.tokenize(prompt, modelProfile);
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
trimmed = true;
}
chat.add(new ChatMessage(role, prompt));
return trimmed;
}
}

87
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java Executable file
View File

@@ -0,0 +1,87 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.util.List;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
final class ChatCompletionRequest {
private String model;
private List<ChatMessage> messages;
private Integer maxTokens;
private Integer maxCompletionTokens;
private Double temperature;
private Double topP;
private Boolean stream;
@JsonProperty("model")
public String getModel() {
return this.model;
}
public void setModel(final String model) {
this.model = model;
}
@JsonProperty("messages")
public List<ChatMessage> getMessages() {
return this.messages;
}
public void setMessages(final List<ChatMessage> messages) {
this.messages = messages;
}
@JsonProperty("max_tokens")
public Integer getMaxTokens() {
return this.maxTokens;
}
public void setMaxTokens(final Integer maxTokens) {
this.maxTokens = maxTokens;
}
@JsonProperty("max_completion_tokens")
public Integer getMaxCompletionTokens() {
return this.maxCompletionTokens;
}
public void setMaxCompletionTokens(final Integer maxCompletionTokens) {
this.maxCompletionTokens = maxCompletionTokens;
}
@JsonProperty("temperature")
public Double getTemperature() {
return this.temperature;
}
public void setTemperature(final Double temperature) {
this.temperature = temperature;
}
@JsonProperty("top_p")
public Double getTopP() {
return this.topP;
}
public void setTopP(final Double topP) {
this.topP = topP;
}
@JsonProperty("stream")
public Boolean isStream() {
return this.stream;
}
public void setStream(final Boolean stream) {
this.stream = stream;
}
}

82
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java Executable file
View File

@@ -0,0 +1,82 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.extio.lmlib.client.oai.completion.Usage;
final class ChatCompletionResponse {
private String id;
private String object;
private Integer created;
private List<ChatChoice> choices;
private Usage usage;
@JsonProperty("id")
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("created")
public Integer getCreated() {
return this.created;
}
public void setCreated(final Integer created) {
this.created = created;
}
@JsonProperty("choices")
public List<ChatChoice> getChoices() {
return this.choices;
}
public void setChoices(final List<ChatChoice> choices) {
this.choices = choices;
}
@JsonProperty("usage")
public Usage getUsage() {
return usage;
}
public void setUsage(final Usage usage) {
this.usage = usage;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("CompletionResponse [id=");
builder.append(this.id);
builder.append(", object=");
builder.append(this.object);
builder.append(", created=");
builder.append(this.created);
builder.append(", choices=");
builder.append(this.choices);
builder.append(", usage=");
builder.append(this.usage);
builder.append("]");
return builder.toString();
}
}

37
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java Normal file
View File

@@ -0,0 +1,37 @@
package de.extio.lmlib.client.oai.completion.chat;
import com.fasterxml.jackson.annotation.JsonProperty;
final class ChatMessage {
private String role;
private String content;
public ChatMessage() {
}
public ChatMessage(final String role, final String content) {
this.role = role;
this.content = content;
}
@JsonProperty("role")
public String getRole() {
return role;
}
public void setRole(final String role) {
this.role = role;
}
@JsonProperty("content")
public String getContent() {
return content;
}
public void setContent(final String content) {
this.content = content;
}
}

2
src/main/java/de/extio/lmlib/client/oai/completion/Choice.java → src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java
View File

@@ -1,4 +1,4 @@
package de.extio.lmlib.client.oai.completion;
package de.extio.lmlib.client.oai.completion.text;
import com.fasterxml.jackson.annotation.JsonProperty;

2
src/main/java/de/extio/lmlib/client/oai/completion/CompletionRequest.java → src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java
View File

@@ -1,4 +1,4 @@
package de.extio.lmlib.client.oai.completion;
package de.extio.lmlib.client.oai.completion.text;
import com.fasterxml.jackson.annotation.JsonProperty;

4
src/main/java/de/extio/lmlib/client/oai/completion/CompletionResponse.java → src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java
View File

@@ -1,9 +1,11 @@
package de.extio.lmlib.client.oai.completion;
package de.extio.lmlib.client.oai.completion.text;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.extio.lmlib.client.oai.completion.Usage;
final class CompletionResponse {
private String id;

149
src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java Executable file
View File

@@ -0,0 +1,149 @@
package de.extio.lmlib.client.oai.completion.text;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import reactor.util.retry.Retry;
@Component
public class TextCompletionClient extends AbstractCompletionClient {
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
@Autowired
private PromptStrategyFactory promptStrategyFactory;
@Override
public ModelProvider getModelProvider() {
return ModelProvider.OAI_TEXT_COMPLETION;
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
if (promptStrategy == null) {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
}
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
request.setPrompt(prompt);
request.setMaxTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
request.setStream(false);
LOGGER.debug("Requesting completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.retrieve()
.bodyToMono(CompletionResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
String content;
CompletionFinishReason finishReason;
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
final var choice = response.getChoices().getFirst();
content = promptStrategy.removeEOT(choice.getText());
finishReason = switch (choice.getFinishReason()) {
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
}
else if (response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
}
else {
throw new IllegalStateException("No parseable response content from server");
}
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent());
return new Completion(content, finishReason, statistics);
}
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokenized.size() > maxLength) {
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, maxLength);
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
}
LOGGER.debug("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
return prompt.toString();
}
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
final StringBuilder prompt = new StringBuilder();
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
Turn systemTurn = null;
Turn userTurn = null;
Turn assistantTurn = null;
boolean first = true;
while (!turns.isEmpty()) {
final Turn turn = turns.pop();
switch (turn.type()) {
case SYSTEM:
systemTurn = turn;
break;
case USER:
userTurn = turn;
break;
case ASSISTANT:
assistantTurn = turn;
break;
default:
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
}
if (first && userTurn != null) {
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
userTurn = null;
first = false;
}
else if (!first && assistantTurn != null && userTurn != null) {
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
userTurn = null;
assistantTurn = null;
}
}
return prompt;
}
}

2
src/main/java/de/extio/lmlib/profile/ModelProfile.java
View File

@@ -17,6 +17,6 @@ public record ModelProfile(
BigDecimal costOutToken) {
public static enum ModelProvider {
AZURE_AI, OAI_TEXT_COMPLETION;
AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION;
}
}

8
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -105,7 +105,7 @@ public class ModelProfileService {
}
}
public List<String> getModelProfileUrls(final ModelProvider modelProvider) {
public List<String> getModelProfileUrls(final ModelProvider... modelProviders) {
final var modelProfiles = Binder.get(this.environment)
.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
.orElse(Collections.emptyMap());
@@ -115,8 +115,10 @@ public class ModelProfileService {
.map(model -> {
try {
final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
for (final ModelProvider modelProvider : modelProviders) {
if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
}
}
return null;
}

48
src/test/java/de/extio/lmlib/client/oai/TextCompletionClientTest.java → src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java
View File

@@ -21,46 +21,62 @@ import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.client.oai.completion.TextCompletionClient;
import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient;
import de.extio.lmlib.client.oai.completion.text.TextCompletionClient;
import de.extio.lmlib.grader.Grader;
import de.extio.lmlib.profile.ModelCategory;
//@Disabled("This test requires a running Llama server")
@Disabled("This test requires a running Llama server")
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@PropertySource("classpath:/application-test.properties")
public class TextCompletionClientTest {
public class CompletionClientTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClientTest.class);
private static final Logger LOGGER = LoggerFactory.getLogger(CompletionClientTest.class);
@Autowired
private TextCompletionClient client;
private TextCompletionClient textCompletionClient;
@Autowired
private ChatCompletionClient chatCompletionclient;
@Test
void completion() throws Exception {
final var completion = this.client.completion(
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"Say the color is green");
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.client));
assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.textCompletionClient));
}
@Test
void chat() throws Exception {
void chatTextCompletion() throws Exception {
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation);
final var completion = this.textCompletionClient.conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.client));
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient));
}
@Test
void chatCompletion() throws Exception {
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.chatCompletionclient.conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient));
}
@Disabled
@Test
void hugePrompt() throws Exception {
@@ -71,7 +87,7 @@ public class TextCompletionClientTest {
}
sb.delete(sb.length() - 2, sb.length());
final var completion = this.client.completion(
final var completion = this.textCompletionClient.completion(
null,
"You are counting machine",
"How many numbers do you count?\n" + sb.toString());
@@ -119,7 +135,7 @@ public class TextCompletionClientTest {
private Tasks createTasks() {
return new Tasks(() -> {
try {
final var completion = this.client.completion(
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the annual profit margin?");
@@ -130,7 +146,7 @@ public class TextCompletionClientTest {
}
}, () -> {
try {
final var completion = this.client.completion(
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the operating profit ratio?");
@@ -141,7 +157,7 @@ public class TextCompletionClientTest {
}
}, () -> {
try {
final var completion = this.client.completion(
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the return of investment?");
@@ -152,7 +168,7 @@ public class TextCompletionClientTest {
}
}, () -> {
try {
final var completion = this.client.completion(
final var completion = this.textCompletionClient.completion(
null,
"You are a helpful assistant",
"How do I calculate the return on net worth?");

4
src/test/resources/local-completion.properties
View File

@@ -1,10 +1,10 @@
prompts=gemma2
prompts=
tokenEncoding=cl100k_base
maxTokens=2000
maxContextLength=8000
temperature=0.4
topP=1.0
modelProvider=OAI_TEXT_COMPLETION
modelProvider=OAI_CHAT_COMPLETION
modelName=
url=http://localhost:5001
apiKey=