exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduced OpenAI chat completion client;

Browse Source
This commit is contained in:
extio-de
2025-06-27 23:39:17 +02:00
parent eee37a5648
commit 2350740136
19 changed files with 757 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@@ -70,13 +70,17 @@ If you use jtokkit tokenizer:
<artifactId>jtokkit</artifactId> <artifactId>jtokkit</artifactId>
</dependency> </dependency>
It is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context). The library supports both chat completions and (legacy) text completions. Choose one of these options in your model profile:
Set the following property in your model profile: modelProvider=OAI_CHAT_COMPLETION
modelProvider=OAI_TEXT_COMPLETION
Chat completions are recommended for most use cases and easy to use, since the library only needs to send the actual conversation to the server, whereas for text completions, the library needs to create a raw prompt that includes the correct instruction formatting (e.g. special tokens). Text completions are still preferred if you need more control over the raw prompt or if you don't use an instruct model. For text completions, it is required to select the right prompt template. See package .lmlib.prompt for available prompt templates or implement your own PromptStrategy (must be a bean in the spring context).
To activate a prompt template, set the following property in your model profile:
prompts=llama3 prompts=llama3
Example profile: src/test/resources/llama3-local.properties Example profile: src/test/resources/xxx-local.properties
## Azure AI client ## Azure AI client

4
src/main/java/de/extio/lmlib/client/azureai/AzureAiClient.java
View File

@@ -106,7 +106,7 @@ public final class AzureAiClient implements Client {
final var chat = this.createChats(conversation, modelProfile); final var chat = this.createChats(conversation, modelProfile);
return this.requestCompletionsContinuations(chat, modelCategory, modelProfile); return this.requestCompletions(chat, modelCategory, modelProfile);
} }
private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) { private List<ChatRequestMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
@@ -180,7 +180,7 @@ public final class AzureAiClient implements Client {
return trimmed; return trimmed;
} }
private Completion requestCompletionsContinuations(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) { private Completion requestCompletions(final List<ChatRequestMessage> chat, final ModelCategory modelCategory, final ModelProfile modelProfile) {
LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName()); LOGGER.debug("Requesting ChatCompletion at {}", modelProfile.modelName());
final LocalDateTime start = LocalDateTime.now(); final LocalDateTime start = LocalDateTime.now();

2
src/main/java/de/extio/lmlib/client/oai/ModelNameSupplier.java
View File

@@ -30,7 +30,7 @@ public class ModelNameSupplier implements InitializingBean {
@Override @Override
public void afterPropertiesSet() throws Exception { public void afterPropertiesSet() throws Exception {
final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION); final var urls = this.modelProfileService.getModelProfileUrls(ModelProvider.OAI_TEXT_COMPLETION, ModelProvider.OAI_CHAT_COMPLETION);
for (final var url : urls) { for (final var url : urls) {
this.countDownLatches.put(url, new CountDownLatch(1)); this.countDownLatches.put(url, new CountDownLatch(1));
this.fetchModelName(url); this.fetchModelName(url);

158
src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java Executable file
View File

@@ -0,0 +1,158 @@
package de.extio.lmlib.client.oai.completion;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.oai.ModelNameSupplier;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.token.Tokenizer;
public abstract class AbstractCompletionClient implements Client, DisposableBean {
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
@Autowired
@Qualifier("lmLibWebClientBuilder")
protected WebClient.Builder webClientBuilder;
@Autowired
protected ModelNameSupplier modelNameSupplier;
@Autowired
protected Tokenizer tokenizer;
@Autowired
protected ModelProfileService modelProfileService;
@Value("${client.collectStatistics:false}")
protected boolean collectStatistics;
protected final CompletionStatistics totalStatistics = new CompletionStatistics();
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.conversation(modelCategory, Conversation.create(system, text));
}
@Override
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || (modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION && modelProfile.modelProvider() != ModelProvider.OAI_CHAT_COMPLETION)) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
return this.requestCompletion(conversation, modelCategory, modelProfile);
}
protected abstract Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile);
protected de.extio.lmlib.client.CompletionStatistics createCompletionStatistics(final ModelProfile modelProfile, final LocalDateTime start, final Usage usage, final String prompt, final String response) {
int inTokens = 0;
int outTokens = 0;
if (usage != null && usage.getPromptTokens() > 0) {
inTokens = usage.getPromptTokens();
outTokens = usage.getCompletionTokens();
}
else {
if (prompt != null) {
inTokens = this.tokenizer.count(prompt, modelProfile);
}
if (response != null) {
outTokens = this.tokenizer.count(response, modelProfile);
}
}
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
1,
Duration.between(start, LocalDateTime.now()),
inTokens,
outTokens,
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
false);
LOGGER.debug("{}", statistics);
if (this.collectStatistics) {
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
}
return statistics;
}
@Override
public void destroy() throws Exception {
if (this.collectStatistics && this.totalStatistics.requests > 0) {
LOGGER.info("Client Statistics: {}", this.totalStatistics);
}
}
static class CompletionStatistics {
private final Instant start = Instant.now();
private int requests;
private Duration duration = Duration.ofMillis(0l);
private long inTokens;
private long outTokens;
private BigDecimal cost = BigDecimal.ZERO;
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
this.cost = this.cost.add(cost);
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("CompletionStatistics [requests=");
builder.append(this.requests);
builder.append(", duration=");
builder.append(this.duration);
builder.append(", inTokens=");
builder.append(this.inTokens);
builder.append(", outTokens=");
builder.append(this.outTokens);
builder.append(", averageTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(", cost=");
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
builder.append("]");
return builder.toString();
}
}
}

276
src/main/java/de/extio/lmlib/client/oai/completion/TextCompletionClient.java
View File

@@ -1,276 +0,0 @@
package de.extio.lmlib.client.oai.completion;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.client.Client;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.ModelNameSupplier;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import de.extio.lmlib.token.Tokenizer;
import reactor.util.retry.Retry;
@Component
public class TextCompletionClient implements Client, DisposableBean {
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
@Autowired
@Qualifier("lmLibWebClientBuilder")
private WebClient.Builder webClientBuilder;
@Autowired
private PromptStrategyFactory promptStrategyFactory;
@Autowired
private ModelNameSupplier modelNameSupplier;
@Autowired
private Tokenizer tokenizer;
@Autowired
private ModelProfileService modelProfileService;
@Value("${client.collectStatistics:false}")
private boolean collectStatistics;
private final CompletionStatistics totalStatistics = new CompletionStatistics();
@Override
public ModelProvider getModelProvider() {
return ModelProvider.OAI_TEXT_COMPLETION;
}
@Override
public Completion completion(final ModelCategory modelCategory, final String system, final String text) {
return this.conversation(modelCategory, Conversation.create(system, text));
}
@Override
public Completion conversation(ModelCategory modelCategory, final Conversation conversation) {
if (modelCategory == null) {
modelCategory = ModelCategory.MEDIUM;
}
final var modelProfile = this.modelProfileService.getModelProfile(modelCategory.getModelProfile());
if (modelProfile == null || modelProfile.modelProvider() != ModelProvider.OAI_TEXT_COMPLETION) {
throw new IllegalArgumentException("ModelProfile " + modelCategory.getModelProfile());
}
final PromptStrategy promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
if (promptStrategy == null) {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
}
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
return this.requestCompletion(prompt, modelProfile, promptStrategy);
}
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokenized.size() > maxLength) {
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, maxLength);
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
}
LOGGER.info("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
return prompt.toString();
}
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
final StringBuilder prompt = new StringBuilder();
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
Turn systemTurn = null;
Turn userTurn = null;
Turn assistantTurn = null;
boolean first = true;
while (!turns.isEmpty()) {
final Turn turn = turns.pop();
switch (turn.type()) {
case SYSTEM:
systemTurn = turn;
break;
case USER:
userTurn = turn;
break;
case ASSISTANT:
assistantTurn = turn;
break;
default:
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
}
if (first && userTurn != null) {
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
userTurn = null;
first = false;
}
else if (!first && assistantTurn != null && userTurn != null) {
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
userTurn = null;
assistantTurn = null;
}
}
return prompt;
}
private Completion requestCompletion(final String prompt, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
LOGGER.debug("Requesting completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
request.setPrompt(prompt);
request.setMaxTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
request.setStream(false);
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.retrieve()
.bodyToMono(CompletionResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
String content;
CompletionFinishReason finishReason;
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
final var choice = response.getChoices().getFirst();
content = promptStrategy.removeEOT(choice.getText());
finishReason = switch (choice.getFinishReason()) {
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
}
else if (response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
}
else {
throw new IllegalStateException("No parseable response content from server");
}
int inTokens;
int outTokens;
if (response.getUsage() != null && response.getUsage().getPromptTokens() > 0) {
inTokens = response.getUsage().getPromptTokens();
outTokens = response.getUsage().getCompletionTokens();
}
else {
inTokens = this.tokenizer.count(prompt, modelProfile);
outTokens = this.tokenizer.count(content, modelProfile);
}
final var statistics = new de.extio.lmlib.client.CompletionStatistics(
1,
Duration.between(start, LocalDateTime.now()),
inTokens,
outTokens,
new BigDecimal(inTokens).multiply(modelProfile.costInToken()).add(new BigDecimal(outTokens).multiply(modelProfile.costOutToken())),
false);
LOGGER.debug("{}", statistics);
if (this.collectStatistics) {
this.totalStatistics.add(statistics.duration(), inTokens, outTokens, statistics.cost());
}
return new Completion(content, finishReason, statistics);
}
@Override
public void destroy() throws Exception {
if (this.collectStatistics && this.totalStatistics.requests > 0) {
LOGGER.info("Client Statistics: {}", this.totalStatistics);
}
}
static class CompletionStatistics {
private final Instant start = Instant.now();
private int requests;
private Duration duration = Duration.ofMillis(0l);
private long inTokens;
private long outTokens;
private BigDecimal cost = BigDecimal.ZERO;
synchronized void add(final Duration duration, final int inTokens, final int outTokens, final BigDecimal cost) {
this.requests++;
this.duration = this.duration.plus(duration);
this.inTokens += inTokens;
this.outTokens += outTokens;
this.cost = this.cost.add(cost);
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("CompletionStatistics [requests=");
builder.append(this.requests);
builder.append(", duration=");
builder.append(this.duration);
builder.append(", inTokens=");
builder.append(this.inTokens);
builder.append(", outTokens=");
builder.append(this.outTokens);
builder.append(", averageTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) this.duration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", averageOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) this.duration.toMillis() / 1000.0)));
builder.append(", effectiveDuration=");
final var effectiveDuration = Duration.between(this.start, Instant.now());
builder.append(effectiveDuration);
builder.append(", effectiveTps=");
builder.append(new DecimalFormat("0.##").format(((double) (this.outTokens + this.inTokens) / (double) this.requests) / ((double) effectiveDuration.toMillis() / 1000.0 / (double) this.requests)));
builder.append(", effectiveOutTps=");
builder.append(new DecimalFormat("0.##").format((double) this.outTokens / ((double) effectiveDuration.toMillis() / 1000.0)));
builder.append(", cost=");
builder.append(new DecimalFormat("0.###").format(cost.doubleValue()));
builder.append("]");
return builder.toString();
}
}
}

2
src/main/java/de/extio/lmlib/client/oai/completion/Usage.java
View File

@@ -2,7 +2,7 @@ package de.extio.lmlib.client.oai.completion;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
final class Usage { public final class Usage {
private int completionTokens; private int completionTokens;

40
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatChoice.java Executable file
View File

@@ -0,0 +1,40 @@
package de.extio.lmlib.client.oai.completion.chat;
import com.fasterxml.jackson.annotation.JsonProperty;
final class ChatChoice {
public static final String FINISH_REASON_STOP = "stop";
public static final String FINISH_REASON_LENGTH = "length";
public static final String FINISH_REASON_CONTENT_FILTER = "content_filter";
private ChatMessage message;
private String finishReason;
@JsonProperty("message")
public ChatMessage getMessage() {
return message;
}
public void setMessage(final ChatMessage message) {
this.message = message;
}
@JsonProperty("finish_reason")
public String getFinishReason() {
return this.finishReason;
}
public void setFinishReason(final String finishReason) {
this.finishReason = finishReason;
}
@Override
public String toString() {
return "ChatChoice [message=" + message + ", finishReason=" + finishReason + "]";
}
}

148
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java Executable file
View File

@@ -0,0 +1,148 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import reactor.util.retry.Retry;
@Component
public class ChatCompletionClient extends AbstractCompletionClient {
private static final Logger LOGGER = LoggerFactory.getLogger(ChatCompletionClient.class);
@Override
public ModelProvider getModelProvider() {
return ModelProvider.OAI_CHAT_COMPLETION;
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
final var chat = createChats(conversation, modelProfile);
final var request = new ChatCompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
request.setMessages(chat);
request.setMaxTokens(modelProfile.maxTokens());
request.setMaxCompletionTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
request.setStream(false);
LOGGER.debug("Requesting chat completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/chat/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.retrieve()
.bodyToMono(ChatCompletionResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
if (response.getChoices() == null || response.getChoices().isEmpty() || response.getChoices().getFirst().getMessage() == null) {
throw new IllegalStateException("No parseable response content from server");
}
final var choice = response.getChoices().getFirst();
final var content = response.getChoices().getFirst().getMessage().getContent();
final var finishReason = switch (choice.getFinishReason()) {
case ChatChoice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case ChatChoice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), null, content);
return new Completion(content, finishReason, statistics);
}
private List<ChatMessage> createChats(final Conversation conversation, final ModelProfile modelProfile) {
final List<ChatMessage> chat = new ArrayList<>();
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
Turn systemTurn = null;
Turn userTurn = null;
Turn assistantTurn = null;
boolean first = true;
final AtomicInteger tokens = new AtomicInteger();
while (!turns.isEmpty()) {
final Turn turn = turns.pop();
switch (turn.type()) {
case SYSTEM:
systemTurn = turn;
break;
case USER:
userTurn = turn;
break;
case ASSISTANT:
assistantTurn = turn;
break;
default:
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
}
if (first && userTurn != null) {
if (systemTurn != null) {
chat.add(new ChatMessage("system", systemTurn.text()));
}
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
break;
}
userTurn = null;
first = false;
}
else if (!first && assistantTurn != null && userTurn != null) {
if (this.addChatRequestMessage("assistant", assistantTurn.text(), tokens, chat, modelProfile)) {
break;
}
if (this.addChatRequestMessage("user", userTurn.text(), tokens, chat, modelProfile)) {
break;
}
userTurn = null;
assistantTurn = null;
}
}
return chat;
}
private boolean addChatRequestMessage(final String role, final String text, final AtomicInteger tokens, final List<ChatMessage> chat, final ModelProfile modelProfile) {
var prompt = text;
var trimmed = false;
final var textTokens = this.tokenizer.count(prompt, modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokens.addAndGet(textTokens) > maxLength) {
var tokenized = this.tokenizer.tokenize(prompt, modelProfile);
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, tokenized.size() - (tokens.get() - maxLength));
prompt = this.tokenizer.detokenize(tokenized, modelProfile);
trimmed = true;
}
chat.add(new ChatMessage(role, prompt));
return trimmed;
}
}

87
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionRequest.java Executable file
View File

@@ -0,0 +1,87 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.util.List;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
final class ChatCompletionRequest {
private String model;
private List<ChatMessage> messages;
private Integer maxTokens;
private Integer maxCompletionTokens;
private Double temperature;
private Double topP;
private Boolean stream;
@JsonProperty("model")
public String getModel() {
return this.model;
}
public void setModel(final String model) {
this.model = model;
}
@JsonProperty("messages")
public List<ChatMessage> getMessages() {
return this.messages;
}
public void setMessages(final List<ChatMessage> messages) {
this.messages = messages;
}
@JsonProperty("max_tokens")
public Integer getMaxTokens() {
return this.maxTokens;
}
public void setMaxTokens(final Integer maxTokens) {
this.maxTokens = maxTokens;
}
@JsonProperty("max_completion_tokens")
public Integer getMaxCompletionTokens() {
return this.maxCompletionTokens;
}
public void setMaxCompletionTokens(final Integer maxCompletionTokens) {
this.maxCompletionTokens = maxCompletionTokens;
}
@JsonProperty("temperature")
public Double getTemperature() {
return this.temperature;
}
public void setTemperature(final Double temperature) {
this.temperature = temperature;
}
@JsonProperty("top_p")
public Double getTopP() {
return this.topP;
}
public void setTopP(final Double topP) {
this.topP = topP;
}
@JsonProperty("stream")
public Boolean isStream() {
return this.stream;
}
public void setStream(final Boolean stream) {
this.stream = stream;
}
}

82
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionResponse.java Executable file
View File

@@ -0,0 +1,82 @@
package de.extio.lmlib.client.oai.completion.chat;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.extio.lmlib.client.oai.completion.Usage;
final class ChatCompletionResponse {
private String id;
private String object;
private Integer created;
private List<ChatChoice> choices;
private Usage usage;
@JsonProperty("id")
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
@JsonProperty("object")
public String getObject() {
return this.object;
}
public void setObject(final String object) {
this.object = object;
}
@JsonProperty("created")
public Integer getCreated() {
return this.created;
}
public void setCreated(final Integer created) {
this.created = created;
}
@JsonProperty("choices")
public List<ChatChoice> getChoices() {
return this.choices;
}
public void setChoices(final List<ChatChoice> choices) {
this.choices = choices;
}
@JsonProperty("usage")
public Usage getUsage() {
return usage;
}
public void setUsage(final Usage usage) {
this.usage = usage;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
builder.append("CompletionResponse [id=");
builder.append(this.id);
builder.append(", object=");
builder.append(this.object);
builder.append(", created=");
builder.append(this.created);
builder.append(", choices=");
builder.append(this.choices);
builder.append(", usage=");
builder.append(this.usage);
builder.append("]");
return builder.toString();
}
}

37
src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatMessage.java Normal file
View File

@@ -0,0 +1,37 @@
package de.extio.lmlib.client.oai.completion.chat;
import com.fasterxml.jackson.annotation.JsonProperty;
final class ChatMessage {
private String role;
private String content;
public ChatMessage() {
}
public ChatMessage(final String role, final String content) {
this.role = role;
this.content = content;
}
@JsonProperty("role")
public String getRole() {
return role;
}
public void setRole(final String role) {
this.role = role;
}
@JsonProperty("content")
public String getContent() {
return content;
}
public void setContent(final String content) {
this.content = content;
}
}

2
src/main/java/de/extio/lmlib/client/oai/completion/Choice.java → src/main/java/de/extio/lmlib/client/oai/completion/text/Choice.java
View File

@@ -1,4 +1,4 @@
package de.extio.lmlib.client.oai.completion; package de.extio.lmlib.client.oai.completion.text;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;

2
src/main/java/de/extio/lmlib/client/oai/completion/CompletionRequest.java → src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionRequest.java
View File

@@ -1,4 +1,4 @@
package de.extio.lmlib.client.oai.completion; package de.extio.lmlib.client.oai.completion.text;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;

4
src/main/java/de/extio/lmlib/client/oai/completion/CompletionResponse.java → src/main/java/de/extio/lmlib/client/oai/completion/text/CompletionResponse.java
View File

@@ -1,9 +1,11 @@
package de.extio.lmlib.client.oai.completion; package de.extio.lmlib.client.oai.completion.text;
import java.util.List; import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import de.extio.lmlib.client.oai.completion.Usage;
final class CompletionResponse { final class CompletionResponse {
private String id; private String id;

149
src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java Executable file
View File

@@ -0,0 +1,149 @@
package de.extio.lmlib.client.oai.completion.text;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import de.extio.lmlib.client.Completion;
import de.extio.lmlib.client.CompletionFinishReason;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.oai.completion.AbstractCompletionClient;
import de.extio.lmlib.profile.ModelCategory;
import de.extio.lmlib.profile.ModelProfile;
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.prompt.PromptStrategy;
import de.extio.lmlib.prompt.PromptStrategyFactory;
import reactor.util.retry.Retry;
@Component
public class TextCompletionClient extends AbstractCompletionClient {
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClient.class);
@Autowired
private PromptStrategyFactory promptStrategyFactory;
@Override
public ModelProvider getModelProvider() {
return ModelProvider.OAI_TEXT_COMPLETION;
}
@Override
protected Completion requestCompletion(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile) {
final var promptStrategy = this.promptStrategyFactory.getStrategy(modelProfile.prompt());
if (promptStrategy == null) {
throw new IllegalArgumentException("Prompt strategy not found: " + modelProfile.prompt());
}
final var prompt = this.createPrompt(conversation, modelCategory, modelProfile, promptStrategy);
final var request = new CompletionRequest();
request.setModel(this.modelNameSupplier.getModelName(modelProfile.url()));
request.setPrompt(prompt);
request.setMaxTokens(modelProfile.maxTokens());
request.setTemperature(modelProfile.temperature());
request.setTopP(modelProfile.topP());
request.setStream(false);
LOGGER.debug("Requesting completion at {}", modelProfile.url());
final LocalDateTime start = LocalDateTime.now();
final var webClient = this.webClientBuilder.baseUrl(modelProfile.url()).build();
final var response = webClient
.method(HttpMethod.POST)
.uri(uriBuilder -> uriBuilder.path("/v1/completions").build())
.header("Content-Type", "application/json")
.bodyValue(request)
.retrieve()
.bodyToMono(CompletionResponse.class)
.retryWhen(Retry.backoff(99, Duration.ofSeconds(1)).maxBackoff(Duration.ofSeconds(10)).jitter(0.25d).doAfterRetry(rs -> LOGGER.warn("Request failed " + rs.failure().getMessage())))
.block();
String content;
CompletionFinishReason finishReason;
if (response.getChoices() != null && !response.getChoices().isEmpty()) {
final var choice = response.getChoices().getFirst();
content = promptStrategy.removeEOT(choice.getText());
finishReason = switch (choice.getFinishReason()) {
case Choice.FINISH_REASON_CONTENT_FILTER -> CompletionFinishReason.CONTENT_FILTERED;
case Choice.FINISH_REASON_LENGTH -> CompletionFinishReason.TOKEN_LIMIT_REACHED;
default -> CompletionFinishReason.DONE;
};
}
else if (response.getContent() != null) {
content = promptStrategy.removeEOT(response.getContent());
finishReason = response.isStoppedEos() ? CompletionFinishReason.DONE : CompletionFinishReason.TOKEN_LIMIT_REACHED;
}
else {
throw new IllegalStateException("No parseable response content from server");
}
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), prompt, response.getContent());
return new Completion(content, finishReason, statistics);
}
private String createPrompt(final Conversation conversation, final ModelCategory modelCategory, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
List<Long> tokenized = this.tokenizer.tokenize(prompt.toString(), modelProfile);
final int maxLength = modelProfile.maxContextLength() - modelProfile.maxTokens() - 15; // 15 is a margin for EOT and other special tokens
if (tokenized.size() > maxLength) {
LOGGER.warn("Prompt too long: {} tokens / {} max. Prompt will be cut!", tokenized.size(), maxLength);
tokenized = tokenized.subList(0, maxLength);
prompt = new StringBuilder(this.tokenizer.detokenize(tokenized, modelProfile));
}
LOGGER.debug("Completion request for {}. Input tokens: {}", modelCategory, tokenized.size());
return prompt.toString();
}
private StringBuilder formatConversation(final Conversation conversation, final PromptStrategy promptStrategy) {
final StringBuilder prompt = new StringBuilder();
final Deque<Turn> turns = new ArrayDeque<>(conversation.getConversation());
Turn systemTurn = null;
Turn userTurn = null;
Turn assistantTurn = null;
boolean first = true;
while (!turns.isEmpty()) {
final Turn turn = turns.pop();
switch (turn.type()) {
case SYSTEM:
systemTurn = turn;
break;
case USER:
userTurn = turn;
break;
case ASSISTANT:
assistantTurn = turn;
break;
default:
throw new IllegalArgumentException("Unknown turn type: " + turn.type());
}
if (first && userTurn != null) {
prompt.append(promptStrategy.start(systemTurn != null ? systemTurn.text() : "", userTurn.text()));
userTurn = null;
first = false;
}
else if (!first && assistantTurn != null && userTurn != null) {
promptStrategy.next(prompt, assistantTurn.text(), userTurn.text());
userTurn = null;
assistantTurn = null;
}
}
return prompt;
}
}

2
src/main/java/de/extio/lmlib/profile/ModelProfile.java
View File

@@ -17,6 +17,6 @@ public record ModelProfile(
BigDecimal costOutToken) { BigDecimal costOutToken) {
public static enum ModelProvider { public static enum ModelProvider {
AZURE_AI, OAI_TEXT_COMPLETION; AZURE_AI, OAI_TEXT_COMPLETION, OAI_CHAT_COMPLETION;
} }
} }

4
src/main/java/de/extio/lmlib/profile/ModelProfileService.java
View File

@@ -105,7 +105,7 @@ public class ModelProfileService {
} }
} }
public List<String> getModelProfileUrls(final ModelProvider modelProvider) { public List<String> getModelProfileUrls(final ModelProvider... modelProviders) {
final var modelProfiles = Binder.get(this.environment) final var modelProfiles = Binder.get(this.environment)
.bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class)) .bind(MODEL_PROFILE_PREFIX, Bindable.mapOf(String.class, String.class))
.orElse(Collections.emptyMap()); .orElse(Collections.emptyMap());
@@ -115,9 +115,11 @@ public class ModelProfileService {
.map(model -> { .map(model -> {
try { try {
final var resource = new ResourcePropertySource("classpath:" + model + ".properties"); final var resource = new ResourcePropertySource("classpath:" + model + ".properties");
for (final ModelProvider modelProvider : modelProviders) {
if (modelProvider.name().equals(resource.getProperty("modelProvider"))) { if (modelProvider.name().equals(resource.getProperty("modelProvider"))) {
return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null; return resource.getProperty("url") != null ? resource.getProperty("url").toString() : null;
} }
}
return null; return null;
} }
catch (final IOException e) { catch (final IOException e) {

46
src/test/java/de/extio/lmlib/client/oai/TextCompletionClientTest.java → src/test/java/de/extio/lmlib/client/oai/CompletionClientTest.java
View File

@@ -21,44 +21,60 @@ import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn; import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType; import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.client.oai.completion.TextCompletionClient; import de.extio.lmlib.client.oai.completion.chat.ChatCompletionClient;
import de.extio.lmlib.client.oai.completion.text.TextCompletionClient;
import de.extio.lmlib.grader.Grader; import de.extio.lmlib.grader.Grader;
import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelCategory;
//@Disabled("This test requires a running Llama server") @Disabled("This test requires a running Llama server")
@SpringBootTest(webEnvironment = WebEnvironment.NONE) @SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration @SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib") @ComponentScan(basePackages = "de.extio.lmlib")
@PropertySource("classpath:/application-test.properties") @PropertySource("classpath:/application-test.properties")
public class TextCompletionClientTest { public class CompletionClientTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClientTest.class); private static final Logger LOGGER = LoggerFactory.getLogger(CompletionClientTest.class);
@Autowired @Autowired
private TextCompletionClient client; private TextCompletionClient textCompletionClient;
@Autowired
private ChatCompletionClient chatCompletionclient;
@Test @Test
void completion() throws Exception { void completion() throws Exception {
final var completion = this.client.completion( final var completion = this.textCompletionClient.completion(
null, null,
"You are a helpful assistant", "You are a helpful assistant",
"Say the color is green"); "Say the color is green");
LOGGER.info(completion.response()); LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.client)); assertTrue(Grader.assessScoreBinary("Does the text mention the color green?", completion.response(), this.textCompletionClient));
} }
@Test @Test
void chat() throws Exception { void chatTextCompletion() throws Exception {
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color."); final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?")); conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!")); conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation); final var completion = this.textCompletionClient.conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.response()); LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.client)); assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient));
}
@Test
void chatCompletion() throws Exception {
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.chatCompletionclient.conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.response());
assertTrue(Grader.assessScoreBinary("Does the text mention a color?", completion.response(), this.textCompletionClient));
} }
@Disabled @Disabled
@@ -71,7 +87,7 @@ public class TextCompletionClientTest {
} }
sb.delete(sb.length() - 2, sb.length()); sb.delete(sb.length() - 2, sb.length());
final var completion = this.client.completion( final var completion = this.textCompletionClient.completion(
null, null,
"You are counting machine", "You are counting machine",
"How many numbers do you count?\n" + sb.toString()); "How many numbers do you count?\n" + sb.toString());
@@ -119,7 +135,7 @@ public class TextCompletionClientTest {
private Tasks createTasks() { private Tasks createTasks() {
return new Tasks(() -> { return new Tasks(() -> {
try { try {
final var completion = this.client.completion( final var completion = this.textCompletionClient.completion(
null, null,
"You are a helpful assistant", "You are a helpful assistant",
"How do I calculate the annual profit margin?"); "How do I calculate the annual profit margin?");
@@ -130,7 +146,7 @@ public class TextCompletionClientTest {
} }
}, () -> { }, () -> {
try { try {
final var completion = this.client.completion( final var completion = this.textCompletionClient.completion(
null, null,
"You are a helpful assistant", "You are a helpful assistant",
"How do I calculate the operating profit ratio?"); "How do I calculate the operating profit ratio?");
@@ -141,7 +157,7 @@ public class TextCompletionClientTest {
} }
}, () -> { }, () -> {
try { try {
final var completion = this.client.completion( final var completion = this.textCompletionClient.completion(
null, null,
"You are a helpful assistant", "You are a helpful assistant",
"How do I calculate the return of investment?"); "How do I calculate the return of investment?");
@@ -152,7 +168,7 @@ public class TextCompletionClientTest {
} }
}, () -> { }, () -> {
try { try {
final var completion = this.client.completion( final var completion = this.textCompletionClient.completion(
null, null,
"You are a helpful assistant", "You are a helpful assistant",
"How do I calculate the return on net worth?"); "How do I calculate the return on net worth?");

4
src/test/resources/local-completion.properties
View File

@@ -1,10 +1,10 @@
prompts=gemma2 prompts=
tokenEncoding=cl100k_base tokenEncoding=cl100k_base
maxTokens=2000 maxTokens=2000
maxContextLength=8000 maxContextLength=8000
temperature=0.4 temperature=0.4
topP=1.0 topP=1.0
modelProvider=OAI_TEXT_COMPLETION modelProvider=OAI_CHAT_COMPLETION
modelName= modelName=
url=http://localhost:5001 url=http://localhost:5001
apiKey= apiKey=