feat: Introduce OpenAiProviderDialect for flexible request shaping and remove reasoning properties from ModelProfile
- Added OpenAiProviderDialect to manage request features for OpenAI-compatible providers. - Refactored ChatCompletionClient and TextCompletionClient to utilize the new dialect for handling reasoning and usage settings. - Removed reasoningEffort and reasoningSummaryDetails from ModelProfile and related services. - Updated README to reflect changes in request shaping and dialect usage.
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
package de.extio.lmlib.client.oai;
|
||||
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
|
||||
public interface OpenAiProviderDialect {
|
||||
|
||||
default boolean sendUsage(final ModelProfile modelProfile) {
|
||||
return true;
|
||||
}
|
||||
|
||||
default boolean sendReasoning(final ModelProfile modelProfile) {
|
||||
return true;
|
||||
}
|
||||
|
||||
default boolean sendMaxCompletionTokens(final ModelProfile modelProfile) {
|
||||
return true;
|
||||
}
|
||||
|
||||
default String reasoningEffort(final ModelProfile modelProfile) {
|
||||
return "medium";
|
||||
}
|
||||
|
||||
default String reasoningSummaryDetails(final ModelProfile modelProfile) {
|
||||
return "concise";
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import de.extio.lmlib.client.Conversation;
|
||||
import de.extio.lmlib.client.ToolCallData;
|
||||
import de.extio.lmlib.client.oai.Model;
|
||||
import de.extio.lmlib.client.oai.ModelsResponse;
|
||||
import de.extio.lmlib.client.oai.OpenAiProviderDialect;
|
||||
import de.extio.lmlib.profile.ModelCategory;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
@@ -44,6 +45,9 @@ import de.extio.lmlib.token.TokenizerResolver;
|
||||
public abstract class AbstractCompletionClient implements Client, DisposableBean {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCompletionClient.class);
|
||||
|
||||
private static final OpenAiProviderDialect DEFAULT_OPENAI_PROVIDER_DIALECT = new OpenAiProviderDialect() {
|
||||
};
|
||||
|
||||
@Autowired
|
||||
@Qualifier("lmLibRestClientBuilder")
|
||||
@@ -54,6 +58,9 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
||||
|
||||
@Autowired
|
||||
protected ModelProfileService modelProfileService;
|
||||
|
||||
@Autowired(required = false)
|
||||
protected OpenAiProviderDialect openAiProviderDialect;
|
||||
|
||||
@Value("${client.collectStatistics:false}")
|
||||
protected boolean collectStatistics;
|
||||
@@ -137,6 +144,30 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean
|
||||
|
||||
protected abstract Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer, final ToolCallData toolCallData);
|
||||
|
||||
protected OpenAiProviderDialect getOpenAiProviderDialect() {
|
||||
return this.openAiProviderDialect == null ? DEFAULT_OPENAI_PROVIDER_DIALECT : this.openAiProviderDialect;
|
||||
}
|
||||
|
||||
protected boolean sendUsage(final ModelProfile modelProfile) {
|
||||
return this.getOpenAiProviderDialect().sendUsage(modelProfile);
|
||||
}
|
||||
|
||||
protected boolean sendReasoning(final ModelProfile modelProfile) {
|
||||
return this.getOpenAiProviderDialect().sendReasoning(modelProfile);
|
||||
}
|
||||
|
||||
protected boolean sendMaxCompletionTokens(final ModelProfile modelProfile) {
|
||||
return this.getOpenAiProviderDialect().sendMaxCompletionTokens(modelProfile);
|
||||
}
|
||||
|
||||
protected String reasoningEffort(final ModelProfile modelProfile) {
|
||||
return this.getOpenAiProviderDialect().reasoningEffort(modelProfile);
|
||||
}
|
||||
|
||||
protected String reasoningSummaryDetails(final ModelProfile modelProfile) {
|
||||
return this.getOpenAiProviderDialect().reasoningSummaryDetails(modelProfile);
|
||||
}
|
||||
|
||||
protected String getPrimaryModelName(final ModelProfile modelProfile) {
|
||||
return this.getModelNames(modelProfile, false).stream().findFirst().orElse("");
|
||||
}
|
||||
|
||||
@@ -48,45 +48,7 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
@Override
|
||||
protected Completion requestCompletion(final Conversation conversation, final ModelProfile modelProfile, final Consumer<Chunk> chunkConsumer, final ToolCallData toolCallData) {
|
||||
final var chat = createChats(conversation, modelProfile);
|
||||
|
||||
final var request = new ChatCompletionRequest();
|
||||
request.setModel(this.getPrimaryModelName(modelProfile));
|
||||
request.setMessages(chat);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(chunkConsumer != null);
|
||||
if (chunkConsumer != null) {
|
||||
this.configureStreamOptions(true, request::setStreamOptions);
|
||||
} else {
|
||||
request.setUsage(true);
|
||||
}
|
||||
if (modelProfile.reasoningEffort() != null && !modelProfile.reasoningEffort().isBlank()) {
|
||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
||||
try {
|
||||
request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(modelProfile.reasoningEffort()));
|
||||
}
|
||||
catch (final IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("Invalid reasoning effort in the model profile: " + modelProfile.reasoningEffort(), e);
|
||||
}
|
||||
}
|
||||
if (modelProfile.reasoningSummaryDetails() != null && !modelProfile.reasoningSummaryDetails().isBlank()) {
|
||||
if (request.getReasoning() == null) {
|
||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
||||
}
|
||||
try {
|
||||
request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(modelProfile.reasoningSummaryDetails()).name());
|
||||
}
|
||||
catch (final IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("Invalid reasoning summary details in the model profile: " + modelProfile.reasoningSummaryDetails(), e);
|
||||
}
|
||||
}
|
||||
if (toolCallData != null && toolCallData.hasTools()) {
|
||||
request.setTools(this.createTools(toolCallData.tools()));
|
||||
request.setToolChoice(this.createToolChoice(toolCallData));
|
||||
request.setParallelToolCalls(toolCallData.parallelToolCalls());
|
||||
}
|
||||
final var request = this.createRequest(chat, modelProfile, chunkConsumer != null, toolCallData);
|
||||
|
||||
String requestBody;
|
||||
try {
|
||||
@@ -203,6 +165,53 @@ public class ChatCompletionClient extends AbstractCompletionClient {
|
||||
return new Completion(content, reasoning, finishReason, statistics, toolCalls);
|
||||
}
|
||||
|
||||
ChatCompletionRequest createRequest(final List<ChatMessage> chat, final ModelProfile modelProfile, final boolean streaming, final ToolCallData toolCallData) {
|
||||
final var request = new ChatCompletionRequest();
|
||||
request.setModel(this.getPrimaryModelName(modelProfile));
|
||||
request.setMessages(chat);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
if (this.sendMaxCompletionTokens(modelProfile)) {
|
||||
request.setMaxCompletionTokens(modelProfile.maxTokens());
|
||||
}
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(streaming);
|
||||
if (streaming) {
|
||||
this.configureStreamOptions(true, request::setStreamOptions);
|
||||
}
|
||||
else if (this.sendUsage(modelProfile)) {
|
||||
request.setUsage(true);
|
||||
}
|
||||
final var reasoningEffort = this.reasoningEffort(modelProfile);
|
||||
if (this.sendReasoning(modelProfile) && reasoningEffort != null && !reasoningEffort.isBlank()) {
|
||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
||||
try {
|
||||
request.getReasoning().setEffort(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.valueOf(reasoningEffort));
|
||||
}
|
||||
catch (final IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("Invalid reasoning effort for the current OAI provider dialect: " + reasoningEffort, e);
|
||||
}
|
||||
}
|
||||
final var reasoningSummaryDetails = this.reasoningSummaryDetails(modelProfile);
|
||||
if (this.sendReasoning(modelProfile) && reasoningSummaryDetails != null && !reasoningSummaryDetails.isBlank()) {
|
||||
if (request.getReasoning() == null) {
|
||||
request.setReasoning(new ChatCompletionRequest.ChatCompletionsRequestReasoning());
|
||||
}
|
||||
try {
|
||||
request.getReasoning().setSummary(ChatCompletionRequest.ChatCompletionsRequestReasoningSummaryDetails.valueOf(reasoningSummaryDetails).name());
|
||||
}
|
||||
catch (final IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("Invalid reasoning summary details for the current OAI provider dialect: " + reasoningSummaryDetails, e);
|
||||
}
|
||||
}
|
||||
if (toolCallData != null && toolCallData.hasTools()) {
|
||||
request.setTools(this.createTools(toolCallData.tools()));
|
||||
request.setToolChoice(this.createToolChoice(toolCallData));
|
||||
request.setParallelToolCalls(toolCallData.parallelToolCalls());
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
private List<ChatCompletionRequest.ChatCompletionTool> createTools(final List<ToolDefinition> toolDefinitions) {
|
||||
return toolDefinitions.stream().map(toolDefinition -> {
|
||||
final var function = new ChatCompletionRequest.ChatCompletionFunction();
|
||||
|
||||
@@ -53,19 +53,7 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
}
|
||||
|
||||
final var prompt = this.createPrompt(conversation, modelProfile, promptStrategy);
|
||||
|
||||
final var request = new CompletionRequest();
|
||||
request.setModel(this.getPrimaryModelName(modelProfile));
|
||||
request.setPrompt(prompt);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(chunkConsumer != null);
|
||||
if (chunkConsumer != null) {
|
||||
this.configureStreamOptions(true, request::setStreamOptions);
|
||||
} else {
|
||||
request.setUsage(true);
|
||||
}
|
||||
final var request = this.createRequest(prompt, modelProfile, chunkConsumer != null);
|
||||
|
||||
String requestBody;
|
||||
try {
|
||||
@@ -196,6 +184,23 @@ public class TextCompletionClient extends AbstractCompletionClient {
|
||||
final var statistics = createCompletionStatistics(modelProfile, start, response.getUsage(), response.getTimings(), prompt, content, reasoning);
|
||||
return new Completion(content, reasoning, finishReason, statistics);
|
||||
}
|
||||
|
||||
CompletionRequest createRequest(final String prompt, final ModelProfile modelProfile, final boolean streaming) {
|
||||
final var request = new CompletionRequest();
|
||||
request.setModel(this.getPrimaryModelName(modelProfile));
|
||||
request.setPrompt(prompt);
|
||||
request.setMaxTokens(modelProfile.maxTokens());
|
||||
request.setTemperature(modelProfile.temperature());
|
||||
request.setTopP(modelProfile.topP());
|
||||
request.setStream(streaming);
|
||||
if (streaming) {
|
||||
this.configureStreamOptions(true, request::setStreamOptions);
|
||||
}
|
||||
else if (this.sendUsage(modelProfile)) {
|
||||
request.setUsage(true);
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
private String createPrompt(final Conversation conversation, final ModelProfile modelProfile, final PromptStrategy promptStrategy) {
|
||||
StringBuilder prompt = this.formatConversation(conversation, promptStrategy);
|
||||
|
||||
@@ -18,8 +18,6 @@ public record ModelProfile(
|
||||
BigDecimal costPerCachedInToken,
|
||||
BigDecimal costPerOutToken,
|
||||
BigDecimal costPerReasoningOutToken,
|
||||
String reasoningEffort,
|
||||
String reasoningSummaryDetails,
|
||||
String category) {
|
||||
|
||||
public ModelProfile {
|
||||
|
||||
@@ -90,12 +90,6 @@ public class ModelProfileService {
|
||||
.map(Object::toString)
|
||||
.map(value -> new BigDecimal(value).divide(new BigDecimal(1000000.0)))
|
||||
.orElse(BigDecimal.ZERO);
|
||||
final var reasoningEffort = Optional.ofNullable(resource.getProperty("reasoningEffort"))
|
||||
.map(Object::toString)
|
||||
.orElse(null);
|
||||
final var reasoningSummaryDetails = Optional.ofNullable(resource.getProperty("reasoningSummaryDetails"))
|
||||
.map(Object::toString)
|
||||
.orElse(null);
|
||||
final var category = Optional
|
||||
.ofNullable(modelCategory)
|
||||
.map(ModelCategory::getShortName)
|
||||
@@ -112,7 +106,7 @@ public class ModelProfileService {
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, reasoningEffort, reasoningSummaryDetails, category);
|
||||
return new ModelProfile(promptTemplate, tokenizer, tokenEncoding, maxTokens, maxContextLength, temperature, topP, modelProvider, modelNameCfg, url, apiKey, costPerInToken, costPerCachedInToken, costPerOutToken, costPerReasoningOutToken, category);
|
||||
}
|
||||
catch (final IOException e) {
|
||||
LOGGER.error("Error while reading model profile", e);
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package de.extio.lmlib.client.oai.completion.chat;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import de.extio.lmlib.client.oai.OpenAiProviderDialect;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
|
||||
class ChatCompletionClientRequestTest {
|
||||
|
||||
@Test
|
||||
void defaultsToAllRequestFieldsWhenNoDialectIsConfigured() {
|
||||
final var client = new TestChatCompletionClient();
|
||||
final var request = client.createRequest(
|
||||
List.of(new ChatMessage("user", "hello", null)),
|
||||
this.createProfile(),
|
||||
false,
|
||||
null);
|
||||
|
||||
assertTrue(request.getUsage());
|
||||
assertNotNull(request.getReasoning());
|
||||
assertEquals(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.medium, request.getReasoning().getEffort());
|
||||
assertEquals("concise", request.getReasoning().getSummary());
|
||||
assertEquals(2048, request.getMaxCompletionTokens());
|
||||
}
|
||||
|
||||
@Test
|
||||
void omitsDialectDisabledRequestFields() {
|
||||
final var client = new TestChatCompletionClient();
|
||||
client.setDialect(new OpenAiProviderDialect() {
|
||||
@Override
|
||||
public boolean sendUsage(final ModelProfile modelProfile) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean sendReasoning(final ModelProfile modelProfile) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean sendMaxCompletionTokens(final ModelProfile modelProfile) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
final var request = client.createRequest(
|
||||
List.of(new ChatMessage("user", "hello", null)),
|
||||
this.createProfile(),
|
||||
false,
|
||||
null);
|
||||
|
||||
assertNull(request.getUsage());
|
||||
assertNull(request.getReasoning());
|
||||
assertNull(request.getMaxCompletionTokens());
|
||||
assertEquals(2048, request.getMaxTokens());
|
||||
assertFalse(request.isStream());
|
||||
}
|
||||
|
||||
@Test
|
||||
void usesDialectProvidedReasoningDefaults() {
|
||||
final var client = new TestChatCompletionClient();
|
||||
client.setDialect(new OpenAiProviderDialect() {
|
||||
@Override
|
||||
public String reasoningEffort(final ModelProfile modelProfile) {
|
||||
return "high";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String reasoningSummaryDetails(final ModelProfile modelProfile) {
|
||||
return "detailed";
|
||||
}
|
||||
});
|
||||
|
||||
final var request = client.createRequest(
|
||||
List.of(new ChatMessage("user", "hello", null)),
|
||||
this.createProfile(),
|
||||
false,
|
||||
null);
|
||||
|
||||
assertNotNull(request.getReasoning());
|
||||
assertEquals(ChatCompletionRequest.ChatCompletionsRequestReasoningEffort.high, request.getReasoning().getEffort());
|
||||
assertEquals("detailed", request.getReasoning().getSummary());
|
||||
}
|
||||
|
||||
private ModelProfile createProfile() {
|
||||
return new ModelProfile(
|
||||
"",
|
||||
"fallback",
|
||||
"cl100k_base",
|
||||
2048,
|
||||
8192,
|
||||
0.4,
|
||||
1.0,
|
||||
ModelProvider.OAI_CHAT_COMPLETION,
|
||||
"test-model",
|
||||
"http://localhost:5001",
|
||||
"",
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
"TEST");
|
||||
}
|
||||
|
||||
private static final class TestChatCompletionClient extends ChatCompletionClient {
|
||||
|
||||
private void setDialect(final OpenAiProviderDialect openAiProviderDialect) {
|
||||
this.openAiProviderDialect = openAiProviderDialect;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package de.extio.lmlib.client.oai.completion.text;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import de.extio.lmlib.client.oai.OpenAiProviderDialect;
|
||||
import de.extio.lmlib.profile.ModelProfile;
|
||||
import de.extio.lmlib.profile.ModelProfile.ModelProvider;
|
||||
|
||||
class TextCompletionClientRequestTest {
|
||||
|
||||
@Test
|
||||
void defaultsToUsageWhenNoDialectIsConfigured() {
|
||||
final var client = new TestTextCompletionClient();
|
||||
final var request = client.createRequest("prompt", this.createProfile(), false);
|
||||
|
||||
assertTrue(request.getUsage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void omitsUsageWhenDialectDisablesIt() {
|
||||
final var client = new TestTextCompletionClient();
|
||||
client.setDialect(new OpenAiProviderDialect() {
|
||||
@Override
|
||||
public boolean sendUsage(final ModelProfile modelProfile) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
final var request = client.createRequest("prompt", this.createProfile(), false);
|
||||
|
||||
assertNull(request.getUsage());
|
||||
}
|
||||
|
||||
private ModelProfile createProfile() {
|
||||
return new ModelProfile(
|
||||
"gpt-oss",
|
||||
"fallback",
|
||||
"cl100k_base",
|
||||
2048,
|
||||
8192,
|
||||
0.4,
|
||||
1.0,
|
||||
ModelProvider.OAI_TEXT_COMPLETION,
|
||||
"test-model",
|
||||
"http://localhost:5001",
|
||||
"",
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
"TEST");
|
||||
}
|
||||
|
||||
private static final class TestTextCompletionClient extends TextCompletionClient {
|
||||
|
||||
private void setDialect(final OpenAiProviderDialect openAiProviderDialect) {
|
||||
this.openAiProviderDialect = openAiProviderDialect;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,8 +55,6 @@ public class TokenizerResolverTest {
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
BigDecimal.ZERO,
|
||||
null,
|
||||
null,
|
||||
"TEST");
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,4 @@ apiKey=
|
||||
cost1MInTokens=0
|
||||
cost1MCachedInTokens=0
|
||||
cost1MOutTokens=0
|
||||
cost1MReasoningOutTokens=0
|
||||
reasoningEffort=medium
|
||||
reasoningSummaryDetails=auto
|
||||
cost1MReasoningOutTokens=0
|
||||
Reference in New Issue
Block a user