diff --git a/pom.xml b/pom.xml index 1368031..5aa41e9 100755 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,13 @@ lm-lib de.extio lm-lib - 1.1.0 + 1.3.0 21 21 21 - 3.4.2 + 3.4.4 1.0.0-beta.2 1.1.0 diff --git a/src/main/java/de/extio/lmlib/client/ClientService.java b/src/main/java/de/extio/lmlib/client/ClientService.java index b8e15e9..9bf2c0f 100755 --- a/src/main/java/de/extio/lmlib/client/ClientService.java +++ b/src/main/java/de/extio/lmlib/client/ClientService.java @@ -9,6 +9,7 @@ import org.springframework.stereotype.Service; import de.extio.lmlib.client.cached.CachedClient; import de.extio.lmlib.client.cached.CachedClientRepository; import de.extio.lmlib.profile.ModelCategory; +import de.extio.lmlib.profile.ModelProfile.ModelProvider; import de.extio.lmlib.profile.ModelProfileService; @Service @@ -23,6 +24,9 @@ public class ClientService { @Autowired(required = false) private CachedClientRepository cachedClientRepository; + @Autowired(required = false) + private List completionInterceptors; + @Cacheable("clients") public Client getClient(final ModelCategory category) { final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile()); @@ -30,7 +34,44 @@ public class ClientService { .filter(client -> client.getModelProvider() == modelProfile.modelProvider()) .findFirst() .map(client -> this.cachedClientRepository == null ? client : new CachedClient(this.cachedClientRepository, this.modelProfileService, client)) + .map(client -> this.completionInterceptors == null || this.completionInterceptors.isEmpty() ? client : new InterceptingClient(client, this.completionInterceptors)) .orElseThrow(() -> new IllegalStateException("No client found for model profile " + category)); } + private static class InterceptingClient implements Client { + + private final Client client; + + private final List completionInterceptors; + + public InterceptingClient(final Client client, final List completionInterceptors) { + this.client = client; + this.completionInterceptors = completionInterceptors; + } + + @Override + public Completion completion(final ModelCategory modelCategory, final String system, final String question, final String fullText) { + return this.client.completion(modelCategory, system, question, fullText); + } + + @Override + public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_) { + Conversation conversation = conversation_; + for (final var interceptor : this.completionInterceptors) { + conversation = interceptor.before(modelCategory, conversation); + } + var completion = this.client.conversation(modelCategory, conversation); + for (final var interceptor : this.completionInterceptors) { + completion = interceptor.after(modelCategory, completion); + } + return completion; + } + + @Override + public ModelProvider getModelProvider() { + return this.client.getModelProvider(); + } + + } + } diff --git a/src/main/java/de/extio/lmlib/client/CompletionInterceptor.java b/src/main/java/de/extio/lmlib/client/CompletionInterceptor.java new file mode 100644 index 0000000..b0a8196 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/CompletionInterceptor.java @@ -0,0 +1,11 @@ +package de.extio.lmlib.client; + +import de.extio.lmlib.profile.ModelCategory; + +public interface CompletionInterceptor { + + Conversation before(ModelCategory modelCategory, Conversation conversation); + + Completion after(ModelCategory modelCategory, Completion completion); + +} diff --git a/src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java b/src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java new file mode 100644 index 0000000..b8dd4d5 --- /dev/null +++ b/src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java @@ -0,0 +1,51 @@ +package de.extio.lmlib.prompt; + +import org.springframework.stereotype.Component; + +/** + * @see https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3 + */ +@Component +public class Llama4PromptStrategy implements PromptStrategy { + + @Override + public StringBuilder start(final String system, final String question, final String text) { + final StringBuilder prompt = new StringBuilder(); + prompt.append("<|begin_of_text|>"); + if (!system.isEmpty()) { + prompt.append("<|header_start|>system<|header_end|>\n\n"); + prompt.append(system); + prompt.append("<|eot|>"); + } + prompt.append("<|header_start|>user<|header_end|>\n\n"); + prompt.append(question); + if (!question.isEmpty() && !text.isEmpty()) { + prompt.append("\n\n"); + } + prompt.append(text); + prompt.append("<|eot|><|header_start|>assistant<|header_end|>\n"); + + return prompt; + } + + @Override + public void continue_(final StringBuilder prompt, final String assistant) { + this.next(prompt, assistant, "Continue"); + } + + @Override + public void next(final StringBuilder prompt, final String assistant, final String user) { + prompt.append(assistant); + prompt.append("<|eot|><|header_start|>user<|header_end|>\n\n" + user + "<|eot|><|header_start|>assistant<|header_end|>\n"); + } + + @Override + public String removeEOT(final String prompt) { + return prompt.strip().replace("<|eot|>", ""); + } + + @Override + public String getPromptName() { + return "llama4"; + } +}