From 04cde2558af2d69306e5de27e32f2d6597e07b92 Mon Sep 17 00:00:00 2001
From: "(no author)" <(no author)@71dcc7fd-b1f2-4e5d-bb8e-274146e81173>
Date: Tue, 15 Apr 2025 17:51:39 +0000
Subject: [PATCH] Merged lm-lib
git-svn-id: svn://home0/lm-lib@3737 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
---
pom.xml | 4 +-
.../de/extio/lmlib/client/ClientService.java | 41 +++++++++++++++
.../lmlib/client/CompletionInterceptor.java | 11 ++++
.../lmlib/prompt/Llama4PromptStrategy.java | 51 +++++++++++++++++++
4 files changed, 105 insertions(+), 2 deletions(-)
create mode 100644 src/main/java/de/extio/lmlib/client/CompletionInterceptor.java
create mode 100644 src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java
diff --git a/pom.xml b/pom.xml
index 1368031..5aa41e9 100755
--- a/pom.xml
+++ b/pom.xml
@@ -4,13 +4,13 @@
lm-lib
de.extio
lm-lib
- 1.1.0
+ 1.3.0
21
21
21
- 3.4.2
+ 3.4.4
1.0.0-beta.2
1.1.0
diff --git a/src/main/java/de/extio/lmlib/client/ClientService.java b/src/main/java/de/extio/lmlib/client/ClientService.java
index b8e15e9..9bf2c0f 100755
--- a/src/main/java/de/extio/lmlib/client/ClientService.java
+++ b/src/main/java/de/extio/lmlib/client/ClientService.java
@@ -9,6 +9,7 @@ import org.springframework.stereotype.Service;
import de.extio.lmlib.client.cached.CachedClient;
import de.extio.lmlib.client.cached.CachedClientRepository;
import de.extio.lmlib.profile.ModelCategory;
+import de.extio.lmlib.profile.ModelProfile.ModelProvider;
import de.extio.lmlib.profile.ModelProfileService;
@Service
@@ -23,6 +24,9 @@ public class ClientService {
@Autowired(required = false)
private CachedClientRepository cachedClientRepository;
+ @Autowired(required = false)
+ private List completionInterceptors;
+
@Cacheable("clients")
public Client getClient(final ModelCategory category) {
final var modelProfile = this.modelProfileService.getModelProfile(category.getModelProfile());
@@ -30,7 +34,44 @@ public class ClientService {
.filter(client -> client.getModelProvider() == modelProfile.modelProvider())
.findFirst()
.map(client -> this.cachedClientRepository == null ? client : new CachedClient(this.cachedClientRepository, this.modelProfileService, client))
+ .map(client -> this.completionInterceptors == null || this.completionInterceptors.isEmpty() ? client : new InterceptingClient(client, this.completionInterceptors))
.orElseThrow(() -> new IllegalStateException("No client found for model profile " + category));
}
+ private static class InterceptingClient implements Client {
+
+ private final Client client;
+
+ private final List completionInterceptors;
+
+ public InterceptingClient(final Client client, final List completionInterceptors) {
+ this.client = client;
+ this.completionInterceptors = completionInterceptors;
+ }
+
+ @Override
+ public Completion completion(final ModelCategory modelCategory, final String system, final String question, final String fullText) {
+ return this.client.completion(modelCategory, system, question, fullText);
+ }
+
+ @Override
+ public Completion conversation(final ModelCategory modelCategory, final Conversation conversation_) {
+ Conversation conversation = conversation_;
+ for (final var interceptor : this.completionInterceptors) {
+ conversation = interceptor.before(modelCategory, conversation);
+ }
+ var completion = this.client.conversation(modelCategory, conversation);
+ for (final var interceptor : this.completionInterceptors) {
+ completion = interceptor.after(modelCategory, completion);
+ }
+ return completion;
+ }
+
+ @Override
+ public ModelProvider getModelProvider() {
+ return this.client.getModelProvider();
+ }
+
+ }
+
}
diff --git a/src/main/java/de/extio/lmlib/client/CompletionInterceptor.java b/src/main/java/de/extio/lmlib/client/CompletionInterceptor.java
new file mode 100644
index 0000000..b0a8196
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/client/CompletionInterceptor.java
@@ -0,0 +1,11 @@
+package de.extio.lmlib.client;
+
+import de.extio.lmlib.profile.ModelCategory;
+
+public interface CompletionInterceptor {
+
+ Conversation before(ModelCategory modelCategory, Conversation conversation);
+
+ Completion after(ModelCategory modelCategory, Completion completion);
+
+}
diff --git a/src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java b/src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java
new file mode 100644
index 0000000..b8dd4d5
--- /dev/null
+++ b/src/main/java/de/extio/lmlib/prompt/Llama4PromptStrategy.java
@@ -0,0 +1,51 @@
+package de.extio.lmlib.prompt;
+
+import org.springframework.stereotype.Component;
+
+/**
+ * @see https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
+ */
+@Component
+public class Llama4PromptStrategy implements PromptStrategy {
+
+ @Override
+ public StringBuilder start(final String system, final String question, final String text) {
+ final StringBuilder prompt = new StringBuilder();
+ prompt.append("<|begin_of_text|>");
+ if (!system.isEmpty()) {
+ prompt.append("<|header_start|>system<|header_end|>\n\n");
+ prompt.append(system);
+ prompt.append("<|eot|>");
+ }
+ prompt.append("<|header_start|>user<|header_end|>\n\n");
+ prompt.append(question);
+ if (!question.isEmpty() && !text.isEmpty()) {
+ prompt.append("\n\n");
+ }
+ prompt.append(text);
+ prompt.append("<|eot|><|header_start|>assistant<|header_end|>\n");
+
+ return prompt;
+ }
+
+ @Override
+ public void continue_(final StringBuilder prompt, final String assistant) {
+ this.next(prompt, assistant, "Continue");
+ }
+
+ @Override
+ public void next(final StringBuilder prompt, final String assistant, final String user) {
+ prompt.append(assistant);
+ prompt.append("<|eot|><|header_start|>user<|header_end|>\n\n" + user + "<|eot|><|header_start|>assistant<|header_end|>\n");
+ }
+
+ @Override
+ public String removeEOT(final String prompt) {
+ return prompt.strip().replace("<|eot|>", "");
+ }
+
+ @Override
+ public String getPromptName() {
+ return "llama4";
+ }
+}