feat: Implement tokenizer resolver and fallback tokenizer; update model profiles and README

2026-04-03 18:40:53 +02:00
parent dda1efd1e3
commit b4d1a44d93
17 changed files with 419 additions and 28 deletions
--- a/src/test/resources/application-test.properties
+++ b/src/test/resources/application-test.properties
@@ -11,5 +11,3 @@ profile.model.chatcompletion=local-chatcompletion
 agent.threads=2

 client.collectStatistics=true
-
-tokenizer.strategy=llamaServer
--- a/src/test/resources/local-chatcompletion.properties
+++ b/src/test/resources/local-chatcompletion.properties
@@ -1,5 +1,6 @@
 category=CHAT
 prompts=
+tokenizer=llamaServer
 tokenEncoding=cl100k_base
 maxTokens=2500
 maxContextLength=16000
--- a/src/test/resources/local-completion.properties
+++ b/src/test/resources/local-completion.properties
@@ -1,5 +1,6 @@
 category=TEXT
 prompts=gpt-oss
+tokenizer=llamaServer
 tokenEncoding=cl100k_base
 maxTokens=2500
 maxContextLength=16000