exo/lm-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lm-lib first commit;

Browse Source 
git-svn-id: svn://home0/lm-lib@3603 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2024-12-16 20:58:04 +00:00
commit c1a283305d
49 changed files with 3159 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

228
src/test/java/de/extio/lmlib/agent/AgentTest.java Normal file
View File

@@ -0,0 +1,228 @@
package de.extio.lmlib.agent;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.ClientService;
import de.extio.lmlib.client.profile.ModelCategory;
import io.netty.util.internal.ThreadLocalRandom;
/**
* These tests have been verified with Llama 3.1 8B and Gemma2 27B models
*/
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@PropertySource("classpath:/application-test.properties")
public class AgentTest {
private static final Logger LOGGER = LoggerFactory.getLogger(AgentTest.class);
@Autowired
private AgentExecutorService agentExecutor;
@Autowired
private ClientService clientService;
@Test
void agenticFlow() throws IOException {
final var agents = Map.of(
"CodeSummarizer",
new Agent("CodeSummarizer",
AgentType.START_CONVERSATION,
ModelCategory.MEDIUM,
"Generate a summary of the following Java source code.",
"The source code is:\n{{code}}",
new TextAgentResponseHandler("summary"),
null,
null,
null,
context -> new AgentNext("FeatureIdentifier", null, null, null)),
"FeatureIdentifier",
new Agent("FeatureIdentifier",
AgentType.CONVERSATION,
ModelCategory.MEDIUM,
"",
"Generate a list of 5 distinct features. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"features\": [\"Feature 1\", \"Feature 2\", ... ] }",
new JsonAgentResponseHandler(),
null,
context -> context.getContext().get("features").forEach(feature -> LOGGER.info(feature.toString())),
null,
context -> new AgentNext("FeatureAnalyzer", null, null, null)),
"FeatureAnalyzer",
new Agent("FeatureAnalyzer",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
"Analyze a feature of the following Java source code and describe the functionality and purpose of this feature. Only describe the implementation, don't suggest any possible improvements.",
"""
The source code is:
{{code}}
The feature is: [[features]]""",
new TextAgentResponseHandler("featureDescription"),
null,
null,
contexts -> Agent.mergeContexts(List.of("features", "featureDescription"), contexts),
context -> new AgentNext("AnswerGenerator", null, null, null)),
"AnswerGenerator",
new Agent("AnswerGenerator",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
"Generate a clear and concise blog post for an announcement from the following code analysis.",
"""
Write a concise description of the purpose in the blog post first and then concise information for the following topics: {{topics}}
The code analysis:
{{summary}}
{{featureDescription}}""",
new TextAgentResponseHandler("blogPost"),
context -> {
context.getContext().put("topics", List.of(String.join(", ", context.getContext().get("features").stream().map(String::valueOf).toArray(size -> new String[size]))));
},
null,
null,
context -> new AgentNext(null, null, null, null)));
final var context = new AgentContext(agents);
context.getContext().put("code", List.of(Files.readString(Path.of("src/main/java/de/extio/lmlib/agent/Agent.java"))));
final var resultContexts = this.agentExecutor.walkGraph(agents.get("CodeSummarizer"), context);
final var resultContext = resultContexts.getFirst();
LOGGER.info(resultContext.getContext().get("blogPost").toString() + "\n\n" + resultContext.getGraph().toString() + "\n" + resultContext.getRequestStatistic().toString());
assertEquals(1, resultContexts.size());
assertEquals(5, resultContext.getContext().get("features").size());
assertEquals(1, resultContext.getContext().get("blogPost").size());
assertEquals(8, resultContext.getRequestStatistic().getRequests().get());
for (final var feature : resultContext.getContext().get("features")) {
assertTrue(this.assessScoreBinary("Does the following blog post mention the feature " + feature, resultContext.getContext().get("blogPost").getFirst().toString()));
}
}
@Test
void doubleBranch() {
final var agents = Map.of(
"FirstLevel",
new Agent("FirstLevel",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
"",
"Generate 5 random numbers from 1 to 10. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"numbers\": [\"Number 1\", \"Number 2\", ... ] }",
new JsonAgentResponseHandler(),
null,
null,
null,
context -> new AgentNext("SecondLevel", null, null, null)),
"SecondLevel",
new Agent("SecondLevel",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
"",
"Generate 5 random numbers from 1 to 10. The previous number was [[numbers]]. Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"random\": [\"Number 1\", \"Number 2\", ... ] }",
new JsonAgentResponseHandler(),
null,
null,
null,
context -> new AgentNext("Calculator", null, null, null)),
"Calculator",
new Agent("Calculator",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
"",
"Add these 2 numbers: {{numbers}} + [[random]] ; Provide no preamble and no explanation. Return the response in JSON format with the following field: { \"result\": \"result\" }",
new JsonAgentResponseHandler(),
null,
null,
null,
context -> new AgentNext("Grader", null, null, null)),
"Grader",
new Agent("Grader",
AgentType.COMPLETION,
ModelCategory.MEDIUM,
"You are a helpful assistant.",
"If you add {{numbers}} marbles to {{random}} marbles, do you get {{result}} marbles as the result? Answer with 'yes' or 'no'",
new TextAgentResponseHandler("grade"),
null,
null,
null,
context -> {
final boolean pass = context.getContext().get("grade").getFirst().toString().strip().toLowerCase().equals("yes");
if (!pass) {
try {
Integer.parseInt(context.getContext().get("numbers").getFirst().toString().strip());
Integer.parseInt(context.getContext().get("random").getFirst().toString().strip());
}
catch (final NumberFormatException exc) {
context.getContext().put("numbers", List.of(String.valueOf(ThreadLocalRandom.current().nextInt(10))));
context.getContext().put("random", List.of(String.valueOf(ThreadLocalRandom.current().nextInt(10))));
}
}
return new AgentNext(pass ? null : "Calculator", pass, null, null);
}));
final var context = new AgentContext(agents);
final var resultContexts = this.agentExecutor.walkGraph(agents.get("FirstLevel"), context);
assertEquals(25, resultContexts.size());
assertTrue(resultContexts.getFirst().getRequestStatistic().getRequests().get() >= 56);
for (final var resultContext : resultContexts) {
final int number0 = Integer.parseInt(resultContext.getContext().get("numbers").getFirst().toString().strip());
final int number1 = Integer.parseInt(resultContext.getContext().get("random").getFirst().toString().strip());
final int expected = number0 + number1;
assertEquals(expected, Integer.parseInt(resultContext.getContext().get("result").getFirst().toString().strip()));
LOGGER.info(number0 + " + " + number1 + " = " + expected + "; " + resultContext.getGraph().toString() + " " + resultContext.getRequestStatistic().toString());
}
}
private boolean assessScoreBinary(final String question, final String text) {
LOGGER.info(question);
final var client = this.clientService.getClient(ModelCategory.MEDIUM);
var score = 0;
while (Math.abs(score) < 3) {
final var completion = client.completion(
ModelCategory.MEDIUM,
"""
You are a grader assessing the truthfulness of a given text to a user question.
Please provide a binary response 'true' or 'false' for the following text.
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
Only provide the response as a single word and no preamble and no explanation.""",
"Here is the text:\n" + text,
"\nHere is the user question: " + question);
LOGGER.info(completion.response());
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
if (Math.abs(score) == 2) {
break;
}
}
final var result = score > 0;
LOGGER.info("->" + String.valueOf(result));
return result;
}
}

71
src/test/java/de/extio/lmlib/client/azureai/AzureAiClientTest.java Executable file
View File

@@ -0,0 +1,71 @@
package de.extio.lmlib.client.azureai;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.client.profile.ModelCategory;
@Disabled
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@PropertySource("classpath:/application-test.properties")
public class AzureAiClientTest {
private static final Logger LOGGER = LoggerFactory.getLogger(AzureAiClientTest.class);
@Autowired
private AzureAiClient client;
@Test
void completion() throws Exception {
final var completion = this.client.completion(
ModelCategory.MEDIUM,
"You are a clever story teller",
"Continue this story:",
"Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
LOGGER.info(completion.response());
}
@Test
void completionLarge() throws Exception {
final var completion = this.client.completion(
ModelCategory.LARGE,
"You are a clever story teller",
"Continue this story:",
"Once upon a time, there was a spaceship travelling to a black hole. The crew were cats!");
LOGGER.info(completion.response());
}
@Test
void chat() throws Exception {
final var conversation = Conversation.create("You are an assistant who stutters", "Hi there, I would like to know the climate in Berlin.");
conversation.addTurn(new Turn(TurnType.ASSISTANT,
"T-t-the climate in B-b-berlin, um, it's, uh, quite, c-c-cool, actually. B-b-berlin has a, um, temperate climate, with, uh, four distinct s-s-seasons. S-s-spring is, um, pretty mild, with, uh, temperatures ranging from, uh, 10 to 20 degrees Celsius. S-s-summer is, um, warm, with, uh, highs around 25 degrees Celsius. F-f-fall is, um, cool, with, uh, temperatures ranging from 5 to 15 degrees Celsius. And, um, w-w-winter is, uh, quite, c-c-cold, with, uh, temperatures ranging from -2 to 5 degrees Celsius."));
conversation.addTurn(new Turn(TurnType.USER, "Great! How is the weather in summer?"));
final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.response());
}
@Test
void longPrompt() throws Exception {
final var completion = this.client.completion(
null,
"You are a helpful assistant",
"",
"Count the numbers: 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9");
LOGGER.info(completion.response());
}
}

198
src/test/java/de/extio/lmlib/client/textcompletion/TextCompletionClientTest.java Executable file
View File

@@ -0,0 +1,198 @@
package de.extio.lmlib.client.textcompletion;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.LocalDateTime;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import de.extio.lmlib.client.Conversation;
import de.extio.lmlib.client.Conversation.Turn;
import de.extio.lmlib.client.Conversation.TurnType;
import de.extio.lmlib.client.profile.ModelCategory;
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@PropertySource("classpath:/application-test.properties")
public class TextCompletionClientTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TextCompletionClientTest.class);
@Autowired
private TextCompletionClient client;
@Test
void completion() throws Exception {
final var completion = this.client.completion(
null,
"You are a helpful assistant",
"",
"Say the color is green");
LOGGER.info(completion.response());
assertTrue(this.assessScoreBinary("Does the text mention the color green?", completion.response()));
}
@Test
void chat() throws Exception {
final var conversation = Conversation.create("You are a helpful assistant", "I would like to paint my canvas in a solid color.");
conversation.addTurn(new Turn(TurnType.ASSISTANT, "Which color do you want to see?"));
conversation.addTurn(new Turn(TurnType.USER, "You choose!"));
final var completion = this.client.conversation(ModelCategory.MEDIUM, conversation);
LOGGER.info(completion.response());
assertTrue(this.assessScoreBinary("Does the text mention a color?", completion.response()));
}
@Disabled
@Test
void hugePrompt() throws Exception {
final StringBuilder sb = new StringBuilder();
for (int i = 0; i < 10000; i++) {
sb.append(i);
sb.append(", ");
}
sb.delete(sb.length() - 2, sb.length());
final var completion = this.client.completion(
null,
"You are counting machine",
"How many numbers do you count?",
sb.toString());
LOGGER.info(completion.response());
}
@Disabled
@Test
void parallelRequests() throws Exception {
final LocalDateTime start = LocalDateTime.now();
final Tasks tasks = this.createTasks();
final ExecutorService executor = Executors.newFixedThreadPool(4);
try {
CompletableFuture.allOf(
CompletableFuture.runAsync(tasks.task1, executor),
CompletableFuture.runAsync(tasks.task2, executor),
CompletableFuture.runAsync(tasks.task3, executor),
CompletableFuture.runAsync(tasks.task4, executor))
.join();
}
finally {
executor.shutdown();
}
LOGGER.info("Duration: " + java.time.Duration.between(start, LocalDateTime.now()));
}
@Disabled
@Test
void serialRequests() throws Exception {
final LocalDateTime start = LocalDateTime.now();
final Tasks tasks = this.createTasks();
tasks.task1.run();
tasks.task2.run();
tasks.task3.run();
tasks.task4.run();
LOGGER.info("Duration: " + java.time.Duration.between(start, LocalDateTime.now()));
}
private Tasks createTasks() {
return new Tasks(() -> {
try {
final var completion = this.client.completion(
null,
"You are a helpful assistant",
"",
"How do I calculate the annual profit margin?");
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
}, () -> {
try {
final var completion = this.client.completion(
null,
"You are a helpful assistant",
"",
"How do I calculate the operating profit ratio?");
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
}, () -> {
try {
final var completion = this.client.completion(
null,
"You are a helpful assistant",
"",
"How do I calculate the return of investment?");
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
}, () -> {
try {
final var completion = this.client.completion(
null,
"You are a helpful assistant",
"",
"How do I calculate the return on net worth?");
LOGGER.info(completion.response());
}
catch (final Exception e) {
LOGGER.error("Exception", e);
}
});
}
static record Tasks(Runnable task1, Runnable task2, Runnable task3, Runnable task4) {
}
private boolean assessScoreBinary(final String question, final String text) {
LOGGER.info(question);
var score = 0;
while (Math.abs(score) < 3) {
final var completion = this.client.completion(
ModelCategory.MEDIUM,
"""
You are a grader assessing the truthfulness of a given text to a user question.
Please provide a binary response 'true' or 'false' for the following text.
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
Only provide the response as a single word and no preamble and no explanation.""",
"Here is the text:\n" + text,
"\nHere is the user question: " + question);
LOGGER.info(completion.response());
score += Boolean.parseBoolean(completion.response().strip()) ? 1 : -1;
if (Math.abs(score) == 2) {
break;
}
}
final var result = score > 0;
LOGGER.info("->" + String.valueOf(result));
return result;
}
}

57
src/test/java/de/extio/lmlib/client/token/TokenizerTest.java Normal file
View File

@@ -0,0 +1,57 @@
package de.extio.lmlib.client.token;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.PropertySource;
import org.springframework.web.reactive.function.client.WebClient;
import de.extio.lmlib.client.profile.ModelCategory;
import de.extio.lmlib.client.profile.ModelProfileService;
@SpringBootTest(webEnvironment = WebEnvironment.NONE)
@SpringBootConfiguration
@ComponentScan(basePackages = "de.extio.lmlib")
@PropertySource("classpath:/application-test.properties")
public class TokenizerTest {
private static final String TEXT = "Hello this is a text! How are you doing? The weather is great today! Super sunny shiny!!!";
@Autowired
private ModelProfileService modelProfileService;
@Autowired
@Qualifier("lmLibWebClientBuilder")
private WebClient.Builder webClientBuilder;
@Test
void jTokkit() throws Exception {
final var tokenizer = new JTokkitTokenizer();
final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
assertEquals(21, tokenizer.count(TEXT, profile));
final var tokens = tokenizer.tokenize(TEXT, profile);
assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString());
assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
}
/**
* Tested with a Llama 3.1 8B model. The test will likely fail if you load a different model.
*/
@Test
void llamaServer() throws Exception {
final var tokenizer = new LlamaServerTokenizer(this.webClientBuilder);
final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile());
assertEquals(21, tokenizer.count(TEXT, profile));
final var tokens = tokenizer.tokenize(TEXT, profile);
assertEquals("[9906, 420, 374, 264, 1495, 0, 2650, 527, 499, 3815, 30, 578, 9282, 374, 2294, 3432, 0, 7445, 40798, 42299, 12340]", tokens.toString());
assertEquals(TEXT, tokenizer.detokenize(tokens, profile));
}
}