feat: Add Levenshtein distance and similarity methods with corresponding tests
This commit is contained in:
@@ -191,4 +191,52 @@ public class TextUtils {
|
|||||||
final Pattern pattern = wholeWordRegexCache.get(regex);
|
final Pattern pattern = wholeWordRegexCache.get(regex);
|
||||||
return pattern.matcher(text).find();
|
return pattern.matcher(text).find();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static int levenshteinDistance(final String s1, final String s2) {
|
||||||
|
if (s1 == null || s2 == null) {
|
||||||
|
throw new IllegalArgumentException("Strings must not be null");
|
||||||
|
}
|
||||||
|
|
||||||
|
final int len1 = s1.length();
|
||||||
|
final int len2 = s2.length();
|
||||||
|
|
||||||
|
if (len1 == 0) {
|
||||||
|
return len2;
|
||||||
|
}
|
||||||
|
if (len2 == 0) {
|
||||||
|
return len1;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int[] prev = new int[len2 + 1];
|
||||||
|
final int[] curr = new int[len2 + 1];
|
||||||
|
|
||||||
|
for (int j = 0; j <= len2; j++) {
|
||||||
|
prev[j] = j;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 1; i <= len1; i++) {
|
||||||
|
curr[0] = i;
|
||||||
|
for (int j = 1; j <= len2; j++) {
|
||||||
|
final int cost = s1.charAt(i - 1) == s2.charAt(j - 1) ? 0 : 1;
|
||||||
|
curr[j] = Math.min(Math.min(curr[j - 1] + 1, prev[j] + 1), prev[j - 1] + cost);
|
||||||
|
}
|
||||||
|
System.arraycopy(curr, 0, prev, 0, len2 + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return prev[len2];
|
||||||
|
}
|
||||||
|
|
||||||
|
public static double levenshteinSimilarity(final String s1, final String s2) {
|
||||||
|
if (s1 == null || s2 == null) {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int maxLength = Math.max(s1.length(), s2.length());
|
||||||
|
if (maxLength == 0) {
|
||||||
|
return 1.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int distance = levenshteinDistance(s1, s2);
|
||||||
|
return 1.0 - ((double) distance / maxLength);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -237,6 +237,108 @@ public class TextUtilsTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@CsvSource({
|
||||||
|
"'', '', 0",
|
||||||
|
"'abc', '', 3",
|
||||||
|
"'', 'abc', 3",
|
||||||
|
"'abc', 'abc', 0",
|
||||||
|
"'abc', 'def', 3",
|
||||||
|
"'cursed_sword', 'cursed_blade', 5",
|
||||||
|
"'main_quest', 'main_quests', 1",
|
||||||
|
"'betrayal', 'betrayel', 1",
|
||||||
|
"'dragon', 'wagon', 2",
|
||||||
|
"'kitten', 'sitting', 3",
|
||||||
|
"'saturday', 'sunday', 3",
|
||||||
|
"'prophecy_fulfilled', 'prophecy_fullfilled', 1",
|
||||||
|
"'dark_forest', 'deep_forest', 3"
|
||||||
|
})
|
||||||
|
void levenshteinDistanceTest(final String s1, final String s2, final int expectedDistance) {
|
||||||
|
final int distance = TextUtils.levenshteinDistance(s1, s2);
|
||||||
|
assertEquals(expectedDistance, distance);
|
||||||
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@CsvSource({
|
||||||
|
"'', '', 1.0",
|
||||||
|
"'abc', 'abc', 1.0",
|
||||||
|
"'abc', 'def', 0.0",
|
||||||
|
"'cursed_sword', 'cursed_blade', 0.5833333333333334",
|
||||||
|
"'main_quest', 'main_quests', 0.9",
|
||||||
|
"'betrayal', 'betrayel', 0.875",
|
||||||
|
"'dragon', 'wagon', 0.6666666666666667",
|
||||||
|
"'kitten', 'sitting', 0.5714285714285714",
|
||||||
|
"'saturday', 'sunday', 0.625",
|
||||||
|
"'prophecy_fulfilled', 'prophecy_fullfilled', 0.9444444444444444",
|
||||||
|
"'dark_forest', 'deep_forest', 0.72",
|
||||||
|
"'quest', 'quests', 0.8333333333333334",
|
||||||
|
"'npc_merchant', 'npc_merchent', 0.9166666666666666"
|
||||||
|
})
|
||||||
|
void levenshteinSimilarityTest(final String s1, final String s2, final double expectedSimilarity) {
|
||||||
|
final double similarity = TextUtils.levenshteinSimilarity(s1, s2);
|
||||||
|
assertEquals(expectedSimilarity, similarity, 0.01);
|
||||||
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@CsvSource({
|
||||||
|
"'main_quest', 'main_quests', 0.85, true",
|
||||||
|
"'betrayal', 'betrayel', 0.85, true",
|
||||||
|
"'dragon', 'wagon', 0.85, false",
|
||||||
|
"'cursed_sword', 'cursed_blade', 0.85, false",
|
||||||
|
"'prophecy_fulfilled', 'prophecy_fullfilled', 0.85, true",
|
||||||
|
"'dark_forest', 'deep_forest', 0.85, false",
|
||||||
|
"'quest', 'quests', 0.80, true",
|
||||||
|
"'npc_merchant', 'npc_merchent', 0.90, true"
|
||||||
|
})
|
||||||
|
void levenshteinSimilarityThresholdTest(final String s1, final String s2, final double threshold, final boolean expectedMatch) {
|
||||||
|
final double similarity = TextUtils.levenshteinSimilarity(s1, s2);
|
||||||
|
final boolean matches = similarity >= threshold;
|
||||||
|
assertEquals(expectedMatch, matches,
|
||||||
|
String.format("'%s' vs '%s' = %.2f (threshold %.2f)", s1, s2, similarity, threshold));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void levenshteinSimilarityBenchmarkTest() {
|
||||||
|
final String[][] testPairs = {
|
||||||
|
{"cursed_sword", "cursed_blade"},
|
||||||
|
{"main_quest", "main_quests"},
|
||||||
|
{"betrayal_warning", "betrayel_warning"},
|
||||||
|
{"ancient_prophecy_fulfilled", "ancient_prophecy_fullfilled"},
|
||||||
|
{"dark_forest_encounter", "deep_forest_encounter"},
|
||||||
|
{"npc_merchant_guild", "npc_merchent_guild"},
|
||||||
|
{"dragon_slayer_quest", "dragon_slayer_quests"},
|
||||||
|
{"mysterious_artifact", "mysterious_artefact"}
|
||||||
|
};
|
||||||
|
|
||||||
|
final int warmupIterations = 250000;
|
||||||
|
final int benchmarkIterations = 250000;
|
||||||
|
|
||||||
|
for (final String[] pair : testPairs) {
|
||||||
|
for (int i = 0; i < warmupIterations; i++) {
|
||||||
|
TextUtils.levenshteinSimilarity(pair[0], pair[1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
final long startTime = System.nanoTime();
|
||||||
|
for (int i = 0; i < benchmarkIterations; i++) {
|
||||||
|
for (final String[] pair : testPairs) {
|
||||||
|
TextUtils.levenshteinSimilarity(pair[0], pair[1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
final long endTime = System.nanoTime();
|
||||||
|
|
||||||
|
final long totalComparisons = (long) benchmarkIterations * testPairs.length;
|
||||||
|
final long totalTimeNanos = endTime - startTime;
|
||||||
|
final double avgTimeNanos = (double) totalTimeNanos / totalComparisons;
|
||||||
|
final double avgTimeMicros = avgTimeNanos / 1000.0;
|
||||||
|
|
||||||
|
System.out.printf("Levenshtein similarity benchmark:%n");
|
||||||
|
System.out.printf(" Total comparisons: %,d%n", totalComparisons);
|
||||||
|
System.out.printf(" Total time: %.2f ms%n", totalTimeNanos / 1_000_000.0);
|
||||||
|
System.out.printf(" Average time per comparison: %.3f µs (%.0f ns)%n", avgTimeMicros, avgTimeNanos);
|
||||||
|
System.out.printf(" Throughput: %,.0f comparisons/second%n", 1_000_000_000.0 / avgTimeNanos);
|
||||||
|
}
|
||||||
|
|
||||||
private static String generateText() {
|
private static String generateText() {
|
||||||
return "The *flow* of the Camel `route` named `outboundConfirmationMessage` from `start` to finish is as `follows`:\r\n"
|
return "The *flow* of the Camel `route` named `outboundConfirmationMessage` from `start` to finish is as `follows`:\r\n"
|
||||||
+ "1. The `route` starts with a from *endpoint* that *listens* on a direct *endpoint* named `direct:outboundConfirmationMessage`.\r\n"
|
+ "1. The `route` starts with a from *endpoint* that *listens* on a direct *endpoint* named `direct:outboundConfirmationMessage`.\r\n"
|
||||||
|
|||||||
Reference in New Issue
Block a user