Corrected q-gram similarity and distance

Author: tdebatty

README.md

@@ -6,7 +6,7 @@ Currently implemeted:
 - Levenshtein edit distance;
 - Jaro-Winkler similarity;
 - Longest Common Subsequence edit distance;
-- Q-Gram (Jaccard index);
+- Q-Gram (Ukkonen);
 - n-Gram distance (Kondrak).
 
 ## Download
@@ -90,7 +90,10 @@ public class MyApp {
 
 ## Q-Gram
 
-Q-Gram similarity, not to confuse with N-Gram distance defined by Kondrak (below), is the relative number of n-grams both strings have in common. It is thus the Jaccard index between the strings considered as sets of n-grams. The computed similarity and distance are relative value (between 0 and 1).
+A-gram similarity and distance, as defined by Ukkonen in "Approximate string-matching with q-grams and maximal matches"
+http://www.sciencedirect.com/science/article/pii/0304397592901434
+
+The distance between two strings is defined as the L1 norm of the difference of their profiles (the number of occurences of each k-shingle). Q-gram distance is a lower bound on Levenshtein distance, but can be computed in O(|A| + |B|), where Levenshtein requires O(|A|.|B|)
 
 ```java
 import info.debatty.java.stringsimilarity.*;
@@ -99,11 +102,14 @@ public class MyApp {
 
     public static void main(String[] args) {
         QGram dig = new QGram(2);
-        
-        // Should be 2: CD and CE
+
+        // AB BC CD CE
+        // 1  1  1  0
+        // 1  1  0  1
+        // Total: 2
         System.out.println(dig.absoluteDistance("ABCD", "ABCE"));
-        
-        // Should be 0.5 (2 / 4)
+
+        // 2 / (3 + 3) = 0.33333
         System.out.println(dig.distance("ABCD", "ABCE"));
     }
 }

pom.xml

@@ -7,7 +7,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>info.debatty</groupId>
     <artifactId>java-string-similarity</artifactId>
-    <version>0.4</version>  
+    <version>0.5</version>  
     <packaging>jar</packaging>
 
     <name>${project.artifactId}</name>

src/main/java/info/debatty/java/stringsimilarity/KShingling.java

@@ -2,7 +2,6 @@
 
 import java.io.Serializable;
 import java.security.InvalidParameterException;
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.regex.Pattern;
 import java.util.Set;
@@ -30,17 +29,33 @@ public static void main(String[] args) {
         ks.parse(s2);
         System.out.println(ks.toString());
 
-        for (boolean b : ks.booleanVectorOf(s1)) {
-            System.out.print(b ? "1" : "0");
-        }
-        System.out.print("\n");
+        printArray(ks.booleanVectorOf(s1));
+        printArray(ks.booleanVectorOf(s2));
+        printArray(ks.profileOf(s1));
+        
+        ks.add("This should trigger an exception!");
+    }
+    
+    public static int countOccurences(String substring, String str){
+        return (str.length() - str.replace(substring, "").length()) / substring.length();
+    }
+    
+    public static void printArray(boolean[] a) {
 
-        for (boolean b : ks.booleanVectorOf(s2)) {
+        System.out.print("[");
+        for (boolean b : a) {
             System.out.print(b ? "1" : "0");
         }
-        System.out.print("\n");
+        System.out.println("]");
+    }
+    
+    public static void printArray(int[] a) {
 
-        ks.add("This should trigger an exception!");
+        System.out.print("[");
+        for (int i : a) {
+            System.out.print("" + i + "\t");
+        }
+        System.out.println("]");
     }
 
     protected int k = 5;
@@ -74,7 +89,17 @@ public final void setK(int k) {
         this.k = k;
     }
 
+    /**
+     * Pattern for finding multiple following spaces
+     */
     private static final Pattern spaceReg = Pattern.compile("\\s+");
+    
+    /**
+     * Extract all k-singles from sting s and add them to the list of possible 
+     * shingles
+     * @param s
+     * @return true
+     */
     public boolean parse(String s) {
         s = spaceReg.matcher(s).replaceAll(" ");
         for (int i = 0; i < (s.length() - k + 1); i++) {
@@ -83,6 +108,11 @@ public boolean parse(String s) {
         return true;
     }
 
+    /**
+     * Add a k-shingle s to the list of possible shingles
+     * @param s
+     * @return 
+     */
     @Override
     public boolean add(String s) {
         if (s.length() != k) {
@@ -93,6 +123,14 @@ public boolean add(String s) {
         return super.add(s);
     }
 
+    /**
+     * Compute and return the boolean vector representation of string s.
+     * E.g. if this set contains the shingles [AB, BC, CD, DE]
+     * and s is ABCD
+     * This will return [true, true, true, false]
+     * @param s
+     * @return 
+     */
     public boolean[] booleanVectorOf(String s) {
         boolean[] r = new boolean[this.size()];
 
@@ -105,6 +143,15 @@ public boolean[] booleanVectorOf(String s) {
         return r;
     }
 
+    /**
+     * Compute the boolean representation of string s, returned as a set of 
+     * position integers.
+     * E.g. if this set contains the shingles [AB, BC, CD, DE]
+     * and s is ABCD
+     * This will return (0, 1, 2)
+     * @param s
+     * @return 
+     */
     public Set<Integer> integerSetOf(String s) {
         Set<Integer> set = new HashSet<Integer>();
         int i = 0;
@@ -118,4 +165,27 @@ public Set<Integer> integerSetOf(String s) {
         return set;
     }
 
+    /**
+     * Compute and return the profile of s, as defined by Ukkonen "Approximate
+     * string-matching with q-grams and maximal matches".
+     * https://www.cs.helsinki.fi/u/ukkonen/TCS92.pdf
+     * The profile is the number of occurences of k-shingles, and is used to 
+     * compute q-gram similarity.
+     * E.g. if this set contains the shingles [AB, BC, CD, DE]
+     * and s is ABCDAB
+     * This will return [2, 1, 1, 0]
+     * @param s
+     * @return 
+     */
+    public int[] profileOf(String s) {
+        int[] p = new int[this.size()];
+        int i = 0;
+        for (String shingle : this) {
+            p[i] = countOccurences(shingle, s);
+            i++;
+        }
+        
+        return p;
+    }
+    
 }

src/main/java/info/debatty/java/stringsimilarity/QGram.java

@@ -1,28 +1,29 @@
 package info.debatty.java.stringsimilarity;
 
 /**
- * QGram similarity is the relative number of n-grams both strings have in 
- * common. It is thus the Jaccard index between strings, considered as sets 
- * of n-grams. The computed similarity and distance are relative value (between 
- * 0 and 1).
- * 
+ * Q-gram similarity and distance.
+ * Defined by Ukkonen in "Approximate string-matching with q-grams and maximal
+ * matches", http://www.sciencedirect.com/science/article/pii/0304397592901434
+ * The distance between two strings is defined as the L1 norm of the difference 
+ * of their profiles (the number of occurences of each k-shingle).
+ * Q-gram distance is a lower bound on Levenshtein distance, but can be computed
+ * in O(|A| + |B|), where Levenshtein requires O(|A|.|B|)
  * @author Thibault Debatty
  */
 public class QGram implements StringSimilarityInterface {
 
     public static void main(String[] args) {
         QGram dig = new QGram(2);
 
-        // Should be 2: CD and CE
+        // AB BC CD CE
+        // 1  1  1  0
+        // 1  1  0  1
+        // Total: 2
         System.out.println(dig.absoluteDistance("ABCD", "ABCE"));
 
-        // Should be 0.5 (2 / 4)
+        // 2 / (3 + 3) = 0.33333
         System.out.println(dig.distance("ABCD", "ABCE"));
 
-        // AB BC CD DE BX XB CE
-        // 2 / 7
-        System.out.println(dig.similarity("ABCDE", "ABXBCE"));
-        
         System.out.println(dig.similarity(
                 "High Qua1ityMedications   Discount On All Reorders = Best Deal Ever! Viagra50/100mg - $1.85 071",
                 "High Qua1ityMedications   Discount On All Reorders = Best Deal Ever! Viagra50/100mg - $1.85 7z3"));
@@ -45,6 +46,14 @@ public double similarity(String s1, String s2) {
 
     @Override
     public double distance(String s1, String s2) {
+        return dist(s1, s2, false);
+    }
+    
+    public int absoluteDistance(String s1, String s2) {
+        return (int) dist(s1, s2, true);
+    }
+    
+    protected double dist(String s1, String s2, boolean abs) {
         if (s1.length() < n || s2.length() < n) {
             return 0;
         }
@@ -53,35 +62,28 @@ public double distance(String s1, String s2) {
         sh.parse(s1);
         sh.parse(s2);
 
-        boolean[] b1 = sh.booleanVectorOf(s1);
-        boolean[] b2 = sh.booleanVectorOf(s2);
+        int[] p1 = sh.profileOf(s1);
+        int[] p2 = sh.profileOf(s2);
+               
 
         int d = 0;
-        for (int i = 0; i < b1.length; i++) {
-            if (b1[i] != b2[i]) {
-                d++;
-            }
+        for (int i = 0; i < p1.length; i++) {
+            d += Math.abs(p1[i] - p2[i]);
         }
 
-        return ((double) d) / sh.size();
-    }
-    
-    public int absoluteDistance(String s1, String s2) {
-        KShingling sh = new KShingling(n);
-        sh.parse(s1);
-        sh.parse(s2);
-        
-        boolean[] b1 = sh.booleanVectorOf(s1);
-        boolean[] b2 = sh.booleanVectorOf(s2);
+        if (abs) {
+            return d;
+        }
 
-        int d = 0;
-        for (int i = 0; i < b1.length; i++) {
-            if (b1[i] != b2[i]) {
-                d++;
-            }
+        int sum = 0;
+        for (int i : p1) {
+            sum += i;
+        }
+        for (int i : p2) {
+            sum += i;
         }
 
-        return d;
+        return (double) d / sum;
     }
 
 }