Fixed collision bug in HashFunction

Author: janrieke

bloom/src/bloom/hash/HashFunction.java

@@ -12,9 +12,11 @@ public class HashFunction implements Hash {
 
 
 	public HashFunction(){}
-	
+
+	@Override
 	public void print() {}
-	
+
+	@Override
 	public int getHashCode(String key){
 		return 0;
 	}
@@ -44,25 +46,55 @@ public static int fnv1Hash(String key, int m){
 	}
 
 	/**
-	 * Returns an array of k hashes by computing linear
-	 * combinations of the fnv1 hash with the MurmurHash
-	 * 
-	 * @param key	Key in which the hash will be performed on
-	 * @param k		Value specifying how many hash values to compute
-	 * @param m		Modulus value
-	 * @return		An array of size K with hash values [0, m-1]
+	 * Returns an array of k hashes by computing linear combinations of the fnv1 hash with the
+	 * MurmurHash
+	 *
+	 * @param key Key in which the hash will be performed on
+	 * @param k Value specifying how many hash values to compute
+	 * @param m Modulus value; m should be much larger than k
+	 * @return An array of size K with hash values [0, m-1]
 	 */
-	public int[] hash(String key,int k, int m){
+	@Override
+	public int[] hash(String key, int k, int m) {
 		int[] hashes = new int[k];
-		for(int ii = 0; ii < k; ii++){
-			long h1 = fnv1Hash(key);
-			long h2 = murmurHash(key);
-			long axb = h1+(ii*h2);
-			hashes[ii] = (int)(axb % m);
+		long h1 = fnv1Hash(key);
+		long h2 = murmurHash(key);
+		for (int ii = 0; ii < k; ii++ ) {
+			long axb = ii + h1 + (ii * h2);
+			int hashCandidate = (int) (axb % m);
+			int iterations = 0;
+			// Check that the generated hash candidate was not generated before.
+			while (iterations < m && contains(hashes, hashCandidate, ii)) {
+				// Try to find a new hash candidate that is not yet in the array.
+				hashCandidate = (int) ((hashCandidate + murmurHash(Integer.toString(iterations + hashCandidate))) % m);
+				iterations++ ;
+			}
+			if (iterations >= m) {
+				throw new RuntimeException("Failed to generate distinct hashes for: " + key);
+			}
+			hashes[ii] = hashCandidate;
 		}
 		return hashes;
 	}
-	
+
+	/**
+	 * Checks if a value is contained in an array.
+	 *
+	 * @param array The array to search in
+	 * @param valueToFind The value to find in the array
+	 * @param lastIndex The last index of the array that is checked
+	 * @return <code>true</code> if the value was found, <code>false</code> otherwise
+	 */
+	private static boolean contains(final int[] array, final int valueToFind, int lastIndex) {
+		lastIndex = Math.min(array.length, lastIndex);
+		for (int i = 0; i <= lastIndex; i++ ) {
+			if (valueToFind == array[i]) {
+				return true;
+			}
+		}
+		return false;
+	}
+
 	/**
 	 * Returns a single min wise independent hash value
 	 * using a linear transformation
@@ -136,13 +168,13 @@ public static long murmurHash(String string) {
 
 		if (left != 0) {
 			if (left >= 3) {
-				hash ^= (int) data[dataLength - 3] << 16;
+				hash ^= data[dataLength - 3] << 16;
 			}
 			if (left >= 2) {
-				hash ^= (int) data[dataLength - 2] << 8;
+				hash ^= data[dataLength - 2] << 8;
 			}
 			if (left >= 1) {
-				hash ^= (int) data[dataLength - 1];
+				hash ^= data[dataLength - 1];
 			}
 
 			hash *= m;
@@ -152,7 +184,7 @@ public static long murmurHash(String string) {
 		hash *= m;
 		hash ^= hash >>> 15;
 
-		return (long)(hash & 0x00000000ffffffffL);
+		return (hash & 0x00000000ffffffffL);
 	}
 
 }

bloom/test/HashFunctionTest.java

@@ -1,6 +1,7 @@
 
 import static org.junit.Assert.*;
 
+import java.util.Arrays;
 import java.util.UUID;
 
 import org.junit.Test;
@@ -10,13 +11,13 @@
 
 
 public class HashFunctionTest {
-//Constants	
+//Constants
 	private static final int M = 1000000;
 	private static final int m = 100;
 	private static final double ERROR = 0.05;
-	
+
 //Tests
-	
+
 	@Test
 	public void testIfFNV1HashIsDeterministic(){
 		for(int ii= 0; ii < M; ii++){
@@ -26,7 +27,7 @@ public void testIfFNV1HashIsDeterministic(){
 			assertEquals(hash1, hash2);
 		}
 	}
-	
+
 	@Test
 	public void testIfMurmurHashIsDeterministic(){
 		for(int ii = 0; ii < M; ii++){
@@ -36,7 +37,7 @@ public void testIfMurmurHashIsDeterministic(){
 			assertEquals(hash1, hash2);
 		}
 	}
-	
+
 	@Test
 	public void testUniformityOfMurmurHash(){
 		// Initialize array of buckets
@@ -59,7 +60,7 @@ public void testUniformityOfMurmurHash(){
 		}
 		System.out.print("\n");
 	}
-	
+
 	@Test
 	public void testUniformityOfFnv1Hash(){
 		int[] buckets = createFreqArray(m);
@@ -81,13 +82,13 @@ public void testUniformityOfFnv1Hash(){
 		}
 		System.out.print("\n");
 	}
-	
+
 	@Test
 	public void testUniformityOf3Hashes(){
 		int k = 3;
 		kHashFunctions(k);
 	}
-	
+
 	@Test
 	public void testUniformityOf4Hashes(){
 		int k = 4;
@@ -101,8 +102,9 @@ private void kHashFunctions(int k) {
 		HashFunction hash = new HashFunction();
 		for (int ii = 0; ii < M; ii++){
 			int[] hashes = hash.hash(UUID.randomUUID().toString().substring(0, 15), k, m);
-			for(int jj = 0; jj < hashes.length; jj++)
+			for(int jj = 0; jj < hashes.length; jj++) {
 				buckets[hashes[jj]] += 1;
+			}
 		}
 		float low = (float) ((M*k / buckets.length) * (1-ERROR));
 		float high = (float) ((M*k / buckets.length) * (1+ERROR));
@@ -115,12 +117,24 @@ private void kHashFunctions(int k) {
 		}
 		System.out.print("\n");
 	}
-	
+
 	private int[] createFreqArray(int m) {
 		int[] buckets = new int[m];
-		for (int ii = 0; ii < m; ii++)
+		for (int ii = 0; ii < m; ii++) {
 			buckets[ii] = 0;
+		}
 		return buckets;
 	}
 
+	@Test
+	public void testHashesAreDistinct() {
+		int k = 4;
+		HashFunction hash = new HashFunction();
+
+		for (int ii = 0; ii < M; ii++ ) {
+			int[] hashes = hash.hash(UUID.randomUUID().toString().substring(0, 15), k, m);
+			assertEquals(k, Arrays.stream(hashes).distinct().count());
+		}
+	}
+
 }