datasketch_minhash_jaccard_similarity_estimation_quickstart.py

1from datasketch import MinHash
2
3data1 = ['minhash', 'is', 'a', 'probabilistic', 'data', 'structure', 'for',
4         'estimating', 'the', 'similarity', 'between', 'datasets']
5data2 = ['minhash', 'is', 'a', 'probability', 'data', 'structure', 'for',
6         'estimating', 'the', 'similarity', 'between', 'documents']
7
8m1, m2 = MinHash(), MinHash()
9for d in data1:
10    m1.update(d.encode('utf8'))
11for d in data2:
12    m2.update(d.encode('utf8'))
13
14print("Estimated Jaccard for data1 and data2 is", m1.jaccard(m2))
15
16s1 = set(data1)
17s2 = set(data2)
18actual_jaccard = float(len(s1.intersection(s2))) / float(len(s1.union(s2)))
19print("Actual Jaccard for data1 and data2 is", actual_jaccard)