From 756c2f44ad6393f83bca80b6282c5a6d228ab024 Mon Sep 17 00:00:00 2001 From: "andrzej@threerings.net" Date: Wed, 11 Aug 2010 05:20:43 +0000 Subject: [PATCH] Added a HashIntSet class that uses hashing with linear probing to get most of the memory benefits of ArrayIntSet and better performance than a HashSet (presumably because of less boxing and better spatial locality). The downside is that you need to pick a sentinel value that can't be stored in the set, but that's not a problem in any application for which I've ever used an IntSet. In some (admittedly simplistic) Google Caliper testing, HashIntSet was 22% faster than ArrayIntSet and 73% faster than HashSet for arrays of size N=10, 100% faster than ArrayIntSet and 69% faster than HashSet for N=100, and 357% faster than ArrayIntSet and 74% faster than HashSet for N=1000. git-svn-id: https://samskivert.googlecode.com/svn/trunk@2801 6335cc39-0255-0410-8fd6-9bcaacd3b74c --- .../com/samskivert/util/AbstractIntSet.java | 36 ++ src/java/com/samskivert/util/ArrayIntSet.java | 36 -- src/java/com/samskivert/util/HashIntSet.java | 450 ++++++++++++++++++ .../util/tests/ArrayIntSetTest.java | 86 +--- .../samskivert/util/tests/HashIntSetTest.java | 37 ++ .../samskivert/util/tests/IntSetTestBase.java | 115 +++++ 6 files changed, 644 insertions(+), 116 deletions(-) create mode 100644 src/java/com/samskivert/util/HashIntSet.java create mode 100644 src/java/com/samskivert/util/tests/HashIntSetTest.java create mode 100644 src/java/com/samskivert/util/tests/IntSetTestBase.java diff --git a/src/java/com/samskivert/util/AbstractIntSet.java b/src/java/com/samskivert/util/AbstractIntSet.java index 2044b731..68431607 100644 --- a/src/java/com/samskivert/util/AbstractIntSet.java +++ b/src/java/com/samskivert/util/AbstractIntSet.java @@ -37,6 +37,42 @@ import java.util.Iterator; public abstract class AbstractIntSet extends AbstractSet implements IntSet { + /** + * Add all of the values in the supplied array to the set. + * + * @param values elements to be added to this set. + * + * @return true if this set did not already contain all of the specified elements. + */ + public boolean add (int[] values) + { + boolean modified = false; + int vlength = values.length; + for (int i = 0; i < vlength; i++) { + modified = (add(values[i]) || modified); + } + return modified; + } + + /** + * Removes all values in the supplied array from the set. Any values that are in the array but + * not in the set are simply ignored. + * + * @param values elements to be removed from the set. + * + * @return true if this set contained any of the specified elements (which will have + * been removed). + */ + public boolean remove (int[] values) + { + boolean modified = false; + int vcount = values.length; + for (int i = 0; i < vcount; i++) { + modified = (remove(values[i]) || modified); + } + return modified; + } + /** * {@inheritDoc} * diff --git a/src/java/com/samskivert/util/ArrayIntSet.java b/src/java/com/samskivert/util/ArrayIntSet.java index 9eef94f8..e8277aa8 100644 --- a/src/java/com/samskivert/util/ArrayIntSet.java +++ b/src/java/com/samskivert/util/ArrayIntSet.java @@ -87,42 +87,6 @@ public class ArrayIntSet extends AbstractIntSet this(DEFAULT_CAPACITY); } - /** - * Add all of the values in the supplied array to the set. - * - * @param values elements to be added to this set. - * - * @return true if this set did not already contain all of the specified elements. - */ - public boolean add (int[] values) - { - boolean modified = false; - int vlength = values.length; - for (int i = 0; i < vlength; i++) { - modified = (add(values[i]) || modified); - } - return modified; - } - - /** - * Removes all values in the supplied array from the set. Any values that are in the array but - * not in the set are simply ignored. - * - * @param values elements to be removed from the set. - * - * @return true if this set contained any of the specified elements (which will have - * been removed). - */ - public boolean remove (int[] values) - { - boolean modified = false; - int vcount = values.length; - for (int i = 0; i < vcount; i++) { - modified = (remove(values[i]) || modified); - } - return modified; - } - /** * Returns the element at the specified index. Note that the elements in the set are unordered * and could change order after insertion or removal. This method is useful only for accessing diff --git a/src/java/com/samskivert/util/HashIntSet.java b/src/java/com/samskivert/util/HashIntSet.java new file mode 100644 index 00000000..c67c82e9 --- /dev/null +++ b/src/java/com/samskivert/util/HashIntSet.java @@ -0,0 +1,450 @@ +// +// $Id$ +// +// samskivert library - useful routines for java programs +// Copyright (C) 2001-2010 Michael Bayne, et al. +// +// This library is free software; you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published +// by the Free Software Foundation; either version 2.1 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package com.samskivert.util; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +import java.util.Arrays; +import java.util.Collection; +import java.util.ConcurrentModificationException; +import java.util.NoSuchElementException; + +/** + * A set of integers that uses hashing with linear probing to provide most of the memory usage + * and garbage creation benefits of {@link ArrayIntSet} and the performance benefits of + * a {@link java.util.HashSet} of {@link Integer}s (and then some, because of better spatial + * locality). The downside is that it requires a sentinel value ({@link Integer#MIN_VALUE} by + * default) that cannot be stored in the set because it is used internally to represent unused + * locations. + */ +public class HashIntSet extends AbstractIntSet + implements Cloneable, Serializable +{ + /** + * Construct a HashIntSet with the specified starting values. + */ + public HashIntSet (int[] values) + { + this(values.length); + add(values); + } + + /** + * Construct a HashIntSet with the specified starting values. + * + * @throws NullPointerException if the collection contains any null values. + */ + public HashIntSet (Collection values) + { + this(values.size()); + addAll(values); + } + + /** + * Creates a new set with the default capacity. + */ + public HashIntSet () + { + this(DEFAULT_CAPACITY); + } + + /** + * Creates a new set with the specified initial capacity. + */ + public HashIntSet (int capacity) + { + this(capacity, Integer.MIN_VALUE); + } + + /** + * Creates a new set with the specified initial capacity and sentinel value. + */ + public HashIntSet (int capacity, int sentinel) + { + _sentinel = sentinel; + createBuckets(getBucketCount(capacity)); + } + + /** + * Sets the sentinel value, which cannot itself be stored in the set because it is used + * internally to represent an unused location. + * + * @exception IllegalArgumentException if the set currently contains the requested sentinel. + */ + public void setSentinel (int sentinel) + { + if (_sentinel == sentinel) { + return; + } + if (contains(sentinel)) { + throw new IllegalArgumentException("Set contains sentinel value " + sentinel); + } + // replace every instance of the old sentinel with the new + for (int ii = 0; ii < _buckets.length; ii++) { + if (_buckets[ii] == _sentinel) { + _buckets[ii] = sentinel; + } + } + _sentinel = sentinel; + } + + /** + * Returns the sentinel value. + */ + public int getSentinel () + { + return _sentinel; + } + + // documentation inherited from interface IntSet + public Interator interator () + { + return new AbstractInterator() { + public boolean hasNext () { + checkConcurrentModification(); + return _pos < _size; + } + public int nextInt () { + checkConcurrentModification(); + if (_pos >= _size) { + throw new NoSuchElementException(); + } + if (_idx == 0) { + // start after a sentinel + while (_buckets[_idx++] != _sentinel); + } + int mask = _buckets.length - 1; + for (; _pos < _size; _idx++) { + int value = _buckets[_idx & mask]; + if (value != _sentinel) { + _pos++; + _idx++; + return value; + } + } + // we shouldn't get here + throw new RuntimeException("Ran out of elements getting next"); + } + public void remove () { + checkConcurrentModification(); + if (_idx == 0) { + throw new IllegalStateException("Next method not yet called"); + } + int pidx = (--_idx) & (_buckets.length - 1); + if (_buckets[pidx] == _sentinel) { + throw new IllegalStateException("No element to remove"); + } + _buckets[pidx] = _sentinel; + _pos--; + _size--; + _omodcount = ++_modcount; + shift(pidx); + } + protected void checkConcurrentModification () { + if (_modcount != _omodcount) { + throw new ConcurrentModificationException(); + } + } + protected int _pos, _idx; + protected int _omodcount = _modcount; + }; + } + + @Override // documentation inherited + public boolean contains (int value) + { + if (value == _sentinel) { + return false; + } + int mask = _buckets.length - 1; + int start = hash(value) & mask, idx = start; + do { + int bvalue = _buckets[idx]; + if (bvalue == value) { + return true; + } else if (bvalue == _sentinel) { + return false; + } + } while ((idx = idx + 1 & mask) != start); + + // we shouldn't get here + throw new RuntimeException("Ran out of buckets looking for value " + value); + } + + @Override // documentation inherited + public int size () + { + return _size; + } + + @Override // documentation inherited + public boolean isEmpty () + { + return _size == 0; + } + + @Override // documentation inherited + public boolean add (int value) + { + if (value == _sentinel) { + throw new IllegalArgumentException("Can't add sentinel value " + value); + } + int mask = _buckets.length - 1; + int start = hash(value) & mask, idx = start; + do { + int bvalue = _buckets[idx]; + if (bvalue == value) { + return false; + + } else if (bvalue == _sentinel) { + _buckets[idx] = value; + _size++; + _modcount++; + int ncount = getBucketCount(_size, MAX_LOAD_FACTOR); + if (ncount > _buckets.length) { + rehash(ncount); + } + return true; + } + } while ((idx = idx + 1 & mask) != start); + + // we shouldn't get here + throw new RuntimeException("Ran out of buckets adding value " + value); + } + + @Override // documentation inherited + public boolean remove (int value) + { + if (value == _sentinel) { + return false; + } + int mask = _buckets.length - 1; + int start = hash(value) & mask, idx = start; + do { + int bvalue = _buckets[idx]; + if (bvalue == value) { + _buckets[idx] = _sentinel; + _size--; + _modcount++; + int ncount = getBucketCount(_size, MIN_LOAD_FACTOR); + if (ncount < _buckets.length) { + rehash(ncount); + } else { + shift(idx); + } + return true; + + } else if (bvalue == _sentinel) { + return false; + } + } while ((idx = idx + 1 & mask) != start); + + // we shouldn't get here + throw new RuntimeException("Ran out of buckets removing value " + value); + } + + @Override // documentation inherited + public void clear () + { + createBuckets(MIN_BUCKET_COUNT); + _size = 0; + _modcount++; + } + + @Override // documentation inherited + public HashIntSet clone () + { + try { + HashIntSet nset = (HashIntSet)super.clone(); + nset._buckets = _buckets.clone(); + return nset; + + } catch (CloneNotSupportedException cnse) { + throw new AssertionError(cnse); // won't happen; we're Cloneable + } + } + + /** + * Recreates the bucket array with the specified new count. + */ + protected void rehash (int ncount) + { + int[] obuckets = _buckets; + createBuckets(ncount); + + for (int idx = 0, pos = 0; pos < _size; idx++) { + int value = obuckets[idx]; + if (value != _sentinel) { + readd(value); + pos++; + } + } + } + + /** + * (Re)creates and initializes the bucket array. + */ + protected void createBuckets (int count) + { + Arrays.fill(_buckets = new int[count], _sentinel); + } + + /** + * Adds a value that we know is neither equal to the sentinel nor already in the set. + */ + protected void readd (int value) + { + int mask = _buckets.length - 1; + int start = hash(value) & mask, idx = start; + do { + if (_buckets[idx] == _sentinel) { + _buckets[idx] = value; + return; + } + } while ((idx = idx + 1 & mask) != start); + + // we shouldn't get here + throw new RuntimeException("Ran out of buckets readding value " + value); + } + + /** + * Shifts elements over to fill a newly empty slot. + */ + protected void shift (int start) + { + // first, scan backwards to find the previous sentinel + int mask = _buckets.length - 1; + int sidx = start; + while ((sidx = sidx + mask & mask) != start) { + if (_buckets[sidx] == _sentinel) { + break; + } + } + + // then forwards to shift elements into place + int idx = start, pidx = start; + while ((idx = idx + 1 & mask) != start) { + int bvalue = _buckets[idx]; + if (bvalue == _sentinel) { + _buckets[pidx] = _sentinel; + return; + } + int bidx = hash(bvalue) & mask; + if (pidx > sidx ? (bidx > sidx && bidx <= pidx) : (bidx > sidx || bidx <= pidx)) { + _buckets[pidx] = bvalue; + pidx = idx; + } + } + + // we shouldn't get here + throw new RuntimeException("Ran out of buckets fixing empty location at " + start); + } + + /** + * Custom serializer. + */ + private void writeObject (ObjectOutputStream out) + throws IOException + { + out.defaultWriteObject(); + for (int idx = 0, pos = 0; pos < _size; idx++) { + int value = _buckets[idx]; + if (value != _sentinel) { + out.writeInt(value); + pos++; + } + } + } + + /** + * Custom deserializer. + */ + private void readObject (ObjectInputStream in) + throws IOException, ClassNotFoundException + { + in.defaultReadObject(); + createBuckets(getBucketCount(_size)); + for (int ii = 0; ii < _size; ii++) { + readd(in.readInt()); + } + } + + /** + * Returns the hash of the specified value. This is copied straight from + * the source of {@link java.util.HashMap} and ensures that we don't rely + * solely on the bits of lower significance. + */ + protected static int hash (int h) + { + h ^= (h >>> 20) ^ (h >>> 12); + return h ^ (h >>> 7) ^ (h >>> 4); + } + + /** + * Computes the number of buckets needed to provide the given capacity with a load factor + * halfway between the minimum and the maximum. + */ + protected static int getBucketCount (int capacity) + { + return getBucketCount(capacity, (MIN_LOAD_FACTOR + MAX_LOAD_FACTOR) * 0.5f); + } + + /** + * Computes the number of buckets needed to provide the given capacity with the specified load + * factor. + */ + protected static int getBucketCount (int capacity, float loadFactor) + { + int size = (int)(capacity / loadFactor); + int highest = Integer.highestOneBit(size); + return Math.max((size == highest) ? highest : (highest << 1), MIN_BUCKET_COUNT); + } + + /** The buckets containing the contents of the set. */ + protected transient int[] _buckets; + + /** The number of elements in the set. */ + protected int _size; + + /** The value that indicates an empty location in the contents. */ + protected int _sentinel; + + /** Incremented on each set modification, used to track concurrent changes. */ + protected transient int _modcount; + + /** The default initial capacity of this set. */ + protected static final int DEFAULT_CAPACITY = 16; + + /** The minimum number of buckets to provide. */ + protected static final int MIN_BUCKET_COUNT = 8; + + /** The maximum load factor (ratio of size to length of contents array). */ + protected static final float MAX_LOAD_FACTOR = 0.7f; + + /** The minimum load factor. */ + protected static final float MIN_LOAD_FACTOR = 0.3f; + + /** Change this if the fields or inheritance hierarchy ever changes (extremely unlikely). */ + private static final long serialVersionUID = 1L; +} diff --git a/src/java/com/samskivert/util/tests/ArrayIntSetTest.java b/src/java/com/samskivert/util/tests/ArrayIntSetTest.java index 65be8c38..5d5c999c 100644 --- a/src/java/com/samskivert/util/tests/ArrayIntSetTest.java +++ b/src/java/com/samskivert/util/tests/ArrayIntSetTest.java @@ -20,92 +20,18 @@ package com.samskivert.util.tests; -import java.util.Arrays; -import java.util.Random; -import java.util.Set; -import java.util.TreeSet; - -import org.junit.*; -import static org.junit.Assert.*; - +import com.samskivert.util.AbstractIntSet; import com.samskivert.util.ArrayIntSet; -public class ArrayIntSetTest +public class ArrayIntSetTest extends IntSetTestBase { - @Test - public void testAdd () + protected AbstractIntSet createSet () { - ArrayIntSet set = new ArrayIntSet(); - set.add(new int[] { 3, 5, 5, 9, 5, 7, 1 }); - int[] values = { 1, 3, 5, 7, 9 }; - assertTrue("values equal", Arrays.equals(values, set.toIntArray())); + return new ArrayIntSet(); } - @Test - public void testConstruct () + protected AbstractIntSet createSet (int[] values) { - int[] values = { 11, 3, 20, 6, 16, 15, 24, 23, 21, 10, 4, 19, 13, 25, 22, 18 }; - ArrayIntSet set1 = new ArrayIntSet(values); - ArrayIntSet set2 = new ArrayIntSet(); - set2.add(values); - assertTrue(set1.equals(set2)); - } - - @Test - public void testIterate () - { - ArrayIntSet set = new ArrayIntSet(new int[] { 3, 5, 5, 9, 5, 7, 1 }); - Set jset = new TreeSet(); - jset.addAll(set); - assertTrue(jset.equals(set)); - } - - @Test - public void testOps () - { - ArrayIntSet set1 = new ArrayIntSet(); - set1.add(new int[] { 1, 2, 3, 5, 7, 12, 19, 35 }); - ArrayIntSet set2 = new ArrayIntSet(); - set2.add(new int[] { 3, 4, 5, 11, 13, 17, 19, 25 }); - - ArrayIntSet set3 = new ArrayIntSet(); - set3.add(new int[] { 3, 5, 19 }); - - // intersect sets 1 and 2; making sure that retainAll() returns - // true to indicate that set 1 was modified - assertTrue("retain modifies", set1.retainAll(set2)); - - // make sure the intersections were correct - assertTrue("intersection", set1.equals(set3)); - - // make sure that retainAll returns false if we do something that - // doesn't modify the set - assertTrue("retain didn't modify", !set1.retainAll(set3)); - - Random rando = new Random(); - for (int i = 0; i < 1000; i++) { - ArrayIntSet s1 = new ArrayIntSet(); - ArrayIntSet s2 = new ArrayIntSet(); - ArrayIntSet s3 = new ArrayIntSet(); - - // add some odd numbers to all three sets - for (int c = 0; c < 100; c++) { - int r = rando.nextInt(5000) * 2 + 1; - s1.add(r); - s2.add(r); - s3.add(r); - } - - // now add some even numbers to each of the first two sets in - // non-overlapping ranges - for (int c = 0; c < 100; c++) { - s1.add(rando.nextInt(5000)*2); - s2.add(rando.nextInt(5000)*2 + 15000); - } - - // now ensure that s1.retainAll(s2) equals s3 - s1.retainAll(s2); - assertTrue("random intersection", s1.equals(s3)); - } + return new ArrayIntSet(values); } } diff --git a/src/java/com/samskivert/util/tests/HashIntSetTest.java b/src/java/com/samskivert/util/tests/HashIntSetTest.java new file mode 100644 index 00000000..cad0df06 --- /dev/null +++ b/src/java/com/samskivert/util/tests/HashIntSetTest.java @@ -0,0 +1,37 @@ +// +// $Id$ +// +// samskivert library - useful routines for java programs +// Copyright (C) 2001-2010 Michael Bayne, et al. +// +// This library is free software; you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published +// by the Free Software Foundation; either version 2.1 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package com.samskivert.util.tests; + +import com.samskivert.util.AbstractIntSet; +import com.samskivert.util.HashIntSet; + +public class HashIntSetTest extends IntSetTestBase +{ + protected AbstractIntSet createSet () + { + return new HashIntSet(); + } + + protected AbstractIntSet createSet (int[] values) + { + return new HashIntSet(values); + } +} diff --git a/src/java/com/samskivert/util/tests/IntSetTestBase.java b/src/java/com/samskivert/util/tests/IntSetTestBase.java new file mode 100644 index 00000000..068fcc0c --- /dev/null +++ b/src/java/com/samskivert/util/tests/IntSetTestBase.java @@ -0,0 +1,115 @@ +// +// $Id$ +// +// samskivert library - useful routines for java programs +// Copyright (C) 2001-2010 Michael Bayne, et al. +// +// This library is free software; you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published +// by the Free Software Foundation; either version 2.1 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package com.samskivert.util.tests; + +import java.util.Arrays; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; + +import org.junit.*; +import static org.junit.Assert.*; + +import com.samskivert.util.AbstractIntSet; + +public abstract class IntSetTestBase +{ + @Test + public void testAdd () + { + AbstractIntSet set = createSet(); + set.add(new int[] { 3, 5, 5, 9, 5, 7, 1 }); + int[] values = { 1, 3, 5, 7, 9 }; + assertTrue("values equal", Arrays.equals(values, set.toIntArray())); + } + + @Test + public void testConstruct () + { + int[] values = { 11, 3, 20, 6, 16, 15, 24, 23, 21, 10, 4, 19, 13, 25, 22, 18 }; + AbstractIntSet set1 = createSet(values); + AbstractIntSet set2 = createSet(); + set2.add(values); + assertTrue(set1.equals(set2)); + } + + @Test + public void testIterate () + { + AbstractIntSet set = createSet(new int[] { 3, 5, 5, 9, 5, 7, 1 }); + Set jset = new TreeSet(); + jset.addAll(set); + assertTrue(jset.equals(set)); + } + + @Test + public void testOps () + { + AbstractIntSet set1 = createSet(); + set1.add(new int[] { 1, 2, 3, 5, 7, 12, 19, 35 }); + AbstractIntSet set2 = createSet(); + set2.add(new int[] { 3, 4, 5, 11, 13, 17, 19, 25 }); + + AbstractIntSet set3 = createSet(); + set3.add(new int[] { 3, 5, 19 }); + + // intersect sets 1 and 2; making sure that retainAll() returns + // true to indicate that set 1 was modified + assertTrue("retain modifies", set1.retainAll(set2)); + + // make sure the intersections were correct + assertTrue("intersection", set1.equals(set3)); + + // make sure that retainAll returns false if we do something that + // doesn't modify the set + assertTrue("retain didn't modify", !set1.retainAll(set3)); + + Random rando = new Random(); + for (int i = 0; i < 1000; i++) { + AbstractIntSet s1 = createSet(); + AbstractIntSet s2 = createSet(); + AbstractIntSet s3 = createSet(); + + // add some odd numbers to all three sets + for (int c = 0; c < 100; c++) { + int r = rando.nextInt(5000) * 2 + 1; + s1.add(r); + s2.add(r); + s3.add(r); + } + + // now add some even numbers to each of the first two sets in + // non-overlapping ranges + for (int c = 0; c < 100; c++) { + s1.add(rando.nextInt(5000)*2); + s2.add(rando.nextInt(5000)*2 + 15000); + } + + // now ensure that s1.retainAll(s2) equals s3 + s1.retainAll(s2); + assertTrue("random intersection", s1.equals(s3)); + } + } + + protected abstract AbstractIntSet createSet (); + + protected abstract AbstractIntSet createSet (int[] values); +}