Added a HashIntSet class that uses hashing with linear probing to get most of

the memory benefits of ArrayIntSet and better performance than a
HashSet<Integer> (presumably because of less boxing and better spatial
locality).  The downside is that you need to pick a sentinel value that can't
be stored in the set, but that's not a problem in any application for which
I've ever used an IntSet.  In some (admittedly simplistic) Google Caliper
testing, HashIntSet was 22% faster than ArrayIntSet and 73% faster than
HashSet for arrays of size N=10, 100% faster than ArrayIntSet and 69% faster
than HashSet for N=100, and 357% faster than ArrayIntSet and 74% faster than
HashSet for N=1000.


git-svn-id: https://samskivert.googlecode.com/svn/trunk@2801 6335cc39-0255-0410-8fd6-9bcaacd3b74c
This commit is contained in:
andrzej@threerings.net
2010-08-11 05:20:43 +00:00
parent 9ed1bfeedc
commit 756c2f44ad
6 changed files with 644 additions and 116 deletions
@@ -37,6 +37,42 @@ import java.util.Iterator;
public abstract class AbstractIntSet extends AbstractSet<Integer>
implements IntSet
{
/**
* Add all of the values in the supplied array to the set.
*
* @param values elements to be added to this set.
*
* @return <tt>true</tt> if this set did not already contain all of the specified elements.
*/
public boolean add (int[] values)
{
boolean modified = false;
int vlength = values.length;
for (int i = 0; i < vlength; i++) {
modified = (add(values[i]) || modified);
}
return modified;
}
/**
* Removes all values in the supplied array from the set. Any values that are in the array but
* not in the set are simply ignored.
*
* @param values elements to be removed from the set.
*
* @return <tt>true</tt> if this set contained any of the specified elements (which will have
* been removed).
*/
public boolean remove (int[] values)
{
boolean modified = false;
int vcount = values.length;
for (int i = 0; i < vcount; i++) {
modified = (remove(values[i]) || modified);
}
return modified;
}
/**
* {@inheritDoc}
*
@@ -87,42 +87,6 @@ public class ArrayIntSet extends AbstractIntSet
this(DEFAULT_CAPACITY);
}
/**
* Add all of the values in the supplied array to the set.
*
* @param values elements to be added to this set.
*
* @return <tt>true</tt> if this set did not already contain all of the specified elements.
*/
public boolean add (int[] values)
{
boolean modified = false;
int vlength = values.length;
for (int i = 0; i < vlength; i++) {
modified = (add(values[i]) || modified);
}
return modified;
}
/**
* Removes all values in the supplied array from the set. Any values that are in the array but
* not in the set are simply ignored.
*
* @param values elements to be removed from the set.
*
* @return <tt>true</tt> if this set contained any of the specified elements (which will have
* been removed).
*/
public boolean remove (int[] values)
{
boolean modified = false;
int vcount = values.length;
for (int i = 0; i < vcount; i++) {
modified = (remove(values[i]) || modified);
}
return modified;
}
/**
* Returns the element at the specified index. Note that the elements in the set are unordered
* and could change order after insertion or removal. This method is useful only for accessing
@@ -0,0 +1,450 @@
//
// $Id$
//
// samskivert library - useful routines for java programs
// Copyright (C) 2001-2010 Michael Bayne, et al.
//
// This library is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation; either version 2.1 of the License, or
// (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package com.samskivert.util;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.NoSuchElementException;
/**
* A set of integers that uses hashing with linear probing to provide most of the memory usage
* and garbage creation benefits of {@link ArrayIntSet} and the performance benefits of
* a {@link java.util.HashSet} of {@link Integer}s (and then some, because of better spatial
* locality). The downside is that it requires a sentinel value ({@link Integer#MIN_VALUE} by
* default) that cannot be stored in the set because it is used internally to represent unused
* locations.
*/
public class HashIntSet extends AbstractIntSet
implements Cloneable, Serializable
{
/**
* Construct a HashIntSet with the specified starting values.
*/
public HashIntSet (int[] values)
{
this(values.length);
add(values);
}
/**
* Construct a HashIntSet with the specified starting values.
*
* @throws NullPointerException if the collection contains any null values.
*/
public HashIntSet (Collection<Integer> values)
{
this(values.size());
addAll(values);
}
/**
* Creates a new set with the default capacity.
*/
public HashIntSet ()
{
this(DEFAULT_CAPACITY);
}
/**
* Creates a new set with the specified initial capacity.
*/
public HashIntSet (int capacity)
{
this(capacity, Integer.MIN_VALUE);
}
/**
* Creates a new set with the specified initial capacity and sentinel value.
*/
public HashIntSet (int capacity, int sentinel)
{
_sentinel = sentinel;
createBuckets(getBucketCount(capacity));
}
/**
* Sets the sentinel value, which cannot itself be stored in the set because it is used
* internally to represent an unused location.
*
* @exception IllegalArgumentException if the set currently contains the requested sentinel.
*/
public void setSentinel (int sentinel)
{
if (_sentinel == sentinel) {
return;
}
if (contains(sentinel)) {
throw new IllegalArgumentException("Set contains sentinel value " + sentinel);
}
// replace every instance of the old sentinel with the new
for (int ii = 0; ii < _buckets.length; ii++) {
if (_buckets[ii] == _sentinel) {
_buckets[ii] = sentinel;
}
}
_sentinel = sentinel;
}
/**
* Returns the sentinel value.
*/
public int getSentinel ()
{
return _sentinel;
}
// documentation inherited from interface IntSet
public Interator interator ()
{
return new AbstractInterator() {
public boolean hasNext () {
checkConcurrentModification();
return _pos < _size;
}
public int nextInt () {
checkConcurrentModification();
if (_pos >= _size) {
throw new NoSuchElementException();
}
if (_idx == 0) {
// start after a sentinel
while (_buckets[_idx++] != _sentinel);
}
int mask = _buckets.length - 1;
for (; _pos < _size; _idx++) {
int value = _buckets[_idx & mask];
if (value != _sentinel) {
_pos++;
_idx++;
return value;
}
}
// we shouldn't get here
throw new RuntimeException("Ran out of elements getting next");
}
public void remove () {
checkConcurrentModification();
if (_idx == 0) {
throw new IllegalStateException("Next method not yet called");
}
int pidx = (--_idx) & (_buckets.length - 1);
if (_buckets[pidx] == _sentinel) {
throw new IllegalStateException("No element to remove");
}
_buckets[pidx] = _sentinel;
_pos--;
_size--;
_omodcount = ++_modcount;
shift(pidx);
}
protected void checkConcurrentModification () {
if (_modcount != _omodcount) {
throw new ConcurrentModificationException();
}
}
protected int _pos, _idx;
protected int _omodcount = _modcount;
};
}
@Override // documentation inherited
public boolean contains (int value)
{
if (value == _sentinel) {
return false;
}
int mask = _buckets.length - 1;
int start = hash(value) & mask, idx = start;
do {
int bvalue = _buckets[idx];
if (bvalue == value) {
return true;
} else if (bvalue == _sentinel) {
return false;
}
} while ((idx = idx + 1 & mask) != start);
// we shouldn't get here
throw new RuntimeException("Ran out of buckets looking for value " + value);
}
@Override // documentation inherited
public int size ()
{
return _size;
}
@Override // documentation inherited
public boolean isEmpty ()
{
return _size == 0;
}
@Override // documentation inherited
public boolean add (int value)
{
if (value == _sentinel) {
throw new IllegalArgumentException("Can't add sentinel value " + value);
}
int mask = _buckets.length - 1;
int start = hash(value) & mask, idx = start;
do {
int bvalue = _buckets[idx];
if (bvalue == value) {
return false;
} else if (bvalue == _sentinel) {
_buckets[idx] = value;
_size++;
_modcount++;
int ncount = getBucketCount(_size, MAX_LOAD_FACTOR);
if (ncount > _buckets.length) {
rehash(ncount);
}
return true;
}
} while ((idx = idx + 1 & mask) != start);
// we shouldn't get here
throw new RuntimeException("Ran out of buckets adding value " + value);
}
@Override // documentation inherited
public boolean remove (int value)
{
if (value == _sentinel) {
return false;
}
int mask = _buckets.length - 1;
int start = hash(value) & mask, idx = start;
do {
int bvalue = _buckets[idx];
if (bvalue == value) {
_buckets[idx] = _sentinel;
_size--;
_modcount++;
int ncount = getBucketCount(_size, MIN_LOAD_FACTOR);
if (ncount < _buckets.length) {
rehash(ncount);
} else {
shift(idx);
}
return true;
} else if (bvalue == _sentinel) {
return false;
}
} while ((idx = idx + 1 & mask) != start);
// we shouldn't get here
throw new RuntimeException("Ran out of buckets removing value " + value);
}
@Override // documentation inherited
public void clear ()
{
createBuckets(MIN_BUCKET_COUNT);
_size = 0;
_modcount++;
}
@Override // documentation inherited
public HashIntSet clone ()
{
try {
HashIntSet nset = (HashIntSet)super.clone();
nset._buckets = _buckets.clone();
return nset;
} catch (CloneNotSupportedException cnse) {
throw new AssertionError(cnse); // won't happen; we're Cloneable
}
}
/**
* Recreates the bucket array with the specified new count.
*/
protected void rehash (int ncount)
{
int[] obuckets = _buckets;
createBuckets(ncount);
for (int idx = 0, pos = 0; pos < _size; idx++) {
int value = obuckets[idx];
if (value != _sentinel) {
readd(value);
pos++;
}
}
}
/**
* (Re)creates and initializes the bucket array.
*/
protected void createBuckets (int count)
{
Arrays.fill(_buckets = new int[count], _sentinel);
}
/**
* Adds a value that we know is neither equal to the sentinel nor already in the set.
*/
protected void readd (int value)
{
int mask = _buckets.length - 1;
int start = hash(value) & mask, idx = start;
do {
if (_buckets[idx] == _sentinel) {
_buckets[idx] = value;
return;
}
} while ((idx = idx + 1 & mask) != start);
// we shouldn't get here
throw new RuntimeException("Ran out of buckets readding value " + value);
}
/**
* Shifts elements over to fill a newly empty slot.
*/
protected void shift (int start)
{
// first, scan backwards to find the previous sentinel
int mask = _buckets.length - 1;
int sidx = start;
while ((sidx = sidx + mask & mask) != start) {
if (_buckets[sidx] == _sentinel) {
break;
}
}
// then forwards to shift elements into place
int idx = start, pidx = start;
while ((idx = idx + 1 & mask) != start) {
int bvalue = _buckets[idx];
if (bvalue == _sentinel) {
_buckets[pidx] = _sentinel;
return;
}
int bidx = hash(bvalue) & mask;
if (pidx > sidx ? (bidx > sidx && bidx <= pidx) : (bidx > sidx || bidx <= pidx)) {
_buckets[pidx] = bvalue;
pidx = idx;
}
}
// we shouldn't get here
throw new RuntimeException("Ran out of buckets fixing empty location at " + start);
}
/**
* Custom serializer.
*/
private void writeObject (ObjectOutputStream out)
throws IOException
{
out.defaultWriteObject();
for (int idx = 0, pos = 0; pos < _size; idx++) {
int value = _buckets[idx];
if (value != _sentinel) {
out.writeInt(value);
pos++;
}
}
}
/**
* Custom deserializer.
*/
private void readObject (ObjectInputStream in)
throws IOException, ClassNotFoundException
{
in.defaultReadObject();
createBuckets(getBucketCount(_size));
for (int ii = 0; ii < _size; ii++) {
readd(in.readInt());
}
}
/**
* Returns the hash of the specified value. This is copied straight from
* the source of {@link java.util.HashMap} and ensures that we don't rely
* solely on the bits of lower significance.
*/
protected static int hash (int h)
{
h ^= (h >>> 20) ^ (h >>> 12);
return h ^ (h >>> 7) ^ (h >>> 4);
}
/**
* Computes the number of buckets needed to provide the given capacity with a load factor
* halfway between the minimum and the maximum.
*/
protected static int getBucketCount (int capacity)
{
return getBucketCount(capacity, (MIN_LOAD_FACTOR + MAX_LOAD_FACTOR) * 0.5f);
}
/**
* Computes the number of buckets needed to provide the given capacity with the specified load
* factor.
*/
protected static int getBucketCount (int capacity, float loadFactor)
{
int size = (int)(capacity / loadFactor);
int highest = Integer.highestOneBit(size);
return Math.max((size == highest) ? highest : (highest << 1), MIN_BUCKET_COUNT);
}
/** The buckets containing the contents of the set. */
protected transient int[] _buckets;
/** The number of elements in the set. */
protected int _size;
/** The value that indicates an empty location in the contents. */
protected int _sentinel;
/** Incremented on each set modification, used to track concurrent changes. */
protected transient int _modcount;
/** The default initial capacity of this set. */
protected static final int DEFAULT_CAPACITY = 16;
/** The minimum number of buckets to provide. */
protected static final int MIN_BUCKET_COUNT = 8;
/** The maximum load factor (ratio of size to length of contents array). */
protected static final float MAX_LOAD_FACTOR = 0.7f;
/** The minimum load factor. */
protected static final float MIN_LOAD_FACTOR = 0.3f;
/** Change this if the fields or inheritance hierarchy ever changes (extremely unlikely). */
private static final long serialVersionUID = 1L;
}
@@ -20,92 +20,18 @@
package com.samskivert.util.tests;
import java.util.Arrays;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import org.junit.*;
import static org.junit.Assert.*;
import com.samskivert.util.AbstractIntSet;
import com.samskivert.util.ArrayIntSet;
public class ArrayIntSetTest
public class ArrayIntSetTest extends IntSetTestBase
{
@Test
public void testAdd ()
protected AbstractIntSet createSet ()
{
ArrayIntSet set = new ArrayIntSet();
set.add(new int[] { 3, 5, 5, 9, 5, 7, 1 });
int[] values = { 1, 3, 5, 7, 9 };
assertTrue("values equal", Arrays.equals(values, set.toIntArray()));
return new ArrayIntSet();
}
@Test
public void testConstruct ()
protected AbstractIntSet createSet (int[] values)
{
int[] values = { 11, 3, 20, 6, 16, 15, 24, 23, 21, 10, 4, 19, 13, 25, 22, 18 };
ArrayIntSet set1 = new ArrayIntSet(values);
ArrayIntSet set2 = new ArrayIntSet();
set2.add(values);
assertTrue(set1.equals(set2));
}
@Test
public void testIterate ()
{
ArrayIntSet set = new ArrayIntSet(new int[] { 3, 5, 5, 9, 5, 7, 1 });
Set<Integer> jset = new TreeSet<Integer>();
jset.addAll(set);
assertTrue(jset.equals(set));
}
@Test
public void testOps ()
{
ArrayIntSet set1 = new ArrayIntSet();
set1.add(new int[] { 1, 2, 3, 5, 7, 12, 19, 35 });
ArrayIntSet set2 = new ArrayIntSet();
set2.add(new int[] { 3, 4, 5, 11, 13, 17, 19, 25 });
ArrayIntSet set3 = new ArrayIntSet();
set3.add(new int[] { 3, 5, 19 });
// intersect sets 1 and 2; making sure that retainAll() returns
// true to indicate that set 1 was modified
assertTrue("retain modifies", set1.retainAll(set2));
// make sure the intersections were correct
assertTrue("intersection", set1.equals(set3));
// make sure that retainAll returns false if we do something that
// doesn't modify the set
assertTrue("retain didn't modify", !set1.retainAll(set3));
Random rando = new Random();
for (int i = 0; i < 1000; i++) {
ArrayIntSet s1 = new ArrayIntSet();
ArrayIntSet s2 = new ArrayIntSet();
ArrayIntSet s3 = new ArrayIntSet();
// add some odd numbers to all three sets
for (int c = 0; c < 100; c++) {
int r = rando.nextInt(5000) * 2 + 1;
s1.add(r);
s2.add(r);
s3.add(r);
}
// now add some even numbers to each of the first two sets in
// non-overlapping ranges
for (int c = 0; c < 100; c++) {
s1.add(rando.nextInt(5000)*2);
s2.add(rando.nextInt(5000)*2 + 15000);
}
// now ensure that s1.retainAll(s2) equals s3
s1.retainAll(s2);
assertTrue("random intersection", s1.equals(s3));
}
return new ArrayIntSet(values);
}
}
@@ -0,0 +1,37 @@
//
// $Id$
//
// samskivert library - useful routines for java programs
// Copyright (C) 2001-2010 Michael Bayne, et al.
//
// This library is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation; either version 2.1 of the License, or
// (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package com.samskivert.util.tests;
import com.samskivert.util.AbstractIntSet;
import com.samskivert.util.HashIntSet;
public class HashIntSetTest extends IntSetTestBase
{
protected AbstractIntSet createSet ()
{
return new HashIntSet();
}
protected AbstractIntSet createSet (int[] values)
{
return new HashIntSet(values);
}
}
@@ -0,0 +1,115 @@
//
// $Id$
//
// samskivert library - useful routines for java programs
// Copyright (C) 2001-2010 Michael Bayne, et al.
//
// This library is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation; either version 2.1 of the License, or
// (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package com.samskivert.util.tests;
import java.util.Arrays;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import org.junit.*;
import static org.junit.Assert.*;
import com.samskivert.util.AbstractIntSet;
public abstract class IntSetTestBase
{
@Test
public void testAdd ()
{
AbstractIntSet set = createSet();
set.add(new int[] { 3, 5, 5, 9, 5, 7, 1 });
int[] values = { 1, 3, 5, 7, 9 };
assertTrue("values equal", Arrays.equals(values, set.toIntArray()));
}
@Test
public void testConstruct ()
{
int[] values = { 11, 3, 20, 6, 16, 15, 24, 23, 21, 10, 4, 19, 13, 25, 22, 18 };
AbstractIntSet set1 = createSet(values);
AbstractIntSet set2 = createSet();
set2.add(values);
assertTrue(set1.equals(set2));
}
@Test
public void testIterate ()
{
AbstractIntSet set = createSet(new int[] { 3, 5, 5, 9, 5, 7, 1 });
Set<Integer> jset = new TreeSet<Integer>();
jset.addAll(set);
assertTrue(jset.equals(set));
}
@Test
public void testOps ()
{
AbstractIntSet set1 = createSet();
set1.add(new int[] { 1, 2, 3, 5, 7, 12, 19, 35 });
AbstractIntSet set2 = createSet();
set2.add(new int[] { 3, 4, 5, 11, 13, 17, 19, 25 });
AbstractIntSet set3 = createSet();
set3.add(new int[] { 3, 5, 19 });
// intersect sets 1 and 2; making sure that retainAll() returns
// true to indicate that set 1 was modified
assertTrue("retain modifies", set1.retainAll(set2));
// make sure the intersections were correct
assertTrue("intersection", set1.equals(set3));
// make sure that retainAll returns false if we do something that
// doesn't modify the set
assertTrue("retain didn't modify", !set1.retainAll(set3));
Random rando = new Random();
for (int i = 0; i < 1000; i++) {
AbstractIntSet s1 = createSet();
AbstractIntSet s2 = createSet();
AbstractIntSet s3 = createSet();
// add some odd numbers to all three sets
for (int c = 0; c < 100; c++) {
int r = rando.nextInt(5000) * 2 + 1;
s1.add(r);
s2.add(r);
s3.add(r);
}
// now add some even numbers to each of the first two sets in
// non-overlapping ranges
for (int c = 0; c < 100; c++) {
s1.add(rando.nextInt(5000)*2);
s2.add(rando.nextInt(5000)*2 + 15000);
}
// now ensure that s1.retainAll(s2) equals s3
s1.retainAll(s2);
assertTrue("random intersection", s1.equals(s3));
}
}
protected abstract AbstractIntSet createSet ();
protected abstract AbstractIntSet createSet (int[] values);
}