While investigating problems, add much paranoid profiling. Also let's do an explicit keyset copy during enumeration rather than the fancy concurrent hashmaps and iterable mappings. Finally, let's fully synchronize the other maps we're using.

src/java/com/samskivert/depot/EHCacheAdapter.java

@@ -25,9 +25,6 @@ import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
 
-import com.google.common.base.Function;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.samskivert.util.Tuple;
@@ -72,17 +69,25 @@ public class EHCacheAdapter
     // from CacheAdapter
     public <T> CachedValue<T> lookup (String cacheId, Serializable key)
     {
+        long now = System.currentTimeMillis();
         @SuppressWarnings("unchecked")
         EHCacheBin<T> bin = (EHCacheBin<T>) _bins.get(cacheId);
         if (bin == null) {
             return null;
         }
-        return lookup(bin.getCache(), cacheId, key);
+        CachedValue<T> result = lookup(bin.getCache(), cacheId, key);
+        long dT = System.currentTimeMillis() - now;
+        if (dT > 50) {
+            log.warning("Aii! A simple ehcache lookup took over 50 ms!", "cacheId", cacheId,
+                "key", key, "dT", dT);
+        }
+        return result;
     }
 
     // from CacheAdapter
     public <T> void store (CacheCategory category, String cacheId, Serializable key, T value)
     {
+        long now = System.currentTimeMillis();
         Ehcache cache = _categories.get(category);
         if (cache == null) {
             throw new IllegalArgumentException("Unknown category: " + category);
@@ -94,35 +99,57 @@ public class EHCacheAdapter
             _bins.put(cacheId, bin);
         }
         bin.getCache().put(new Element(new EHCacheKey(cacheId, key), value != null ? value : NULL));
+        long dT = System.currentTimeMillis() - now;
+        if (dT > 50) {
+            log.warning("Aii! A simple ehcache store took over 50 ms!", "cacheId", cacheId,
+                "key", key, "dT", dT);
+        }
     }
 
     // from CacheAdapter
     public void remove (String cacheId, Serializable key)
     {
+        long now = System.currentTimeMillis();
         EHCacheBin<?> bin = _bins.get(cacheId);
         if (bin != null) {
             bin.getCache().remove(new EHCacheKey(cacheId, key));
         }
+        long dT = System.currentTimeMillis() - now;
+        if (dT > 50) {
+            log.warning("Aii! A simple ehcache remove took over 50 ms!", "cacheId", cacheId,
+                "key", key, "dT", dT);
+        }
     }
 
     // from CacheAdapter
     public <T> Iterable<Tuple<Serializable, CachedValue<T>>> enumerate (final String cacheId)
     {
+        if (_slowEnumerations > 5) {
+            return Collections.emptySet();
+        }
+        long now = System.currentTimeMillis();
+
         EHCacheBin<?> bin = _bins.get(cacheId);
         if (bin == null) {
             return Collections.emptySet();
         }
 
-        final Ehcache cache = bin.getCache();
+        Set<Tuple<Serializable, CachedValue<T>>> result = Sets.newHashSet();
-        Iterable<Tuple<Serializable, CachedValue<T>>> tuples = Iterables.transform(
+        Ehcache cache = bin.getCache();
-            bin.getKeys(), new Function<Serializable, Tuple<Serializable, CachedValue<T>>> () {
-                public Tuple<Serializable, CachedValue<T>> apply (Serializable key) {
-                    CachedValue<T> value = lookup(cache, cacheId, key);
-                    return (value != null) ? Tuple.newTuple(key, value) : null;
-                }
-            });
 
-        return Iterables.filter(tuples, Predicates.notNull());
+        for (Serializable key : bin.getKeys()) {
+            CachedValue<T> value = lookup(cache, cacheId, key);
+            if (value != null) {
+                result.add(Tuple.newTuple(key, value));
+            }
+        }
+        long dT = System.currentTimeMillis() - now;
+        if (dT > 50) {
+            _slowEnumerations ++;
+            log.warning("Aii! Enumerating cache took a long time!", "cacheId", cacheId, "dT", dT,
+                "cacheSize", result.size(), "disabled", _slowEnumerations > 5);
+        }
+        return result;
     }
 
     // from CacheAdapter
@@ -163,8 +190,13 @@ public class EHCacheAdapter
         return cache;
     }
 
-    protected Map<CacheCategory, Ehcache> _categories = Maps.newConcurrentHashMap();
+    // TODO: To be removed when we're done investigating potential sluggishness
-    protected Map<String, EHCacheBin<?>> _bins = Maps.newConcurrentHashMap();
+    protected int _slowEnumerations = 0;
+
+    protected Map<CacheCategory, Ehcache> _categories =
+        Collections.synchronizedMap(Maps.<CacheCategory, Ehcache>newHashMap());
+    protected Map<String, EHCacheBin<?>> _bins =
+        Collections.synchronizedMap(Maps.<String, EHCacheBin<?>> newHashMap());
 
     protected CacheEventListener _cacheEventListener = new CacheEventListener() {
         public Object clone () throws CloneNotSupportedException {