Updated the dictionary service, now with Real Functionality :) - loads language-specific

dictionary files, and acually uses them for user queries.



git-svn-id: svn+ssh://src.earth.threerings.net/vilya/trunk@185 c613c5cb-e716-0410-b11b-feb51c14d237

src/java/com/threerings/ezgame/server/DictionaryService.java

@@ -28,26 +28,167 @@ import com.threerings.crowd.server.CrowdServer;
 
 import com.threerings.presents.client.InvocationService;
 
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.File;
 import java.lang.reflect.Array;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Random;
+import java.util.Set;
+import java.util.Vector;
+import java.util.logging.Level;
+
 import static com.threerings.ezgame.server.Log.log;
 
 /**
  * Manages loading and querying word dictionaries in multiple languages.
  *
  * NOTE: the service supports lazy loading of language files, but does not
- * _unload_ them from memory, leading to increasing memory usage. 
+ * _unload_ them from memory, leading to increasing memory usage.
+ *
+ * NOTE: the dictionary service has not yet been tested with language
+ * files written in non-default character encodings.
  */
 public class DictionaryService
 {
     /**
-     * Creates the singleton instance of the dictionary service.
+     * Helper class, encapsulates a sorted array of word hashes,
+     * which can be used to look up the existence of a word.
+     */
+    private class Dictionary
+    {
+        /**
+         * Constructor, loads up the word list and initializes storage.
+         * This naive version assumes language files are simple list of words,
+         * with one word per line.
+         */
+        public Dictionary (File wordfile)
+        {
+            try
+            {
+                if (wordfile.exists() && wordfile.isFile() && wordfile.canRead())
+                {
+                    // File length will give us a rough starting point for the word array
+                    long bytecount = wordfile.length();
+                    int approxWords = (int) (bytecount / 5); // just a heuristic,
+                    _hashes.ensureCapacity (approxWords);    //   we'll trim the vector later
+
+                    // Read it line by line
+                    BufferedReader reader = new BufferedReader (new FileReader (wordfile));
+                    String line = null;
+                    while ((line = reader.readLine()) != null)
+                    {
+                        // Add the word to the dictionary
+                        int hash = hashWord (line); 
+                        _hashes.add (hash);
+                        
+                        // Add each letter to a letter set
+                        int len = line.length ();
+                        for (int i = 0; i < len; i++)
+                        {
+                            char ch = line.charAt (i);
+                            if (! _letters.containsKey (ch))
+                            {
+                                _letters.put (ch, 1);
+                            }
+                            else
+                            {
+                                _letters.put (ch, _letters.get (ch) + 1);
+                            }
+                        }
+                        
+                    }
+
+                    // Trim and sort the vector
+                    _hashes.trimToSize();
+                    Collections.sort (_hashes);
+
+                    log.log (Level.INFO,
+                             "Loaded dictionary file " + wordfile.getName () +
+                             " with " + _hashes.size () + " entries, " +
+                             _letters.size () + " letters.");
+                                                          
+                }
+                else
+                {
+                    log.log (Level.WARNING,
+                             "Could not access dictionary file " + wordfile.getAbsolutePath ());
+                }
+            }
+            catch (Exception ex)
+            {
+                log.log (Level.WARNING, "Failed to load dictionary file", ex);
+                _hashes.clear();  // dump everything
+            }
+        }
+
+        /** Checks if the specified word exists in the word list */
+        public boolean contains (String word)
+        {
+            // Hash the word and check
+            int hash = hashWord (word);
+            int result = Collections.binarySearch (_hashes, hash);
+            return (result >= 0);
+        }
+
+        /** Gets an array of random letters for the language, with uniform distribution. */
+        public char[] randomLetters (int count)
+        {
+            Set <Character> letterSet = _letters.keySet();
+            int letterCount = letterSet.size();
+            Character [] letters = new Character [letterCount];
+            letterSet.toArray (letters);
+            
+            char [] results = new char [count];
+            
+            for (int i = 0; i < count; i++)
+            {
+                int r = _random.nextInt (letterCount);
+                char ch = letters[r];
+                results[i] = ch;
+            }
+
+            return results;
+        }
+            
+
+        /** Hashes the word, for use in storage */
+        private int hashWord (String word)
+        {
+            return word.toLowerCase().hashCode();
+        }
+
+        // PRIVATE STORAGE
+
+        /** Sorted vector of word hashes */
+        private Vector <Integer> _hashes = new Vector <Integer> ();
+
+        /** Mapping from letters in this language to their total count */
+        private HashMap <Character, Integer> _letters = new HashMap <Character, Integer> ();
+
+        /** Random number generator */
+        private Random _random = new Random ();
+    }
+
+
+    /**
+     * Obsolete init function. I'm keeping it around to prevent
+     * any automated build breakage between the upcoming Vilya checkin
+     * and the subsequent MSoy checkin. But this will go away real soon.
      */
     public static void init ()
     {
-        _singleton = new DictionaryService ();
+    }
+
+    /**
+     * Creates the singleton instance of the dictionary service.
+     */
+    public static void init (File dictionaryRoot)
+    {
+        _singleton = new DictionaryService (dictionaryRoot);
     }
 
     /**
@@ -61,9 +202,9 @@ public class DictionaryService
     /**
      * Protected constructor.
      */
-    protected DictionaryService () 
+    protected DictionaryService (File dictionaryRoot) 
     {
-        // TODO: set up file paths here
+        _dictionaryRoot = dictionaryRoot;
     }
 
     /**
@@ -90,27 +231,18 @@ public class DictionaryService
         Invoker.Unit work = new Invoker.Unit ("DictionaryService.getLetterSet") {
             public boolean invoke ()
             {
-                if (loadLanguageFile (locale))
+                Dictionary dict = getDictionary (locale);
+                char[] chars = dict.randomLetters (count);
+                StringBuilder sb = new StringBuilder ();
+                for (char c : chars)
                 {
-                    // TODO: actual logic will go here
+                    sb.append (c);
-                    
+                    sb.append (',');
-                    Random r = new Random ();
-                    char[] chars = new char[count];
-                    for (int i = 0; i < count; i++)
-                    {
-                        int c = r.nextInt (24) + 65;
-                        chars[i] = (char) (c);
-                    }
-                    StringBuilder sb = new StringBuilder ();
-                    for (char c : chars)
-                    {
-                        sb.append (c);
-                        sb.append (',');
-                    }
-                    sb.deleteCharAt (sb.length() - 1);
-                        
-                    listener.requestProcessed (sb.toString());
                 }
+                sb.deleteCharAt (sb.length() - 1);
+                
+                listener.requestProcessed (sb.toString());
+            
                 return true;
             }
         };
@@ -129,11 +261,10 @@ public class DictionaryService
         Invoker.Unit work = new Invoker.Unit ("DictionaryService.checkWord") {
             public boolean invoke ()
             {
-                if (loadLanguageFile (locale))
+                Dictionary dict = getDictionary (locale);
-                {
+                boolean result = (dict != null && dict.contains (word));
-                    // TODO: actual logic
+                listener.requestProcessed (result);
-                    listener.requestProcessed (true);
+
-                }
                 return true;
             }
         };
@@ -142,23 +273,46 @@ public class DictionaryService
         CrowdServer.invoker.postUnit (work);    
     }
 
+    
     /**
-     * Loads the language file, if necessary
+     * Retrieves the dictionary object for a given locale.
+     * Forces the dictionary file to be loaded, if it hasn't already.
      */
-    public boolean loadLanguageFile (String locale)
+    private Dictionary getDictionary (String locale)
     {
-        // this is boring :)
+        locale = locale.toLowerCase ();
-        try {
+        if (! _dictionaries.containsKey (locale))
-            Thread.sleep (500);
+        {
-        } catch (InterruptedException e) { }
+            try
-        return true;
+            {
+                // Make a file name
+                String filename = locale + ".wordlist";
+                File file = new File (_dictionaryRoot, filename);
+                _dictionaries.put (locale, new Dictionary (file));
+            }
+            catch (Exception e)
+            {
+                log.log (Level.WARNING, "Failed to load language file", e);
+            }
+        }
+
+        return _dictionaries.get (locale);
     }
 
 
 
+
     // PRIVATE VARIABLES
 
     /** Singleton instance pointer */
     private static DictionaryService _singleton;
 
+    /** Root directory where we find dictionary files */
+    private File _dictionaryRoot;
+
+    /** Map from locale name to Dictionary object */
+    private HashMap <String, Dictionary> _dictionaries = new HashMap <String, Dictionary> ();
+
+ 
+
 }