AfflictionNetworks/vilya
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated the dictionary service, now with Real Functionality :) - loads language-specific

Browse Source 
dictionary files, and acually uses them for user queries.



git-svn-id: svn+ssh://src.earth.threerings.net/vilya/trunk@185 c613c5cb-e716-0410-b11b-feb51c14d237
This commit is contained in:
Robert Zubeck
2007-02-14 00:30:46 +00:00
parent 7ab410dde4
commit 00e771e7a5
1 changed files with 190 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/java/com/threerings/ezgame/server/DictionaryService.java
+182 -28
View File
@@ -28,10 +28,19 @@ import com.threerings.crowd.server.CrowdServer;
import com.threerings.presents.client.InvocationService;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.File;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Random;
import java.util.Set;
import java.util.Vector;
import java.util.logging.Level;
import static com.threerings.ezgame.server.Log.log;
/**
@@ -39,15 +48,147 @@ import static com.threerings.ezgame.server.Log.log;
*
* NOTE: the service supports lazy loading of language files, but does not
* _unload_ them from memory, leading to increasing memory usage.
*
* NOTE: the dictionary service has not yet been tested with language
* files written in non-default character encodings.
*/
public class DictionaryService
{
/**
* Creates the singleton instance of the dictionary service.
* Helper class, encapsulates a sorted array of word hashes,
* which can be used to look up the existence of a word.
*/
private class Dictionary
{
/**
* Constructor, loads up the word list and initializes storage.
* This naive version assumes language files are simple list of words,
* with one word per line.
*/
public Dictionary (File wordfile)
{
try
{
if (wordfile.exists() && wordfile.isFile() && wordfile.canRead())
{
// File length will give us a rough starting point for the word array
long bytecount = wordfile.length();
int approxWords = (int) (bytecount / 5); // just a heuristic,
_hashes.ensureCapacity (approxWords); // we'll trim the vector later
// Read it line by line
BufferedReader reader = new BufferedReader (new FileReader (wordfile));
String line = null;
while ((line = reader.readLine()) != null)
{
// Add the word to the dictionary
int hash = hashWord (line);
_hashes.add (hash);
// Add each letter to a letter set
int len = line.length ();
for (int i = 0; i < len; i++)
{
char ch = line.charAt (i);
if (! _letters.containsKey (ch))
{
_letters.put (ch, 1);
}
else
{
_letters.put (ch, _letters.get (ch) + 1);
}
}
}
// Trim and sort the vector
_hashes.trimToSize();
Collections.sort (_hashes);
log.log (Level.INFO,
"Loaded dictionary file " + wordfile.getName () +
" with " + _hashes.size () + " entries, " +
_letters.size () + " letters.");
}
else
{
log.log (Level.WARNING,
"Could not access dictionary file " + wordfile.getAbsolutePath ());
}
}
catch (Exception ex)
{
log.log (Level.WARNING, "Failed to load dictionary file", ex);
_hashes.clear(); // dump everything
}
}
/** Checks if the specified word exists in the word list */
public boolean contains (String word)
{
// Hash the word and check
int hash = hashWord (word);
int result = Collections.binarySearch (_hashes, hash);
return (result >= 0);
}
/** Gets an array of random letters for the language, with uniform distribution. */
public char[] randomLetters (int count)
{
Set <Character> letterSet = _letters.keySet();
int letterCount = letterSet.size();
Character [] letters = new Character [letterCount];
letterSet.toArray (letters);
char [] results = new char [count];
for (int i = 0; i < count; i++)
{
int r = _random.nextInt (letterCount);
char ch = letters[r];
results[i] = ch;
}
return results;
}
/** Hashes the word, for use in storage */
private int hashWord (String word)
{
return word.toLowerCase().hashCode();
}
// PRIVATE STORAGE
/** Sorted vector of word hashes */
private Vector <Integer> _hashes = new Vector <Integer> ();
/** Mapping from letters in this language to their total count */
private HashMap <Character, Integer> _letters = new HashMap <Character, Integer> ();
/** Random number generator */
private Random _random = new Random ();
}
/**
* Obsolete init function. I'm keeping it around to prevent
* any automated build breakage between the upcoming Vilya checkin
* and the subsequent MSoy checkin. But this will go away real soon.
*/
public static void init ()
{
_singleton = new DictionaryService ();
}
/**
* Creates the singleton instance of the dictionary service.
*/
public static void init (File dictionaryRoot)
{
_singleton = new DictionaryService (dictionaryRoot);
}
/**
@@ -61,9 +202,9 @@ public class DictionaryService
/**
* Protected constructor.
*/
protected DictionaryService ()
protected DictionaryService (File dictionaryRoot)
{
// TODO: set up file paths here
_dictionaryRoot = dictionaryRoot;
}
/**
@@ -90,17 +231,8 @@ public class DictionaryService
Invoker.Unit work = new Invoker.Unit ("DictionaryService.getLetterSet") {
public boolean invoke ()
{
if (loadLanguageFile (locale))
{
// TODO: actual logic will go here
Random r = new Random ();
char[] chars = new char[count];
for (int i = 0; i < count; i++)
{
int c = r.nextInt (24) + 65;
chars[i] = (char) (c);
}
Dictionary dict = getDictionary (locale);
char[] chars = dict.randomLetters (count);
StringBuilder sb = new StringBuilder ();
for (char c : chars)
{
@@ -110,7 +242,7 @@ public class DictionaryService
sb.deleteCharAt (sb.length() - 1);
listener.requestProcessed (sb.toString());
}
return true;
}
};
@@ -129,11 +261,10 @@ public class DictionaryService
Invoker.Unit work = new Invoker.Unit ("DictionaryService.checkWord") {
public boolean invoke ()
{
if (loadLanguageFile (locale))
{
// TODO: actual logic
listener.requestProcessed (true);
}
Dictionary dict = getDictionary (locale);
boolean result = (dict != null && dict.contains (word));
listener.requestProcessed (result);
return true;
}
};
@@ -142,17 +273,32 @@ public class DictionaryService
CrowdServer.invoker.postUnit (work);
}
/**
* Loads the language file, if necessary
* Retrieves the dictionary object for a given locale.
* Forces the dictionary file to be loaded, if it hasn't already.
*/
public boolean loadLanguageFile (String locale)
private Dictionary getDictionary (String locale)
{
// this is boring :)
try {
Thread.sleep (500);
} catch (InterruptedException e) { }
return true;
locale = locale.toLowerCase ();
if (! _dictionaries.containsKey (locale))
{
try
{
// Make a file name
String filename = locale + ".wordlist";
File file = new File (_dictionaryRoot, filename);
_dictionaries.put (locale, new Dictionary (file));
}
catch (Exception e)
{
log.log (Level.WARNING, "Failed to load language file", e);
}
}
return _dictionaries.get (locale);
}
@@ -161,4 +307,12 @@ public class DictionaryService
/** Singleton instance pointer */
private static DictionaryService _singleton;
/** Root directory where we find dictionary files */
private File _dictionaryRoot;
/** Map from locale name to Dictionary object */
private HashMap <String, Dictionary> _dictionaries = new HashMap <String, Dictionary> ();
}