Download resource bundles asynchronously and provide facilities for

observing bundle downloading progress.


git-svn-id: svn+ssh://src.earth.threerings.net/narya/trunk@1595 542714f4-19e9-0310-aa3c-eee0fc999fb1

src/java/com/threerings/resource/DownloadManager.java

@@ -0,0 +1,338 @@
+//
+// $Id: DownloadManager.java,v 1.1 2002/07/19 20:12:23 shaper Exp $
+
+package com.threerings.resource;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.samskivert.util.Queue;
+import com.samskivert.util.StringUtil;
+
+/**
+ * Manages the asynchronous downloading of files that are usually located
+ * on an HTTP server but that may also be located on the local filesystem.
+ */
+public class DownloadManager
+{
+    /**
+     * Provides facilities for notifying an observer of file download
+     * progress.
+     */
+    public interface DownloadObserver
+    {
+        /**
+         * Called when the download manager is about to check all
+         * downloads to see whether they are in need of an update.
+         */
+        public void resolvingDownloads ();
+
+        /**
+         * Called to inform the observer of ongoing progress toward
+         * completion of the overall downloading task.  The caller is
+         * guaranteed to get at least one call reporting 100% completion.
+         *
+         * @param percent the percent completion, in terms of total file
+         * size, of the download request.
+         */
+        public void downloadProgress (int percent);
+
+        /**
+         * Called if a failure occurs while checking for an update or
+         * downloading a file.
+         *
+         * @param desc the file that was being downloaded when the error
+         * occurred, or <code>null</code> if the failure occurred while
+         * resolving downloads.
+         * @param e the exception detailing the failure.
+         */
+        public void downloadFailed (DownloadDescriptor desc, Exception e);
+    }
+
+    /**
+     * Describes a single file to be downloaded.
+     */
+    public static class DownloadDescriptor
+    {
+        /** The URL from which the file is to be downloaded. */
+        public URL sourceURL;
+
+        /** The destination file to which the file is to be written. */
+        public File destFile;
+
+        /** The last-modified timestamp of the source file. */
+        public long lastModified;
+
+        /** The size in bytes of the source files. */
+        public long fileSize;
+
+        /** The last-modified timestamp of the destination file. */
+        public long destLastModified;
+
+        /** The size in bytes of the destination file. */
+        public long destFileSize;
+
+        /**
+         * Constructs a download descriptor to retrieve the given file
+         * from the given URL.
+         */
+        public DownloadDescriptor (URL url, File file)
+        {
+            this.sourceURL = url;
+            this.destFile = file;
+        }
+
+        /** Returns a string representation of this instance. */
+        public String toString ()
+        {
+            return StringUtil.fieldsToString(this);
+        }
+    }
+
+    /**
+     * Downloads the supplied list of descriptors, notifying the given
+     * download observer of download progress and status.
+     *
+     * @param descriptors the list of files to be downloaded.
+     * @param fragile if true, reports failure and ceases any further
+     * downloading if an error occurs; else will continue attempting to
+     * download the remainder of the descriptors.
+     * @param downloadObs the observer to notify of progress, success and
+     * failure.
+     */
+    public void download (
+        List descriptors, boolean fragile, DownloadObserver downloadObs)
+    {
+        // add the download request to the download queue
+        DownloadRecord dlrec = new DownloadRecord();
+        dlrec.descriptors = descriptors;
+        dlrec.obs = downloadObs;
+        dlrec.fragile = fragile;
+        _dlqueue.append(dlrec);
+
+        synchronized (this) {
+            // if we've not yet got our downloading thread...
+            if (_dlthread == null) {
+                // create the thread
+                _dlthread = new Thread() {
+                    public void run () {
+                        processDownloads();
+                    }
+                };
+                _dlthread.setDaemon(true);
+                // and start it going
+                _dlthread.start();
+            }
+        }
+    }
+
+    /**
+     * Called by the download thread to process all download requests in
+     * the download queue.
+     */
+    protected void processDownloads ()
+    {
+        // create the data buffer to be used when reading files
+        _buffer = new byte[BUFFER_SIZE];
+
+        DownloadRecord dlrec;
+        while (true) {
+            synchronized (this) {
+                // kill the download thread if we have no remaining
+                // download requests
+                if (_dlqueue.size() == 0) {
+                    _dlthread = null;
+                    // free up the data buffer
+                    _buffer = null;
+                    return;
+                }
+
+                // pop download off the queue
+                dlrec = (DownloadRecord)_dlqueue.getNonBlocking();
+            }
+
+            // handle the download request
+            processDownloadRequest(dlrec.descriptors, dlrec.obs, dlrec.fragile);
+        }
+    }
+
+    /**
+     * Processes a single download request.
+     */
+    protected void processDownloadRequest (
+        List descriptors, DownloadObserver obs, boolean fragile)
+    {
+        // let the observer know that we're about to resolve all files to
+        // be downloaded
+        obs.resolvingDownloads();
+
+        // check the size and last-modified information for each file to
+        // ascertain whether our local copy needs to be refreshed
+        ArrayList fetch = new ArrayList();
+        long totalSize = 0;
+        int size = descriptors.size();
+        for (int ii = 0; ii < size; ii++) {
+            DownloadDescriptor desc = (DownloadDescriptor)descriptors.get(ii);
+
+            try {
+                // get the source file information
+                desc.fileSize = 0;
+                desc.lastModified = 0;
+                String protocol = desc.sourceURL.getProtocol();
+                if (protocol.equals("file")) {
+                    // read the file information directly from the file system
+                    File tfile = new File(desc.sourceURL.getPath());
+                    desc.fileSize = tfile.length();
+                    desc.lastModified = tfile.lastModified();
+
+                } else if (protocol.equals("http")) {
+                    // read the file information via an HTTP HEAD request
+                    HttpURLConnection ucon = (HttpURLConnection)
+                        desc.sourceURL.openConnection();
+                    ucon.setRequestMethod("HEAD");
+                    ucon.connect();
+                    desc.fileSize = ucon.getContentLength();
+                    desc.lastModified = ucon.getLastModified();
+
+                } else {
+                    throw new IOException(
+                        "Unknown source file protocol " +
+                        "[protocol=" + protocol + ", desc=" + desc + "].");
+                }
+
+                // determine whether we actually need to fetch the file by
+                // checking to see whether we have a local copy at all,
+                // and if so, whether its file size or last-modified time
+                // differs from the source file
+                desc.destFileSize = desc.destFile.length();
+                desc.destLastModified = desc.destFile.lastModified();
+                if ((!desc.destFile.exists()) ||
+                    (desc.destFileSize != desc.fileSize) ||
+                    (desc.destLastModified != desc.lastModified)) {
+                    // increment the total file size to be fetched
+                    totalSize += desc.fileSize;
+                    // add the file to the list of files to be fetched
+                    fetch.add(desc);
+                    Log.debug("File deemed stale " +
+                              "[url=" + desc.sourceURL + "].");
+
+                } else {
+                    Log.debug("File deemed up-to-date " +
+                              "[url=" + desc.sourceURL + "].");
+                }
+
+            } catch (IOException ioe) {
+                obs.downloadFailed(null, ioe);
+                if (fragile) {
+                    return;
+                }
+            }
+        }
+
+        // download all stale files
+        size = fetch.size();
+        long currentSize = 0;
+        boolean complete = false;
+        for (int ii = 0; ii < size; ii++) {
+            DownloadDescriptor desc = (DownloadDescriptor)fetch.get(ii);
+            try {
+                complete = processDownload(desc, obs, currentSize, totalSize);
+                currentSize += desc.fileSize;
+
+            } catch (IOException ioe) {
+                obs.downloadFailed(desc, ioe);
+                if (fragile) {
+                    return;
+                }
+            }
+        }
+
+        // make sure to always let the observer know that we've wrapped up
+        // by reporting 100% completion
+        if (!complete) {
+            obs.downloadProgress(100);
+        }
+    }
+
+    /**
+     * Processes a single download descriptor.  Returns whether the
+     * download observer was notified of a 100% complete progress update.
+     */
+    protected boolean processDownload (
+        DownloadDescriptor desc, DownloadObserver obs, long currentSize,
+        long totalSize)
+        throws IOException
+    {
+        // download the resource bundle from the specified URL
+        URLConnection ucon = desc.sourceURL.openConnection();
+
+        // prepare to read the data from the URL into the cache file
+        Log.info("Downloading file [url=" + desc.sourceURL + "].");
+        InputStream in = ucon.getInputStream();
+        FileOutputStream out = new FileOutputStream(desc.destFile);
+        int read;
+        boolean complete = false;
+
+        // read in the file data
+        while ((read = in.read(_buffer)) != -1) {
+            // write it out to our local copy
+            out.write(_buffer, 0, read);
+
+            // report our progress to the download observer as a
+            // percentage of the total file data to be transferred
+            currentSize += read;
+            int pctdone = (int)((currentSize / (float)totalSize) * 100f);
+            complete = (pctdone >= 100);
+            obs.downloadProgress(pctdone);
+        }
+
+        // close the streams
+        in.close();
+        out.close();
+
+        // if we have a last modified time, we want to adjust our cache
+        // file accordingly
+        if (desc.lastModified != 0) {
+            desc.destFile.setLastModified(desc.lastModified);
+        }
+
+        return complete;
+    }
+
+    /**
+     * A record describing a single download request.
+     */
+    protected static class DownloadRecord
+    {
+        /** The list of download descriptors to be downloaded. */
+        public List descriptors;
+
+        /** The download observer to notify of download progress. */
+        public DownloadObserver obs;
+
+        /** Whether to abort downloading if an error occurs. */
+        public boolean fragile;
+    }
+
+    /** The downloading thread. */
+    protected Thread _dlthread;
+
+    /** The queue of download requests. */
+    protected Queue _dlqueue = new Queue();
+
+    /** The data buffer used when reading file data. */
+    protected byte[] _buffer;
+
+    /** The data buffer size for reading file data. */
+    protected static final int BUFFER_SIZE = 2048;
+}

src/java/com/threerings/resource/ResourceBundle.java

@@ -1,5 +1,5 @@
 //
-// $Id: ResourceBundle.java,v 1.2 2001/11/20 03:46:28 mdb Exp $
+// $Id: ResourceBundle.java,v 1.3 2002/07/19 20:12:23 shaper Exp $
 
 package com.threerings.resource;
 
@@ -10,6 +10,8 @@ import java.io.InputStream;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 
+import com.samskivert.io.NestableIOException;
+
 /**
  * A resource bundle provides access to the resources in a jar file.
  */
@@ -19,14 +21,10 @@ public class ResourceBundle
      * Constructs a resource bundle with the supplied jar file.
      *
      * @param source a file object that references our source jar file.
-     *
-     * @exception IOException thrown if an error occurs reading our jar
-     * file.
      */
     public ResourceBundle (File source)
-        throws IOException
     {
-        _source = new JarFile(source);
+        _source = source;
     }
 
     /**
@@ -45,12 +43,13 @@ public class ResourceBundle
     public InputStream getResource (String path)
         throws IOException
     {
+        resolveJarFile();
         // TBD: determine whether or not we need to convert the path into
         // a platform-dependent path if we're on Windows
-        JarEntry entry = _source.getJarEntry(path);
+        JarEntry entry = _jarSource.getJarEntry(path);
         InputStream stream = null;
         if (entry != null) {
-            stream = _source.getInputStream(entry);
+            stream = _jarSource.getInputStream(entry);
         }
         return stream;
     }
@@ -62,7 +61,12 @@ public class ResourceBundle
      */
     public boolean containsResource (String path)
     {
-        return (_source.getJarEntry(path) != null);
+        try {
+            resolveJarFile();
+            return (_jarSource.getJarEntry(path) != null);
+        } catch (IOException ioe) {
+            return false;
+        }
     }
 
     /**
@@ -70,10 +74,40 @@ public class ResourceBundle
      */
     public String toString ()
     {
-        return "[path=" + _source.getName() +
+        try {
-            ", entries=" + _source.size() + "]";
+            resolveJarFile();
+            return "[path=" + _jarSource.getName() +
+                ", entries=" + _jarSource.size() + "]";
+        } catch (IOException ioe) {
+            return "[file=" + _source + ", ioe=" + ioe + "]";
+        }
     }
 
+    /**
+     * Creates the internal jar file reference if we've not already got
+     * it; we do this lazily so as to avoid any jar- or zip-file-related
+     * antics until and unless doing so is required, and because the
+     * resource manager would like to be able to create bundles before the
+     * associated files have been fully downloaded.
+     */
+    protected void resolveJarFile ()
+        throws IOException
+    {
+        try {
+            if (_jarSource == null) {
+                _jarSource = new JarFile(_source);
+            }
+
+        } catch (IOException ioe) {
+            throw new NestableIOException(
+                "Failed to resolve resource bundle jar file " +
+                "[file=" + _source + "]", ioe);
+        }
+    }
+
+    /** The file from which we construct our jar file. */
+    protected File _source;
+
     /** The jar file from which we load resources. */
-    protected JarFile _source;
+    protected JarFile _jarSource;
 }

src/java/com/threerings/resource/ResourceManager.java

@@ -1,5 +1,5 @@
 //
-// $Id: ResourceManager.java,v 1.13 2002/04/01 20:47:28 mdb Exp $
+// $Id: ResourceManager.java,v 1.14 2002/07/19 20:12:23 shaper Exp $
 
 package com.threerings.resource;
 
@@ -17,12 +17,15 @@ import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Properties;
 import java.util.StringTokenizer;
 
-import org.apache.commons.io.StreamUtils;
 import com.samskivert.util.StringUtil;
 
+import com.threerings.resource.DownloadManager.DownloadDescriptor;
+import com.threerings.resource.DownloadManager.DownloadObserver;
+
 /**
  * The resource manager is responsible for maintaining a repository of
  * resources that are synchronized with a remote source. This is
@@ -38,13 +41,15 @@ import com.samskivert.util.StringUtil;
  * locate a resource in the default resource set, it falls back to loading
  * the resource via the classloader (which will search the classpath).
  *
- * <p> The resource manager must be provided with the URL of a resource
+ * <p> Applications that wish to make use of resource sets and their
- * definition file which describes these resource sets at construct
+ * associated bundles must call {@link #initBundles} after constructing
- * time. The definition file will be loaded and the resource bundles
+ * the resource manager, providing the URL of a resource definition file
- * defined within will be loaded relative to the resource definition URL.
+ * which describes these resource sets. The definition file will be loaded
- * The bundles will be cached in the user's home directory and only
+ * and the resource bundles defined within will be loaded relative to the
- * reloaded when the source resources have been updated. The resource
+ * resource definition URL.  The bundles will be cached in the user's home
- * definition file looks something like the following:
+ * directory and only reloaded when the source resources have been
+ * updated. The resource definition file looks something like the
+ * following:
  *
  * <pre>
  * resource.set.default = sets/misc/config.jar: \
@@ -71,11 +76,35 @@ import com.samskivert.util.StringUtil;
 public class ResourceManager
 {
     /**
-     * Constructs a resource manager which will load resources as
+     * Provides facilities for notifying an observer of resource bundle
-     * specified in the configuration file, the path to which is supplied
+     * download progress.
-     * via <code>configPath</code>. If resource sets are not needed and
+     */
-     * resources will only be loaded via the classpath, null may be passed
+    public interface BundleDownloadObserver
-     * in <code>resourceURL</code> and <code>configPath</code>.
+    {
+        /**
+         * Called when the resource manager is about to check for an
+         * update of any of our resource sets.
+         */
+        public void checkingForUpdate ();
+
+        /**
+         * Called to inform the observer of ongoing progress toward
+         * completion of the overall bundle downloading task.  The caller
+         * is guaranteed to get at least one call reporting 100%
+         * completion.
+         */
+        public void downloadProgress (int percent);
+
+        /**
+         * Called if a failure occurs while checking for an update or
+         * downloading all resource sets.
+         */
+        public void downloadFailed (Exception e);
+    }
+
+    /**
+     * Constructs a resource manager which will load resources via the
+     * classloader, prepending <code>resourceRoot</code> to their path.
      *
      * @param resourceRoot the path to prepend to resource paths prior to
      * attempting to load them via the classloader. When resources are
@@ -85,15 +114,8 @@ public class ResourceManager
      * to isolate them from the rest of the files in the classpath. This
      * is not a platform dependent path (forward slash is always used to
      * separate path elements).
-     * @param resourceURL the base URL from which resources are loaded.
-     * Relative paths specified in the resource definition file will be
-     * loaded relative to this path. If this is null, the system property
-     * <code>resource_url</code> will be used, if available.
-     * @param configPath the path (relative to the resource URL) of the
-     * resource definition file.
      */
-    public ResourceManager (
+    public ResourceManager (String resourceRoot)
-        String resourceRoot, String resourceURL, String configPath)
     {
         // keep track of our root path
         _rootPath = resourceRoot;
@@ -107,7 +129,28 @@ public class ResourceManager
 
         // use the classloader that loaded us
         _loader = getClass().getClassLoader();
+    }
 
+    /**
+     * Initializes the bundle sets to be made available by this resource
+     * manager.  Applications that wish to make use of resource bundles
+     * should call this method after constructing the resource manager.
+     *
+     * @param resourceURL the base URL from which resources are loaded.
+     * Relative paths specified in the resource definition file will be
+     * loaded relative to this path. If this is null, the system property
+     * <code>resource_url</code> will be used, if available.
+     * @param configPath the path (relative to the resource URL) of the
+     * resource definition file.
+     * @param downloadObs the bundle download observer to notify of
+     * download progress and success or failure, or <code>null</code> if
+     * the caller doesn't care to be informed; note that in the latter
+     * case, the calling thread will block until bundle updating is
+     * complete.
+     */
+    public void initBundles (String resourceURL, String configPath,
+                             BundleDownloadObserver downloadObs)
+    {
         // if the resource URL wasn't provided, we try to figure it out
         // for ourselves
         if (resourceURL == null) {
@@ -142,6 +185,7 @@ public class ResourceManager
         Properties config = loadConfig(rurl, configPath);
 
         // resolve the configured resource sets
+        ArrayList dlist = new ArrayList();
         Enumeration names = config.propertyNames();
         while (names.hasMoreElements()) {
             String key = (String)names.nextElement();
@@ -149,8 +193,87 @@ public class ResourceManager
                 continue;
             }
             String setName = key.substring(RESOURCE_SET_PREFIX.length());
-            resolveResourceSet(rurl, setName, config.getProperty(key));
+            resolveResourceSet(rurl, setName, config.getProperty(key), dlist);
         }
+
+        // start the download, blocking if we've no observer
+        DownloadManager dlmgr = new DownloadManager();
+        if (downloadObs == null) {
+            downloadBlocking(dlmgr, dlist);
+        } else {
+            downloadNonBlocking(dlmgr, dlist, downloadObs);
+        }
+    }
+
+    /**
+     * Downloads the files in the supplied download list, blocking the
+     * calling thread until the download is complete or a failure has
+     * occurred.
+     */
+    protected void downloadBlocking (DownloadManager dlmgr, List dlist)
+    {
+        // create an object to wait on while the download takes place
+        final Object lock = new Object();
+
+        // pass the descriptors on to the download manager
+        dlmgr.download(dlist, true, new DownloadObserver() {
+            public void resolvingDownloads () {
+                // nothing for now
+            }
+
+            public void downloadProgress (int percent) {
+                if (percent == 100) {
+                    synchronized (lock) {
+                        // wake things up as the download is finished
+                        lock.notify();
+                    }
+                }
+            }
+
+            public void downloadFailed (DownloadDescriptor desc, Exception e) {
+                Log.warning("Failed to download file " +
+                            "[desc=" + desc + ", e=" + e + "].");
+                synchronized (lock) {
+                    // wake things up since we're fragile and so a
+                    // single failure means all is booched
+                    lock.notify();
+                }
+            }
+        });
+
+        try {
+            synchronized (lock) {
+                // block until the download has completed
+                lock.wait();
+            }
+
+        } catch (InterruptedException ie) {
+            Log.warning("Thread interrupted while waiting for download " +
+                        "to complete [ie=" + ie + "].");
+        }
+    }
+
+    /**
+     * Downloads the files in the supplied download list asynchronously,
+     * notifying the download observer of ongoing progress.
+     */
+    protected void downloadNonBlocking (
+        DownloadManager dlmgr, List dlist, final BundleDownloadObserver obs)
+    {
+        // pass the descriptors on to the download manager
+        dlmgr.download(dlist, true, new DownloadObserver() {
+            public void resolvingDownloads () {
+                obs.checkingForUpdate();
+            }
+
+            public void downloadProgress (int percent) {
+                obs.downloadProgress(percent);
+            }
+
+            public void downloadFailed (DownloadDescriptor desc, Exception e) {
+                obs.downloadFailed(e);
+            }
+        });
     }
 
     /**
@@ -262,7 +385,7 @@ public class ResourceManager
      * information.
      */
     protected void resolveResourceSet (
-        URL resourceURL, String setName, String definition)
+        URL resourceURL, String setName, String definition, List dlist)
     {
         StringTokenizer tok = new StringTokenizer(definition, ":");
         ArrayList set = new ArrayList();
@@ -280,73 +403,18 @@ public class ResourceManager
                 // compute the path to the cache file for this bundle
                 File cfile = new File(genCachePath(setName, path));
 
-                // download the resource bundle from the specified URL
+                // slap this on the list for retrieval or update by the
-                URLConnection ucon = burl.openConnection();
+                // download manager
-                boolean readData = true;
+                dlist.add(new DownloadDescriptor(burl, cfile));
 
-                // set the last-modified time we're looking for
+                // finally, add the file that will be cached to the set as
-                long lastModified = 0;
+                // a resource bundle
-                if (cfile.exists()) {
-                    lastModified = cfile.lastModified();
-                    ucon.setIfModifiedSince(lastModified);
-                }
-
-                // connect the URL
-                ucon.connect();
-
-                // if this is an HTTP connection, we want to use
-                // if-modified-since
-                if (lastModified != 0) {
-                    if (ucon instanceof HttpURLConnection) {
-                        HttpURLConnection hucon = (HttpURLConnection)ucon;
-                        readData = (hucon.getResponseCode() !=
-                                    HttpURLConnection.HTTP_NOT_MODIFIED);
-
-                    } else if (burl.getProtocol().equals("file")) {
-                        // do some jockeying for file: URLs to determine
-                        // whether or not the data is newer
-                        File tfile = new File(burl.getPath());
-                        readData = (tfile.lastModified() > lastModified);
-                    }
-                }
-
-                // if this is a URL request, we want to keep track of the
-                // last modified time
-                if (ucon instanceof HttpURLConnection) {
-                    HttpURLConnection hucon = (HttpURLConnection)ucon;
-                    lastModified = hucon.getLastModified();
-                } else {
-                    lastModified = 0;
-                }
-
-                // read the data from the URL into the cache file
-                if (readData) {
-                    Log.info("Downloading bundle [url=" + burl + "].");
-                    InputStream in = ucon.getInputStream();
-                    FileOutputStream out = new FileOutputStream(cfile);
-                    // pipe the input stream into the output stream
-                    StreamUtils.pipe(in, out);
-                    in.close();
-                    out.close();
-                    // if we have a last modified time, we want to adjust
-                    // our cache file accordingly
-                    if (lastModified != 0) {
-                        cfile.setLastModified(lastModified);
-                    }
-                }
-
-                // finally add this newly cached file to the set as a
-                // resource bundle
                 set.add(new ResourceBundle(cfile));
 
             } catch (MalformedURLException mue) {
                 Log.warning("Unable to create URL for resource " +
                             "[set=" + setName + ", path=" + path +
                             ", error=" + mue + "].");
-
-            } catch (IOException ioe) {
-                Log.warning("Error processing resource set entry " +
-                            "[url=" + burl + ", error=" + ioe + "].");
             }
         }