From 9f641c579623bd716c44feeb1dc5ba621d7767a4 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 19 Jul 2002 20:12:23 +0000 Subject: [PATCH] Download resource bundles asynchronously and provide facilities for observing bundle downloading progress. git-svn-id: svn+ssh://src.earth.threerings.net/narya/trunk@1595 542714f4-19e9-0310-aa3c-eee0fc999fb1 --- .../threerings/resource/DownloadManager.java | 338 ++++++++++++++++++ .../threerings/resource/ResourceBundle.java | 58 ++- .../threerings/resource/ResourceManager.java | 236 +++++++----- 3 files changed, 536 insertions(+), 96 deletions(-) create mode 100644 src/java/com/threerings/resource/DownloadManager.java diff --git a/src/java/com/threerings/resource/DownloadManager.java b/src/java/com/threerings/resource/DownloadManager.java new file mode 100644 index 000000000..6e71e2d3f --- /dev/null +++ b/src/java/com/threerings/resource/DownloadManager.java @@ -0,0 +1,338 @@ +// +// $Id: DownloadManager.java,v 1.1 2002/07/19 20:12:23 shaper Exp $ + +package com.threerings.resource; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; + +import java.util.ArrayList; +import java.util.List; + +import com.samskivert.util.Queue; +import com.samskivert.util.StringUtil; + +/** + * Manages the asynchronous downloading of files that are usually located + * on an HTTP server but that may also be located on the local filesystem. + */ +public class DownloadManager +{ + /** + * Provides facilities for notifying an observer of file download + * progress. + */ + public interface DownloadObserver + { + /** + * Called when the download manager is about to check all + * downloads to see whether they are in need of an update. + */ + public void resolvingDownloads (); + + /** + * Called to inform the observer of ongoing progress toward + * completion of the overall downloading task. The caller is + * guaranteed to get at least one call reporting 100% completion. + * + * @param percent the percent completion, in terms of total file + * size, of the download request. + */ + public void downloadProgress (int percent); + + /** + * Called if a failure occurs while checking for an update or + * downloading a file. + * + * @param desc the file that was being downloaded when the error + * occurred, or null if the failure occurred while + * resolving downloads. + * @param e the exception detailing the failure. + */ + public void downloadFailed (DownloadDescriptor desc, Exception e); + } + + /** + * Describes a single file to be downloaded. + */ + public static class DownloadDescriptor + { + /** The URL from which the file is to be downloaded. */ + public URL sourceURL; + + /** The destination file to which the file is to be written. */ + public File destFile; + + /** The last-modified timestamp of the source file. */ + public long lastModified; + + /** The size in bytes of the source files. */ + public long fileSize; + + /** The last-modified timestamp of the destination file. */ + public long destLastModified; + + /** The size in bytes of the destination file. */ + public long destFileSize; + + /** + * Constructs a download descriptor to retrieve the given file + * from the given URL. + */ + public DownloadDescriptor (URL url, File file) + { + this.sourceURL = url; + this.destFile = file; + } + + /** Returns a string representation of this instance. */ + public String toString () + { + return StringUtil.fieldsToString(this); + } + } + + /** + * Downloads the supplied list of descriptors, notifying the given + * download observer of download progress and status. + * + * @param descriptors the list of files to be downloaded. + * @param fragile if true, reports failure and ceases any further + * downloading if an error occurs; else will continue attempting to + * download the remainder of the descriptors. + * @param downloadObs the observer to notify of progress, success and + * failure. + */ + public void download ( + List descriptors, boolean fragile, DownloadObserver downloadObs) + { + // add the download request to the download queue + DownloadRecord dlrec = new DownloadRecord(); + dlrec.descriptors = descriptors; + dlrec.obs = downloadObs; + dlrec.fragile = fragile; + _dlqueue.append(dlrec); + + synchronized (this) { + // if we've not yet got our downloading thread... + if (_dlthread == null) { + // create the thread + _dlthread = new Thread() { + public void run () { + processDownloads(); + } + }; + _dlthread.setDaemon(true); + // and start it going + _dlthread.start(); + } + } + } + + /** + * Called by the download thread to process all download requests in + * the download queue. + */ + protected void processDownloads () + { + // create the data buffer to be used when reading files + _buffer = new byte[BUFFER_SIZE]; + + DownloadRecord dlrec; + while (true) { + synchronized (this) { + // kill the download thread if we have no remaining + // download requests + if (_dlqueue.size() == 0) { + _dlthread = null; + // free up the data buffer + _buffer = null; + return; + } + + // pop download off the queue + dlrec = (DownloadRecord)_dlqueue.getNonBlocking(); + } + + // handle the download request + processDownloadRequest(dlrec.descriptors, dlrec.obs, dlrec.fragile); + } + } + + /** + * Processes a single download request. + */ + protected void processDownloadRequest ( + List descriptors, DownloadObserver obs, boolean fragile) + { + // let the observer know that we're about to resolve all files to + // be downloaded + obs.resolvingDownloads(); + + // check the size and last-modified information for each file to + // ascertain whether our local copy needs to be refreshed + ArrayList fetch = new ArrayList(); + long totalSize = 0; + int size = descriptors.size(); + for (int ii = 0; ii < size; ii++) { + DownloadDescriptor desc = (DownloadDescriptor)descriptors.get(ii); + + try { + // get the source file information + desc.fileSize = 0; + desc.lastModified = 0; + String protocol = desc.sourceURL.getProtocol(); + if (protocol.equals("file")) { + // read the file information directly from the file system + File tfile = new File(desc.sourceURL.getPath()); + desc.fileSize = tfile.length(); + desc.lastModified = tfile.lastModified(); + + } else if (protocol.equals("http")) { + // read the file information via an HTTP HEAD request + HttpURLConnection ucon = (HttpURLConnection) + desc.sourceURL.openConnection(); + ucon.setRequestMethod("HEAD"); + ucon.connect(); + desc.fileSize = ucon.getContentLength(); + desc.lastModified = ucon.getLastModified(); + + } else { + throw new IOException( + "Unknown source file protocol " + + "[protocol=" + protocol + ", desc=" + desc + "]."); + } + + // determine whether we actually need to fetch the file by + // checking to see whether we have a local copy at all, + // and if so, whether its file size or last-modified time + // differs from the source file + desc.destFileSize = desc.destFile.length(); + desc.destLastModified = desc.destFile.lastModified(); + if ((!desc.destFile.exists()) || + (desc.destFileSize != desc.fileSize) || + (desc.destLastModified != desc.lastModified)) { + // increment the total file size to be fetched + totalSize += desc.fileSize; + // add the file to the list of files to be fetched + fetch.add(desc); + Log.debug("File deemed stale " + + "[url=" + desc.sourceURL + "]."); + + } else { + Log.debug("File deemed up-to-date " + + "[url=" + desc.sourceURL + "]."); + } + + } catch (IOException ioe) { + obs.downloadFailed(null, ioe); + if (fragile) { + return; + } + } + } + + // download all stale files + size = fetch.size(); + long currentSize = 0; + boolean complete = false; + for (int ii = 0; ii < size; ii++) { + DownloadDescriptor desc = (DownloadDescriptor)fetch.get(ii); + try { + complete = processDownload(desc, obs, currentSize, totalSize); + currentSize += desc.fileSize; + + } catch (IOException ioe) { + obs.downloadFailed(desc, ioe); + if (fragile) { + return; + } + } + } + + // make sure to always let the observer know that we've wrapped up + // by reporting 100% completion + if (!complete) { + obs.downloadProgress(100); + } + } + + /** + * Processes a single download descriptor. Returns whether the + * download observer was notified of a 100% complete progress update. + */ + protected boolean processDownload ( + DownloadDescriptor desc, DownloadObserver obs, long currentSize, + long totalSize) + throws IOException + { + // download the resource bundle from the specified URL + URLConnection ucon = desc.sourceURL.openConnection(); + + // prepare to read the data from the URL into the cache file + Log.info("Downloading file [url=" + desc.sourceURL + "]."); + InputStream in = ucon.getInputStream(); + FileOutputStream out = new FileOutputStream(desc.destFile); + int read; + boolean complete = false; + + // read in the file data + while ((read = in.read(_buffer)) != -1) { + // write it out to our local copy + out.write(_buffer, 0, read); + + // report our progress to the download observer as a + // percentage of the total file data to be transferred + currentSize += read; + int pctdone = (int)((currentSize / (float)totalSize) * 100f); + complete = (pctdone >= 100); + obs.downloadProgress(pctdone); + } + + // close the streams + in.close(); + out.close(); + + // if we have a last modified time, we want to adjust our cache + // file accordingly + if (desc.lastModified != 0) { + desc.destFile.setLastModified(desc.lastModified); + } + + return complete; + } + + /** + * A record describing a single download request. + */ + protected static class DownloadRecord + { + /** The list of download descriptors to be downloaded. */ + public List descriptors; + + /** The download observer to notify of download progress. */ + public DownloadObserver obs; + + /** Whether to abort downloading if an error occurs. */ + public boolean fragile; + } + + /** The downloading thread. */ + protected Thread _dlthread; + + /** The queue of download requests. */ + protected Queue _dlqueue = new Queue(); + + /** The data buffer used when reading file data. */ + protected byte[] _buffer; + + /** The data buffer size for reading file data. */ + protected static final int BUFFER_SIZE = 2048; +} diff --git a/src/java/com/threerings/resource/ResourceBundle.java b/src/java/com/threerings/resource/ResourceBundle.java index b2d354c6b..7061ba7c6 100644 --- a/src/java/com/threerings/resource/ResourceBundle.java +++ b/src/java/com/threerings/resource/ResourceBundle.java @@ -1,5 +1,5 @@ // -// $Id: ResourceBundle.java,v 1.2 2001/11/20 03:46:28 mdb Exp $ +// $Id: ResourceBundle.java,v 1.3 2002/07/19 20:12:23 shaper Exp $ package com.threerings.resource; @@ -10,6 +10,8 @@ import java.io.InputStream; import java.util.jar.JarEntry; import java.util.jar.JarFile; +import com.samskivert.io.NestableIOException; + /** * A resource bundle provides access to the resources in a jar file. */ @@ -19,14 +21,10 @@ public class ResourceBundle * Constructs a resource bundle with the supplied jar file. * * @param source a file object that references our source jar file. - * - * @exception IOException thrown if an error occurs reading our jar - * file. */ public ResourceBundle (File source) - throws IOException { - _source = new JarFile(source); + _source = source; } /** @@ -45,12 +43,13 @@ public class ResourceBundle public InputStream getResource (String path) throws IOException { + resolveJarFile(); // TBD: determine whether or not we need to convert the path into // a platform-dependent path if we're on Windows - JarEntry entry = _source.getJarEntry(path); + JarEntry entry = _jarSource.getJarEntry(path); InputStream stream = null; if (entry != null) { - stream = _source.getInputStream(entry); + stream = _jarSource.getInputStream(entry); } return stream; } @@ -62,7 +61,12 @@ public class ResourceBundle */ public boolean containsResource (String path) { - return (_source.getJarEntry(path) != null); + try { + resolveJarFile(); + return (_jarSource.getJarEntry(path) != null); + } catch (IOException ioe) { + return false; + } } /** @@ -70,10 +74,40 @@ public class ResourceBundle */ public String toString () { - return "[path=" + _source.getName() + - ", entries=" + _source.size() + "]"; + try { + resolveJarFile(); + return "[path=" + _jarSource.getName() + + ", entries=" + _jarSource.size() + "]"; + } catch (IOException ioe) { + return "[file=" + _source + ", ioe=" + ioe + "]"; + } } + /** + * Creates the internal jar file reference if we've not already got + * it; we do this lazily so as to avoid any jar- or zip-file-related + * antics until and unless doing so is required, and because the + * resource manager would like to be able to create bundles before the + * associated files have been fully downloaded. + */ + protected void resolveJarFile () + throws IOException + { + try { + if (_jarSource == null) { + _jarSource = new JarFile(_source); + } + + } catch (IOException ioe) { + throw new NestableIOException( + "Failed to resolve resource bundle jar file " + + "[file=" + _source + "]", ioe); + } + } + + /** The file from which we construct our jar file. */ + protected File _source; + /** The jar file from which we load resources. */ - protected JarFile _source; + protected JarFile _jarSource; } diff --git a/src/java/com/threerings/resource/ResourceManager.java b/src/java/com/threerings/resource/ResourceManager.java index 008271331..d87e69e9a 100644 --- a/src/java/com/threerings/resource/ResourceManager.java +++ b/src/java/com/threerings/resource/ResourceManager.java @@ -1,5 +1,5 @@ // -// $Id: ResourceManager.java,v 1.13 2002/04/01 20:47:28 mdb Exp $ +// $Id: ResourceManager.java,v 1.14 2002/07/19 20:12:23 shaper Exp $ package com.threerings.resource; @@ -17,12 +17,15 @@ import java.net.URLConnection; import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; +import java.util.List; import java.util.Properties; import java.util.StringTokenizer; -import org.apache.commons.io.StreamUtils; import com.samskivert.util.StringUtil; +import com.threerings.resource.DownloadManager.DownloadDescriptor; +import com.threerings.resource.DownloadManager.DownloadObserver; + /** * The resource manager is responsible for maintaining a repository of * resources that are synchronized with a remote source. This is @@ -38,13 +41,15 @@ import com.samskivert.util.StringUtil; * locate a resource in the default resource set, it falls back to loading * the resource via the classloader (which will search the classpath). * - *

The resource manager must be provided with the URL of a resource - * definition file which describes these resource sets at construct - * time. The definition file will be loaded and the resource bundles - * defined within will be loaded relative to the resource definition URL. - * The bundles will be cached in the user's home directory and only - * reloaded when the source resources have been updated. The resource - * definition file looks something like the following: + *

Applications that wish to make use of resource sets and their + * associated bundles must call {@link #initBundles} after constructing + * the resource manager, providing the URL of a resource definition file + * which describes these resource sets. The definition file will be loaded + * and the resource bundles defined within will be loaded relative to the + * resource definition URL. The bundles will be cached in the user's home + * directory and only reloaded when the source resources have been + * updated. The resource definition file looks something like the + * following: * * 

  * resource.set.default = sets/misc/config.jar: \
@@ -71,11 +76,35 @@ import com.samskivert.util.StringUtil;
 public class ResourceManager
 {
     /**
-     * Constructs a resource manager which will load resources as
-     * specified in the configuration file, the path to which is supplied
-     * via configPath. If resource sets are not needed and
-     * resources will only be loaded via the classpath, null may be passed
-     * in resourceURL and configPath.
+     * Provides facilities for notifying an observer of resource bundle
+     * download progress.
+     */
+    public interface BundleDownloadObserver
+    {
+        /**
+         * Called when the resource manager is about to check for an
+         * update of any of our resource sets.
+         */
+        public void checkingForUpdate ();
+
+        /**
+         * Called to inform the observer of ongoing progress toward
+         * completion of the overall bundle downloading task.  The caller
+         * is guaranteed to get at least one call reporting 100%
+         * completion.
+         */
+        public void downloadProgress (int percent);
+
+        /**
+         * Called if a failure occurs while checking for an update or
+         * downloading all resource sets.
+         */
+        public void downloadFailed (Exception e);
+    }
+
+    /**
+     * Constructs a resource manager which will load resources via the
+     * classloader, prepending resourceRoot to their path.
      *
      * @param resourceRoot the path to prepend to resource paths prior to
      * attempting to load them via the classloader. When resources are
@@ -85,15 +114,8 @@ public class ResourceManager
      * to isolate them from the rest of the files in the classpath. This
      * is not a platform dependent path (forward slash is always used to
      * separate path elements).
-     * @param resourceURL the base URL from which resources are loaded.
-     * Relative paths specified in the resource definition file will be
-     * loaded relative to this path. If this is null, the system property
-     * resource_url will be used, if available.
-     * @param configPath the path (relative to the resource URL) of the
-     * resource definition file.
      */
-    public ResourceManager (
-        String resourceRoot, String resourceURL, String configPath)
+    public ResourceManager (String resourceRoot)
     {
         // keep track of our root path
         _rootPath = resourceRoot;
@@ -107,7 +129,28 @@ public class ResourceManager
 
         // use the classloader that loaded us
         _loader = getClass().getClassLoader();
+    }
 
+    /**
+     * Initializes the bundle sets to be made available by this resource
+     * manager.  Applications that wish to make use of resource bundles
+     * should call this method after constructing the resource manager.
+     *
+     * @param resourceURL the base URL from which resources are loaded.
+     * Relative paths specified in the resource definition file will be
+     * loaded relative to this path. If this is null, the system property
+     * resource_url will be used, if available.
+     * @param configPath the path (relative to the resource URL) of the
+     * resource definition file.
+     * @param downloadObs the bundle download observer to notify of
+     * download progress and success or failure, or null if
+     * the caller doesn't care to be informed; note that in the latter
+     * case, the calling thread will block until bundle updating is
+     * complete.
+     */
+    public void initBundles (String resourceURL, String configPath,
+                             BundleDownloadObserver downloadObs)
+    {
         // if the resource URL wasn't provided, we try to figure it out
         // for ourselves
         if (resourceURL == null) {
@@ -142,6 +185,7 @@ public class ResourceManager
         Properties config = loadConfig(rurl, configPath);
 
         // resolve the configured resource sets
+        ArrayList dlist = new ArrayList();
         Enumeration names = config.propertyNames();
         while (names.hasMoreElements()) {
             String key = (String)names.nextElement();
@@ -149,8 +193,87 @@ public class ResourceManager
                 continue;
             }
             String setName = key.substring(RESOURCE_SET_PREFIX.length());
-            resolveResourceSet(rurl, setName, config.getProperty(key));
+            resolveResourceSet(rurl, setName, config.getProperty(key), dlist);
         }
+
+        // start the download, blocking if we've no observer
+        DownloadManager dlmgr = new DownloadManager();
+        if (downloadObs == null) {
+            downloadBlocking(dlmgr, dlist);
+        } else {
+            downloadNonBlocking(dlmgr, dlist, downloadObs);
+        }
+    }
+
+    /**
+     * Downloads the files in the supplied download list, blocking the
+     * calling thread until the download is complete or a failure has
+     * occurred.
+     */
+    protected void downloadBlocking (DownloadManager dlmgr, List dlist)
+    {
+        // create an object to wait on while the download takes place
+        final Object lock = new Object();
+
+        // pass the descriptors on to the download manager
+        dlmgr.download(dlist, true, new DownloadObserver() {
+            public void resolvingDownloads () {
+                // nothing for now
+            }
+
+            public void downloadProgress (int percent) {
+                if (percent == 100) {
+                    synchronized (lock) {
+                        // wake things up as the download is finished
+                        lock.notify();
+                    }
+                }
+            }
+
+            public void downloadFailed (DownloadDescriptor desc, Exception e) {
+                Log.warning("Failed to download file " +
+                            "[desc=" + desc + ", e=" + e + "].");
+                synchronized (lock) {
+                    // wake things up since we're fragile and so a
+                    // single failure means all is booched
+                    lock.notify();
+                }
+            }
+        });
+
+        try {
+            synchronized (lock) {
+                // block until the download has completed
+                lock.wait();
+            }
+
+        } catch (InterruptedException ie) {
+            Log.warning("Thread interrupted while waiting for download " +
+                        "to complete [ie=" + ie + "].");
+        }
+    }
+
+    /**
+     * Downloads the files in the supplied download list asynchronously,
+     * notifying the download observer of ongoing progress.
+     */
+    protected void downloadNonBlocking (
+        DownloadManager dlmgr, List dlist, final BundleDownloadObserver obs)
+    {
+        // pass the descriptors on to the download manager
+        dlmgr.download(dlist, true, new DownloadObserver() {
+            public void resolvingDownloads () {
+                obs.checkingForUpdate();
+            }
+
+            public void downloadProgress (int percent) {
+                obs.downloadProgress(percent);
+            }
+
+            public void downloadFailed (DownloadDescriptor desc, Exception e) {
+                obs.downloadFailed(e);
+            }
+        });
     }
 
     /**
@@ -262,7 +385,7 @@ public class ResourceManager
      * information.
      */
     protected void resolveResourceSet (
-        URL resourceURL, String setName, String definition)
+        URL resourceURL, String setName, String definition, List dlist)
     {
         StringTokenizer tok = new StringTokenizer(definition, ":");
         ArrayList set = new ArrayList();
@@ -280,73 +403,18 @@ public class ResourceManager
                 // compute the path to the cache file for this bundle
                 File cfile = new File(genCachePath(setName, path));
 
-                // download the resource bundle from the specified URL
-                URLConnection ucon = burl.openConnection();
-                boolean readData = true;
+                // slap this on the list for retrieval or update by the
+                // download manager
+                dlist.add(new DownloadDescriptor(burl, cfile));
 
-                // set the last-modified time we're looking for
-                long lastModified = 0;
-                if (cfile.exists()) {
-                    lastModified = cfile.lastModified();
-                    ucon.setIfModifiedSince(lastModified);
-                }
-
-                // connect the URL
-                ucon.connect();
-
-                // if this is an HTTP connection, we want to use
-                // if-modified-since
-                if (lastModified != 0) {
-                    if (ucon instanceof HttpURLConnection) {
-                        HttpURLConnection hucon = (HttpURLConnection)ucon;
-                        readData = (hucon.getResponseCode() !=
-                                    HttpURLConnection.HTTP_NOT_MODIFIED);
-
-                    } else if (burl.getProtocol().equals("file")) {
-                        // do some jockeying for file: URLs to determine
-                        // whether or not the data is newer
-                        File tfile = new File(burl.getPath());
-                        readData = (tfile.lastModified() > lastModified);
-                    }
-                }
-
-                // if this is a URL request, we want to keep track of the
-                // last modified time
-                if (ucon instanceof HttpURLConnection) {
-                    HttpURLConnection hucon = (HttpURLConnection)ucon;
-                    lastModified = hucon.getLastModified();
-                } else {
-                    lastModified = 0;
-                }
-
-                // read the data from the URL into the cache file
-                if (readData) {
-                    Log.info("Downloading bundle [url=" + burl + "].");
-                    InputStream in = ucon.getInputStream();
-                    FileOutputStream out = new FileOutputStream(cfile);
-                    // pipe the input stream into the output stream
-                    StreamUtils.pipe(in, out);
-                    in.close();
-                    out.close();
-                    // if we have a last modified time, we want to adjust
-                    // our cache file accordingly
-                    if (lastModified != 0) {
-                        cfile.setLastModified(lastModified);
-                    }
-                }
-
-                // finally add this newly cached file to the set as a
-                // resource bundle
+                // finally, add the file that will be cached to the set as
+                // a resource bundle
                 set.add(new ResourceBundle(cfile));
 
             } catch (MalformedURLException mue) {
                 Log.warning("Unable to create URL for resource " +
                             "[set=" + setName + ", path=" + path +
                             ", error=" + mue + "].");
-
-            } catch (IOException ioe) {
-                Log.warning("Error processing resource set entry " +
-                            "[url=" + burl + ", error=" + ioe + "].");
             }
         }