AfflictionNetworks/narya
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Download resource bundles asynchronously and provide facilities for

Browse Source 
observing bundle downloading progress.


git-svn-id: svn+ssh://src.earth.threerings.net/narya/trunk@1595 542714f4-19e9-0310-aa3c-eee0fc999fb1
This commit is contained in:
Walter Korman
2002-07-19 20:12:23 +00:00
parent 6d8f304aa3
commit 9f641c5796
3 changed files with 536 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/java/com/threerings/resource/DownloadManager.java
+338
View File
@@ -0,0 +1,338 @@
//
// $Id: DownloadManager.java,v 1.1 2002/07/19 20:12:23 shaper Exp $
package com.threerings.resource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import com.samskivert.util.Queue;
import com.samskivert.util.StringUtil;
/**
* Manages the asynchronous downloading of files that are usually located
* on an HTTP server but that may also be located on the local filesystem.
*/
public class DownloadManager
{
/**
* Provides facilities for notifying an observer of file download
* progress.
*/
public interface DownloadObserver
{
/**
* Called when the download manager is about to check all
* downloads to see whether they are in need of an update.
*/
public void resolvingDownloads ();
/**
* Called to inform the observer of ongoing progress toward
* completion of the overall downloading task. The caller is
* guaranteed to get at least one call reporting 100% completion.
*
* @param percent the percent completion, in terms of total file
* size, of the download request.
*/
public void downloadProgress (int percent);
/**
* Called if a failure occurs while checking for an update or
* downloading a file.
*
* @param desc the file that was being downloaded when the error
* occurred, or <code>null</code> if the failure occurred while
* resolving downloads.
* @param e the exception detailing the failure.
*/
public void downloadFailed (DownloadDescriptor desc, Exception e);
}
/**
* Describes a single file to be downloaded.
*/
public static class DownloadDescriptor
{
/** The URL from which the file is to be downloaded. */
public URL sourceURL;
/** The destination file to which the file is to be written. */
public File destFile;
/** The last-modified timestamp of the source file. */
public long lastModified;
/** The size in bytes of the source files. */
public long fileSize;
/** The last-modified timestamp of the destination file. */
public long destLastModified;
/** The size in bytes of the destination file. */
public long destFileSize;
/**
* Constructs a download descriptor to retrieve the given file
* from the given URL.
*/
public DownloadDescriptor (URL url, File file)
{
this.sourceURL = url;
this.destFile = file;
}
/** Returns a string representation of this instance. */
public String toString ()
{
return StringUtil.fieldsToString(this);
}
}
/**
* Downloads the supplied list of descriptors, notifying the given
* download observer of download progress and status.
*
* @param descriptors the list of files to be downloaded.
* @param fragile if true, reports failure and ceases any further
* downloading if an error occurs; else will continue attempting to
* download the remainder of the descriptors.
* @param downloadObs the observer to notify of progress, success and
* failure.
*/
public void download (
List descriptors, boolean fragile, DownloadObserver downloadObs)
{
// add the download request to the download queue
DownloadRecord dlrec = new DownloadRecord();
dlrec.descriptors = descriptors;
dlrec.obs = downloadObs;
dlrec.fragile = fragile;
_dlqueue.append(dlrec);
synchronized (this) {
// if we've not yet got our downloading thread...
if (_dlthread == null) {
// create the thread
_dlthread = new Thread() {
public void run () {
processDownloads();
}
};
_dlthread.setDaemon(true);
// and start it going
_dlthread.start();
}
}
}
/**
* Called by the download thread to process all download requests in
* the download queue.
*/
protected void processDownloads ()
{
// create the data buffer to be used when reading files
_buffer = new byte[BUFFER_SIZE];
DownloadRecord dlrec;
while (true) {
synchronized (this) {
// kill the download thread if we have no remaining
// download requests
if (_dlqueue.size() == 0) {
_dlthread = null;
// free up the data buffer
_buffer = null;
return;
}
// pop download off the queue
dlrec = (DownloadRecord)_dlqueue.getNonBlocking();
}
// handle the download request
processDownloadRequest(dlrec.descriptors, dlrec.obs, dlrec.fragile);
}
}
/**
* Processes a single download request.
*/
protected void processDownloadRequest (
List descriptors, DownloadObserver obs, boolean fragile)
{
// let the observer know that we're about to resolve all files to
// be downloaded
obs.resolvingDownloads();
// check the size and last-modified information for each file to
// ascertain whether our local copy needs to be refreshed
ArrayList fetch = new ArrayList();
long totalSize = 0;
int size = descriptors.size();
for (int ii = 0; ii < size; ii++) {
DownloadDescriptor desc = (DownloadDescriptor)descriptors.get(ii);
try {
// get the source file information
desc.fileSize = 0;
desc.lastModified = 0;
String protocol = desc.sourceURL.getProtocol();
if (protocol.equals("file")) {
// read the file information directly from the file system
File tfile = new File(desc.sourceURL.getPath());
desc.fileSize = tfile.length();
desc.lastModified = tfile.lastModified();
} else if (protocol.equals("http")) {
// read the file information via an HTTP HEAD request
HttpURLConnection ucon = (HttpURLConnection)
desc.sourceURL.openConnection();
ucon.setRequestMethod("HEAD");
ucon.connect();
desc.fileSize = ucon.getContentLength();
desc.lastModified = ucon.getLastModified();
} else {
throw new IOException(
"Unknown source file protocol " +
"[protocol=" + protocol + ", desc=" + desc + "].");
}
// determine whether we actually need to fetch the file by
// checking to see whether we have a local copy at all,
// and if so, whether its file size or last-modified time
// differs from the source file
desc.destFileSize = desc.destFile.length();
desc.destLastModified = desc.destFile.lastModified();
if ((!desc.destFile.exists()) ||
(desc.destFileSize != desc.fileSize) ||
(desc.destLastModified != desc.lastModified)) {
// increment the total file size to be fetched
totalSize += desc.fileSize;
// add the file to the list of files to be fetched
fetch.add(desc);
Log.debug("File deemed stale " +
"[url=" + desc.sourceURL + "].");
} else {
Log.debug("File deemed up-to-date " +
"[url=" + desc.sourceURL + "].");
}
} catch (IOException ioe) {
obs.downloadFailed(null, ioe);
if (fragile) {
return;
}
}
}
// download all stale files
size = fetch.size();
long currentSize = 0;
boolean complete = false;
for (int ii = 0; ii < size; ii++) {
DownloadDescriptor desc = (DownloadDescriptor)fetch.get(ii);
try {
complete = processDownload(desc, obs, currentSize, totalSize);
currentSize += desc.fileSize;
} catch (IOException ioe) {
obs.downloadFailed(desc, ioe);
if (fragile) {
return;
}
}
}
// make sure to always let the observer know that we've wrapped up
// by reporting 100% completion
if (!complete) {
obs.downloadProgress(100);
}
}
/**
* Processes a single download descriptor. Returns whether the
* download observer was notified of a 100% complete progress update.
*/
protected boolean processDownload (
DownloadDescriptor desc, DownloadObserver obs, long currentSize,
long totalSize)
throws IOException
{
// download the resource bundle from the specified URL
URLConnection ucon = desc.sourceURL.openConnection();
// prepare to read the data from the URL into the cache file
Log.info("Downloading file [url=" + desc.sourceURL + "].");
InputStream in = ucon.getInputStream();
FileOutputStream out = new FileOutputStream(desc.destFile);
int read;
boolean complete = false;
// read in the file data
while ((read = in.read(_buffer)) != -1) {
// write it out to our local copy
out.write(_buffer, 0, read);
// report our progress to the download observer as a
// percentage of the total file data to be transferred
currentSize += read;
int pctdone = (int)((currentSize / (float)totalSize) * 100f);
complete = (pctdone >= 100);
obs.downloadProgress(pctdone);
}
// close the streams
in.close();
out.close();
// if we have a last modified time, we want to adjust our cache
// file accordingly
if (desc.lastModified != 0) {
desc.destFile.setLastModified(desc.lastModified);
}
return complete;
}
/**
* A record describing a single download request.
*/
protected static class DownloadRecord
{
/** The list of download descriptors to be downloaded. */
public List descriptors;
/** The download observer to notify of download progress. */
public DownloadObserver obs;
/** Whether to abort downloading if an error occurs. */
public boolean fragile;
}
/** The downloading thread. */
protected Thread _dlthread;
/** The queue of download requests. */
protected Queue _dlqueue = new Queue();
/** The data buffer used when reading file data. */
protected byte[] _buffer;
/** The data buffer size for reading file data. */
protected static final int BUFFER_SIZE = 2048;
}
src/java/com/threerings/resource/ResourceBundle.java
+46 -12
View File
@@ -1,5 +1,5 @@
//
// $Id: ResourceBundle.java,v 1.2 2001/11/20 03:46:28 mdb Exp $
// $Id: ResourceBundle.java,v 1.3 2002/07/19 20:12:23 shaper Exp $
package com.threerings.resource;
@@ -10,6 +10,8 @@ import java.io.InputStream;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import com.samskivert.io.NestableIOException;
/**
* A resource bundle provides access to the resources in a jar file.
*/
@@ -19,14 +21,10 @@ public class ResourceBundle
* Constructs a resource bundle with the supplied jar file.
*
* @param source a file object that references our source jar file.
*
* @exception IOException thrown if an error occurs reading our jar
* file.
*/
public ResourceBundle (File source)
throws IOException
{
_source = new JarFile(source);
_source = source;
}
/**
@@ -45,12 +43,13 @@ public class ResourceBundle
public InputStream getResource (String path)
throws IOException
{
resolveJarFile();
// TBD: determine whether or not we need to convert the path into
// a platform-dependent path if we're on Windows
JarEntry entry = _source.getJarEntry(path);
JarEntry entry = _jarSource.getJarEntry(path);
InputStream stream = null;
if (entry != null) {
stream = _source.getInputStream(entry);
stream = _jarSource.getInputStream(entry);
}
return stream;
}
@@ -62,7 +61,12 @@ public class ResourceBundle
*/
public boolean containsResource (String path)
{
return (_source.getJarEntry(path) != null);
try {
resolveJarFile();
return (_jarSource.getJarEntry(path) != null);
} catch (IOException ioe) {
return false;
}
}
/**
@@ -70,10 +74,40 @@ public class ResourceBundle
*/
public String toString ()
{
return "[path=" + _source.getName() +
", entries=" + _source.size() + "]";
try {
resolveJarFile();
return "[path=" + _jarSource.getName() +
", entries=" + _jarSource.size() + "]";
} catch (IOException ioe) {
return "[file=" + _source + ", ioe=" + ioe + "]";
}
}
/**
* Creates the internal jar file reference if we've not already got
* it; we do this lazily so as to avoid any jar- or zip-file-related
* antics until and unless doing so is required, and because the
* resource manager would like to be able to create bundles before the
* associated files have been fully downloaded.
*/
protected void resolveJarFile ()
throws IOException
{
try {
if (_jarSource == null) {
_jarSource = new JarFile(_source);
}
} catch (IOException ioe) {
throw new NestableIOException(
"Failed to resolve resource bundle jar file " +
"[file=" + _source + "]", ioe);
}
}
/** The file from which we construct our jar file. */
protected File _source;
/** The jar file from which we load resources. */
protected JarFile _source;
protected JarFile _jarSource;
}
src/java/com/threerings/resource/ResourceManager.java
+152 -84
View File
@@ -1,5 +1,5 @@
//
// $Id: ResourceManager.java,v 1.13 2002/04/01 20:47:28 mdb Exp $
// $Id: ResourceManager.java,v 1.14 2002/07/19 20:12:23 shaper Exp $
package com.threerings.resource;
@@ -17,12 +17,15 @@ import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
import org.apache.commons.io.StreamUtils;
import com.samskivert.util.StringUtil;
import com.threerings.resource.DownloadManager.DownloadDescriptor;
import com.threerings.resource.DownloadManager.DownloadObserver;
/**
* The resource manager is responsible for maintaining a repository of
* resources that are synchronized with a remote source. This is
@@ -38,13 +41,15 @@ import com.samskivert.util.StringUtil;
* locate a resource in the default resource set, it falls back to loading
* the resource via the classloader (which will search the classpath).
*
* <p> The resource manager must be provided with the URL of a resource
* definition file which describes these resource sets at construct
* time. The definition file will be loaded and the resource bundles
* defined within will be loaded relative to the resource definition URL.
* The bundles will be cached in the user's home directory and only
* reloaded when the source resources have been updated. The resource
* definition file looks something like the following:
* <p> Applications that wish to make use of resource sets and their
* associated bundles must call {@link #initBundles} after constructing
* the resource manager, providing the URL of a resource definition file
* which describes these resource sets. The definition file will be loaded
* and the resource bundles defined within will be loaded relative to the
* resource definition URL. The bundles will be cached in the user's home
* directory and only reloaded when the source resources have been
* updated. The resource definition file looks something like the
* following:
*
* <pre>
* resource.set.default = sets/misc/config.jar: \
@@ -71,11 +76,35 @@ import com.samskivert.util.StringUtil;
public class ResourceManager
{
/**
* Constructs a resource manager which will load resources as
* specified in the configuration file, the path to which is supplied
* via <code>configPath</code>. If resource sets are not needed and
* resources will only be loaded via the classpath, null may be passed
* in <code>resourceURL</code> and <code>configPath</code>.
* Provides facilities for notifying an observer of resource bundle
* download progress.
*/
public interface BundleDownloadObserver
{
/**
* Called when the resource manager is about to check for an
* update of any of our resource sets.
*/
public void checkingForUpdate ();
/**
* Called to inform the observer of ongoing progress toward
* completion of the overall bundle downloading task. The caller
* is guaranteed to get at least one call reporting 100%
* completion.
*/
public void downloadProgress (int percent);
/**
* Called if a failure occurs while checking for an update or
* downloading all resource sets.
*/
public void downloadFailed (Exception e);
}
/**
* Constructs a resource manager which will load resources via the
* classloader, prepending <code>resourceRoot</code> to their path.
*
* @param resourceRoot the path to prepend to resource paths prior to
* attempting to load them via the classloader. When resources are
@@ -85,15 +114,8 @@ public class ResourceManager
* to isolate them from the rest of the files in the classpath. This
* is not a platform dependent path (forward slash is always used to
* separate path elements).
* @param resourceURL the base URL from which resources are loaded.
* Relative paths specified in the resource definition file will be
* loaded relative to this path. If this is null, the system property
* <code>resource_url</code> will be used, if available.
* @param configPath the path (relative to the resource URL) of the
* resource definition file.
*/
public ResourceManager (
String resourceRoot, String resourceURL, String configPath)
public ResourceManager (String resourceRoot)
{
// keep track of our root path
_rootPath = resourceRoot;
@@ -107,7 +129,28 @@ public class ResourceManager
// use the classloader that loaded us
_loader = getClass().getClassLoader();
}
/**
* Initializes the bundle sets to be made available by this resource
* manager. Applications that wish to make use of resource bundles
* should call this method after constructing the resource manager.
*
* @param resourceURL the base URL from which resources are loaded.
* Relative paths specified in the resource definition file will be
* loaded relative to this path. If this is null, the system property
* <code>resource_url</code> will be used, if available.
* @param configPath the path (relative to the resource URL) of the
* resource definition file.
* @param downloadObs the bundle download observer to notify of
* download progress and success or failure, or <code>null</code> if
* the caller doesn't care to be informed; note that in the latter
* case, the calling thread will block until bundle updating is
* complete.
*/
public void initBundles (String resourceURL, String configPath,
BundleDownloadObserver downloadObs)
{
// if the resource URL wasn't provided, we try to figure it out
// for ourselves
if (resourceURL == null) {
@@ -142,6 +185,7 @@ public class ResourceManager
Properties config = loadConfig(rurl, configPath);
// resolve the configured resource sets
ArrayList dlist = new ArrayList();
Enumeration names = config.propertyNames();
while (names.hasMoreElements()) {
String key = (String)names.nextElement();
@@ -149,8 +193,87 @@ public class ResourceManager
continue;
}
String setName = key.substring(RESOURCE_SET_PREFIX.length());
resolveResourceSet(rurl, setName, config.getProperty(key));
resolveResourceSet(rurl, setName, config.getProperty(key), dlist);
}
// start the download, blocking if we've no observer
DownloadManager dlmgr = new DownloadManager();
if (downloadObs == null) {
downloadBlocking(dlmgr, dlist);
} else {
downloadNonBlocking(dlmgr, dlist, downloadObs);
}
}
/**
* Downloads the files in the supplied download list, blocking the
* calling thread until the download is complete or a failure has
* occurred.
*/
protected void downloadBlocking (DownloadManager dlmgr, List dlist)
{
// create an object to wait on while the download takes place
final Object lock = new Object();
// pass the descriptors on to the download manager
dlmgr.download(dlist, true, new DownloadObserver() {
public void resolvingDownloads () {
// nothing for now
}
public void downloadProgress (int percent) {
if (percent == 100) {
synchronized (lock) {
// wake things up as the download is finished
lock.notify();
}
}
}
public void downloadFailed (DownloadDescriptor desc, Exception e) {
Log.warning("Failed to download file " +
"[desc=" + desc + ", e=" + e + "].");
synchronized (lock) {
// wake things up since we're fragile and so a
// single failure means all is booched
lock.notify();
}
}
});
try {
synchronized (lock) {
// block until the download has completed
lock.wait();
}
} catch (InterruptedException ie) {
Log.warning("Thread interrupted while waiting for download " +
"to complete [ie=" + ie + "].");
}
}
/**
* Downloads the files in the supplied download list asynchronously,
* notifying the download observer of ongoing progress.
*/
protected void downloadNonBlocking (
DownloadManager dlmgr, List dlist, final BundleDownloadObserver obs)
{
// pass the descriptors on to the download manager
dlmgr.download(dlist, true, new DownloadObserver() {
public void resolvingDownloads () {
obs.checkingForUpdate();
}
public void downloadProgress (int percent) {
obs.downloadProgress(percent);
}
public void downloadFailed (DownloadDescriptor desc, Exception e) {
obs.downloadFailed(e);
}
});
}
/**
@@ -262,7 +385,7 @@ public class ResourceManager
* information.
*/
protected void resolveResourceSet (
URL resourceURL, String setName, String definition)
URL resourceURL, String setName, String definition, List dlist)
{
StringTokenizer tok = new StringTokenizer(definition, ":");
ArrayList set = new ArrayList();
@@ -280,73 +403,18 @@ public class ResourceManager
// compute the path to the cache file for this bundle
File cfile = new File(genCachePath(setName, path));
// download the resource bundle from the specified URL
URLConnection ucon = burl.openConnection();
boolean readData = true;
// slap this on the list for retrieval or update by the
// download manager
dlist.add(new DownloadDescriptor(burl, cfile));
// set the last-modified time we're looking for
long lastModified = 0;
if (cfile.exists()) {
lastModified = cfile.lastModified();
ucon.setIfModifiedSince(lastModified);
}
// connect the URL
ucon.connect();
// if this is an HTTP connection, we want to use
// if-modified-since
if (lastModified != 0) {
if (ucon instanceof HttpURLConnection) {
HttpURLConnection hucon = (HttpURLConnection)ucon;
readData = (hucon.getResponseCode() !=
HttpURLConnection.HTTP_NOT_MODIFIED);
} else if (burl.getProtocol().equals("file")) {
// do some jockeying for file: URLs to determine
// whether or not the data is newer
File tfile = new File(burl.getPath());
readData = (tfile.lastModified() > lastModified);
}
}
// if this is a URL request, we want to keep track of the
// last modified time
if (ucon instanceof HttpURLConnection) {
HttpURLConnection hucon = (HttpURLConnection)ucon;
lastModified = hucon.getLastModified();
} else {
lastModified = 0;
}
// read the data from the URL into the cache file
if (readData) {
Log.info("Downloading bundle [url=" + burl + "].");
InputStream in = ucon.getInputStream();
FileOutputStream out = new FileOutputStream(cfile);
// pipe the input stream into the output stream
StreamUtils.pipe(in, out);
in.close();
out.close();
// if we have a last modified time, we want to adjust
// our cache file accordingly
if (lastModified != 0) {
cfile.setLastModified(lastModified);
}
}
// finally add this newly cached file to the set as a
// resource bundle
// finally, add the file that will be cached to the set as
// a resource bundle
set.add(new ResourceBundle(cfile));
} catch (MalformedURLException mue) {
Log.warning("Unable to create URL for resource " +
"[set=" + setName + ", path=" + path +
", error=" + mue + "].");
} catch (IOException ioe) {
Log.warning("Error processing resource set entry " +
"[url=" + burl + ", error=" + ioe + "].");
}
}