AfflictionNetworks/getdown
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Abstract the Downloader class, separate download protocol. Add bittorrent support (alpha)

Browse Source
This commit is contained in:
Elizabeth Fong
2006-07-13 18:17:31 +00:00
parent 5c7b887f59
commit d07042efee
4 changed files with 229 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/java/com/threerings/getdown/launcher/Downloader.java
+39 -93
View File
@@ -21,17 +21,10 @@
package com.threerings.getdown.launcher;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.util.Iterator;
import java.util.List;
import com.samskivert.io.StreamUtil;
import com.threerings.getdown.Log;
import com.threerings.getdown.data.Resource;
@@ -40,7 +33,7 @@ import com.threerings.getdown.data.Resource;
* requests to obtain size information and then downloading the files
* individually, reporting progress back via a callback interface.
*/
public class Downloader extends Thread
public abstract class Downloader extends Thread
{
/**
* An interface used to communicate status back to an external entity.
@@ -88,7 +81,7 @@ public class Downloader extends Thread
* #start} method must be called on the downloader to initiate the
* download process.
*/
public Downloader (List resources, Observer obs)
public Downloader (List<Resource> resources, Observer obs)
{
super("Downloader");
_resources = resources;
@@ -109,8 +102,8 @@ public class Downloader extends Thread
}
// first compute the total size of our download
for (Iterator iter = _resources.iterator(); iter.hasNext(); ) {
discoverSize((Resource)iter.next());
for (Resource resource : _resources) {
discoverSize(resource);
}
Log.info("Downloading " + _totalSize + " bytes...");
@@ -119,9 +112,8 @@ public class Downloader extends Thread
_start = System.currentTimeMillis();
// now actually download the files
for (Iterator iter = _resources.iterator(); iter.hasNext(); ) {
current = (Resource)iter.next();
download(current);
for (Resource resource : _resources) {
download(resource);
}
// finally report our download completion if we did not
@@ -140,29 +132,17 @@ public class Downloader extends Thread
}
/**
* Issues a HEAD request for the specified resource and notes the
* amount of data we will be downloading to account for it.
* Notes the amount of data needed to download the given resource..
*/
protected void discoverSize (Resource rsrc)
throws IOException
{
// read the file information via an HTTP HEAD request
HttpURLConnection ucon = (HttpURLConnection)
rsrc.getRemote().openConnection();
ucon.setRequestMethod("HEAD");
ucon.connect();
// make sure we got a satisfactory response code
if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
String errmsg = "Unable to check up-to-date for " +
rsrc.getRemote() + ": " + ucon.getResponseCode();
throw new IOException(errmsg);
}
// add this resource's size to our total download size
_totalSize += ucon.getContentLength();
_totalSize += checkSize(rsrc);
}
protected abstract long checkSize (Resource rsrc) throws IOException;
/**
* Downloads the specified resource from its remote location to its
* local location.
@@ -179,79 +159,45 @@ public class Downloader extends Thread
"certainly fail.");
}
}
doDownload(rsrc);
}
// download the resource from the specified URL
HttpURLConnection ucon = (HttpURLConnection)
rsrc.getRemote().openConnection();
ucon.connect();
protected void updateObserver ()
{
// notify the observer if it's been sufficiently long
// since our last notification
long now = System.currentTimeMillis();
if ((now - _lastUpdate) >= UPDATE_DELAY) {
_lastUpdate = now;
// make sure we got a satisfactory response code
if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
String errmsg = "Unable to download resource " +
rsrc.getRemote() + ": " + ucon.getResponseCode();
throw new IOException(errmsg);
}
// compute our bytes per second
long secs = (now - _start) / 1000L;
long bps = (secs == 0) ? 0 : (_currentSize / secs);
Log.info("Downloading resource [url=" + rsrc.getRemote() + "].");
InputStream in = null;
FileOutputStream out = null;
try {
in = ucon.getInputStream();
out = new FileOutputStream(rsrc.getLocal());
int read;
// compute our percentage completion
int pctdone = (int)(
(_currentSize / (float)_totalSize) * 100f);
// TODO: look to see if we have a download info file
// containing info on potentially partially downloaded data;
// if so, use a "Range: bytes=HAVE-" header.
// estimate our time remaining
long remaining = (bps <= 0) ? -1 :
(_totalSize - _currentSize) / bps;
// read in the file data
while ((read = in.read(_buffer)) != -1) {
// write it out to our local copy
out.write(_buffer, 0, read);
// if we have no observer, then don't bother computing
// download statistics
if (_obs == null) {
continue;
}
// note that we've downloaded some data
_currentSize += read;
// notify the observer if it's been sufficiently long
// since our last notification
long now = System.currentTimeMillis();
if ((now - _lastUpdate) >= UPDATE_DELAY) {
_lastUpdate = now;
// compute our bytes per second
long secs = (now - _start) / 1000L;
long bps = (secs == 0) ? 0 : (_currentSize / secs);
// compute our percentage completion
int pctdone = (int)(
(_currentSize / (float)_totalSize) * 100f);
// estimate our time remaining
long remaining = (bps <= 0) ? -1 :
(_totalSize - _currentSize) / bps;
// make sure we only report 100% exactly once
if (pctdone < 100 || !_complete) {
_complete = (pctdone == 100);
_obs.downloadProgress(pctdone, remaining);
}
}
// make sure we only report 100% exactly once
if (pctdone < 100 || !_complete) {
_complete = (pctdone == 100);
_obs.downloadProgress(pctdone, remaining);
}
} finally {
StreamUtil.close(in);
StreamUtil.close(out);
}
}
/**
* Accomplishes the copying of the resource from remote location to
* local location using transport-specific code
*/
protected abstract void doDownload (Resource rsrc) throws IOException;
/** The list of resources to be downloaded. */
protected List _resources;
protected List<Resource> _resources;
/** The observer with whom we are communicating. */
protected Observer _obs;
src/java/com/threerings/getdown/launcher/Getdown.java
+10 -6
View File
@@ -24,15 +24,12 @@ import java.awt.BorderLayout;
import java.awt.Container;
import java.awt.EventQueue;
import java.awt.Image;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.awt.image.BufferedImage;
import javax.imageio.ImageIO;
import javax.swing.JApplet;
import javax.swing.JFrame;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
@@ -370,7 +367,7 @@ public abstract class Getdown extends Thread
// now verify our resources...
setStatus("m.validating", -1, -1L, false);
List failures = _app.verifyResources(_progobs);
List<Resource> failures = _app.verifyResources(_progobs);
if (failures == null) {
Log.info("Resources verified.");
launch();
@@ -474,7 +471,7 @@ public abstract class Getdown extends Thread
* Called if the application is determined to require resource
* downloads.
*/
protected void download (List resources)
protected void download (List<Resource> resources)
{
final Object lock = new Object();
@@ -506,7 +503,14 @@ public abstract class Getdown extends Thread
}
}
};
Downloader dl = new Downloader(resources, obs);
// Torrent downloading is disabled by default until the kinks are out
Downloader dl;
if (false) {
dl = new TorrentDownloader(resources, obs);
} else {
dl = new HTTPDownloader(resources, obs);
}
dl.start();
// now wait for it to complete
src/java/com/threerings/getdown/launcher/HTTPDownloader.java
+90
View File
@@ -0,0 +1,90 @@
package com.threerings.getdown.launcher;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.util.List;
import com.samskivert.io.StreamUtil;
import com.threerings.getdown.Log;
import com.threerings.getdown.data.Resource;
public class HTTPDownloader extends Downloader
{
public HTTPDownloader (List<Resource> resources, Observer obs)
{
super(resources, obs);
}
/**
* Issues a HEAD request for the specified resource and notes the
* amount of data we will be downloading to account for it.
*/
protected long checkSize (Resource rsrc)
throws IOException
{
// read the file information via an HTTP HEAD request
HttpURLConnection ucon = (HttpURLConnection)
rsrc.getRemote().openConnection();
ucon.setRequestMethod("HEAD");
ucon.connect();
// make sure we got a satisfactory response code
if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
String errmsg = "Unable to check up-to-date for " +
rsrc.getRemote() + ": " + ucon.getResponseCode();
throw new IOException(errmsg);
}
return ucon.getContentLength();
}
protected void doDownload (Resource rsrc)
throws IOException
{
// download the resource from the specified URL
HttpURLConnection ucon = (HttpURLConnection)
rsrc.getRemote().openConnection();
ucon.connect();
// make sure we got a satisfactory response code
if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
String errmsg = "Unable to download resource " +
rsrc.getRemote() + ": " + ucon.getResponseCode();
throw new IOException(errmsg);
}
Log.info("Downloading resource [url=" + rsrc.getRemote() + "].");
InputStream in = null;
FileOutputStream out = null;
try {
in = ucon.getInputStream();
out = new FileOutputStream(rsrc.getLocal());
int read;
// TODO: look to see if we have a download info file
// containing info on potentially partially downloaded data;
// if so, use a "Range: bytes=HAVE-" header.
// read in the file data
while ((read = in.read(_buffer)) != -1) {
// write it out to our local copy
out.write(_buffer, 0, read);
// if we have no observer, then don't bother computing
// download statistics
if (_obs == null) {
continue;
}
// note that we've downloaded some data
_currentSize += read;
updateObserver();
}
} finally {
StreamUtil.close(in);
StreamUtil.close(out);
}
}
}
src/java/com/threerings/getdown/launcher/TorrentDownloader.java
+83
View File
@@ -0,0 +1,83 @@
package com.threerings.getdown.launcher;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import org.klomp.snark.Snark;
import org.klomp.snark.SnarkShutdown;
import com.threerings.getdown.Log;
import com.threerings.getdown.data.Resource;
public class TorrentDownloader extends Downloader
{
public TorrentDownloader (List<Resource> resources, Observer obs)
{
super(resources, obs);
Log.info("Using bittorrent to fetch files");
for (Resource resource : resources) {
String url = resource.getRemote().toString() + ".torrent";
Snark snark = new Snark(url, null, -1, null, null);
snark.command_interpreter = false;
_torrentmap.put(resource, snark);
}
}
@Override
protected long checkSize(Resource rsrc)
throws IOException
{
Snark snark = _torrentmap.get(rsrc);
long length = -1;
try {
snark.setupNetwork();
length = snark.meta.getTotalLength();
} catch (Exception e) {
// Unfortunately, the snark library does System.exit(-1) right now
// instead of properly passing the exception up the chain.
Log.warning("Bittorrent failed, falling back to HTTP");
snark.shutdown();
_fallback = new HTTPDownloader(_resources, _obs);
length = _fallback.checkSize(rsrc);
}
return length;
}
@Override
protected void doDownload(Resource rsrc)
throws IOException
{
if (_fallback != null) {
_fallback.doDownload(rsrc);
return;
}
Snark snark = _torrentmap.get(rsrc);
snark.collectPieces();
SnarkShutdown snarkhook = new SnarkShutdown(snark.storage,
snark.coordinator, snark.acceptor, snark.trackerclient, snark);
Runtime.getRuntime().addShutdownHook(snarkhook);
while (_currentSize != snark.meta.getTotalLength()) {
long now = System.currentTimeMillis();
if ((now - _lastUpdate) >= UPDATE_DELAY) {
_currentSize = snark.coordinator.getDownloaded();
if (_currentSize < SIZE_THRESHOLD &&
(now - _start) >= TIME_THRESHOLD) {
// The download isn't going as planned, abort;
snark.shutdown();
_fallback = new HTTPDownloader(_resources, _obs);
_fallback.doDownload(rsrc);
return;
}
}
updateObserver();
}
snark.shutdown();
}
protected HashMap<Resource, Snark> _torrentmap =
new HashMap<Resource, Snark>();
protected HTTPDownloader _fallback = null;
protected static final long TIME_THRESHOLD = 60 * 1000l;
protected static final long SIZE_THRESHOLD = 4000l;
}