Abstract the Downloader class, separate download protocol. Add bittorrent support (alpha)

src/java/com/threerings/getdown/launcher/Downloader.java

@@ -21,17 +21,10 @@
 package com.threerings.getdown.launcher;
 
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 
-import java.net.HttpURLConnection;
-
-import java.util.Iterator;
 import java.util.List;
 
-import com.samskivert.io.StreamUtil;
-
 import com.threerings.getdown.Log;
 import com.threerings.getdown.data.Resource;
 
@@ -40,7 +33,7 @@ import com.threerings.getdown.data.Resource;
  * requests to obtain size information and then downloading the files
  * individually, reporting progress back via a callback interface.
  */
-public class Downloader extends Thread
+public abstract class Downloader extends Thread
 {
     /**
      * An interface used to communicate status back to an external entity.
@@ -88,7 +81,7 @@ public class Downloader extends Thread
      * #start} method must be called on the downloader to initiate the
      * download process.
      */
-    public Downloader (List resources, Observer obs)
+    public Downloader (List<Resource> resources, Observer obs)
     {
         super("Downloader");
         _resources = resources;
@@ -109,8 +102,8 @@ public class Downloader extends Thread
             }
 
             // first compute the total size of our download
-            for (Iterator iter = _resources.iterator(); iter.hasNext(); ) {
+            for (Resource resource : _resources) {
-                discoverSize((Resource)iter.next());
+                discoverSize(resource);
             }
 
             Log.info("Downloading " + _totalSize + " bytes...");
@@ -119,9 +112,8 @@ public class Downloader extends Thread
             _start = System.currentTimeMillis();
 
             // now actually download the files
-            for (Iterator iter = _resources.iterator(); iter.hasNext(); ) {
+            for (Resource resource : _resources) {
-                current = (Resource)iter.next();
+                download(resource);
-                download(current);
             }
 
             // finally report our download completion if we did not
@@ -140,29 +132,17 @@ public class Downloader extends Thread
     }
 
     /**
-     * Issues a HEAD request for the specified resource and notes the
+     * Notes the amount of data needed to download the given resource..
-     * amount of data we will be downloading to account for it.
      */
     protected void discoverSize (Resource rsrc)
         throws IOException
     {
-        // read the file information via an HTTP HEAD request
-        HttpURLConnection ucon = (HttpURLConnection)
-            rsrc.getRemote().openConnection();
-        ucon.setRequestMethod("HEAD");
-        ucon.connect();
-
-        // make sure we got a satisfactory response code
-        if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
-            String errmsg = "Unable to check up-to-date for " +
-                rsrc.getRemote() + ": " + ucon.getResponseCode();
-            throw new IOException(errmsg);
-        }
-
         // add this resource's size to our total download size
-        _totalSize += ucon.getContentLength();
+        _totalSize += checkSize(rsrc);
     }
 
+    protected abstract long checkSize (Resource rsrc) throws IOException;
+
     /**
      * Downloads the specified resource from its remote location to its
      * local location.
@@ -179,45 +159,11 @@ public class Downloader extends Thread
                             "certainly fail.");
             }
         }
-
+        doDownload(rsrc);
-        // download the resource from the specified URL
-        HttpURLConnection ucon = (HttpURLConnection)
-            rsrc.getRemote().openConnection();
-        ucon.connect();
-
-        // make sure we got a satisfactory response code
-        if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
-            String errmsg = "Unable to download resource " +
-                rsrc.getRemote() + ": " + ucon.getResponseCode();
-            throw new IOException(errmsg);
     }
 
-        Log.info("Downloading resource [url=" + rsrc.getRemote() + "].");
+    protected void updateObserver ()
-        InputStream in = null;
+    {
-        FileOutputStream out = null;
-        try {
-            in = ucon.getInputStream();
-            out = new FileOutputStream(rsrc.getLocal());
-            int read;
-
-            // TODO: look to see if we have a download info file
-            // containing info on potentially partially downloaded data;
-            // if so, use a "Range: bytes=HAVE-" header.
-
-            // read in the file data
-            while ((read = in.read(_buffer)) != -1) {
-                // write it out to our local copy
-                out.write(_buffer, 0, read);
-
-                // if we have no observer, then don't bother computing
-                // download statistics
-                if (_obs == null) {
-                    continue;
-                }
-
-                // note that we've downloaded some data
-                _currentSize += read;
-
         // notify the observer if it's been sufficiently long
         // since our last notification
         long now = System.currentTimeMillis();
@@ -244,14 +190,14 @@ public class Downloader extends Thread
         }   
     }
 
-        } finally {
+    /**
-            StreamUtil.close(in);
+     * Accomplishes the copying of the resource from remote location to
-            StreamUtil.close(out);
+     * local location using transport-specific code
-        }
+     */
-    }
+    protected abstract void doDownload (Resource rsrc) throws IOException;
 
     /** The list of resources to be downloaded. */
-    protected List _resources;
+    protected List<Resource> _resources;
 
     /** The observer with whom we are communicating. */
     protected Observer _obs;

src/java/com/threerings/getdown/launcher/Getdown.java

@@ -24,15 +24,12 @@ import java.awt.BorderLayout;
 import java.awt.Container;
 import java.awt.EventQueue;
 import java.awt.Image;
-import java.awt.event.WindowAdapter;
-import java.awt.event.WindowEvent;
 import java.awt.image.BufferedImage;
 
 import javax.imageio.ImageIO;
 import javax.swing.JApplet;
 import javax.swing.JFrame;
 
-import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -370,7 +367,7 @@ public abstract class Getdown extends Thread
 
                 // now verify our resources...
                 setStatus("m.validating", -1, -1L, false);
-                List failures = _app.verifyResources(_progobs);
+                List<Resource> failures = _app.verifyResources(_progobs);
                 if (failures == null) {
                     Log.info("Resources verified.");
                     launch();
@@ -474,7 +471,7 @@ public abstract class Getdown extends Thread
      * Called if the application is determined to require resource
      * downloads.
      */
-    protected void download (List resources)
+    protected void download (List<Resource> resources)
     {
         final Object lock = new Object();
 
@@ -506,7 +503,14 @@ public abstract class Getdown extends Thread
                 }
             }
         };
-        Downloader dl = new Downloader(resources, obs);
+
+        // Torrent downloading is disabled by default until the kinks are out
+        Downloader dl;
+        if (false) {
+            dl = new TorrentDownloader(resources, obs);
+        } else {
+            dl = new HTTPDownloader(resources, obs);
+        }
         dl.start();
 
         // now wait for it to complete

src/java/com/threerings/getdown/launcher/HTTPDownloader.java

@@ -0,0 +1,90 @@
+package com.threerings.getdown.launcher;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.util.List;
+
+import com.samskivert.io.StreamUtil;
+import com.threerings.getdown.Log;
+import com.threerings.getdown.data.Resource;
+
+public class HTTPDownloader extends Downloader
+{
+    public HTTPDownloader (List<Resource> resources, Observer obs)
+    {
+        super(resources, obs);
+    }
+
+    /**
+     * Issues a HEAD request for the specified resource and notes the
+     * amount of data we will be downloading to account for it.
+     */
+    protected long checkSize (Resource rsrc)
+        throws IOException
+    {
+        // read the file information via an HTTP HEAD request
+        HttpURLConnection ucon = (HttpURLConnection)
+            rsrc.getRemote().openConnection();
+        ucon.setRequestMethod("HEAD");
+        ucon.connect();
+
+        // make sure we got a satisfactory response code
+        if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
+            String errmsg = "Unable to check up-to-date for " +
+                rsrc.getRemote() + ": " + ucon.getResponseCode();
+            throw new IOException(errmsg);
+        }
+
+        return ucon.getContentLength();
+    }
+
+    protected void doDownload (Resource rsrc)
+        throws IOException
+    {
+        // download the resource from the specified URL
+        HttpURLConnection ucon = (HttpURLConnection)
+            rsrc.getRemote().openConnection();
+        ucon.connect();
+
+        // make sure we got a satisfactory response code
+        if (ucon.getResponseCode() != HttpURLConnection.HTTP_OK) {
+            String errmsg = "Unable to download resource " +
+                rsrc.getRemote() + ": " + ucon.getResponseCode();
+            throw new IOException(errmsg);
+        }
+
+        Log.info("Downloading resource [url=" + rsrc.getRemote() + "].");
+        InputStream in = null;
+        FileOutputStream out = null;
+        try {
+            in = ucon.getInputStream();
+            out = new FileOutputStream(rsrc.getLocal());
+            int read;
+
+            // TODO: look to see if we have a download info file
+            // containing info on potentially partially downloaded data;
+            // if so, use a "Range: bytes=HAVE-" header.
+
+            // read in the file data
+            while ((read = in.read(_buffer)) != -1) {
+                // write it out to our local copy
+                out.write(_buffer, 0, read);
+
+                // if we have no observer, then don't bother computing
+                // download statistics
+                if (_obs == null) {
+                    continue;
+                }
+
+                // note that we've downloaded some data
+                _currentSize += read;
+                updateObserver();
+            }
+        } finally {
+            StreamUtil.close(in);
+            StreamUtil.close(out);
+        }
+    }
+}

src/java/com/threerings/getdown/launcher/TorrentDownloader.java

@@ -0,0 +1,83 @@
+package com.threerings.getdown.launcher;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+
+import org.klomp.snark.Snark;
+import org.klomp.snark.SnarkShutdown;
+
+import com.threerings.getdown.Log;
+import com.threerings.getdown.data.Resource;
+
+public class TorrentDownloader extends Downloader
+{
+    public TorrentDownloader (List<Resource> resources, Observer obs)
+    {
+        super(resources, obs);
+        Log.info("Using bittorrent to fetch files");
+        for (Resource resource : resources) {
+            String url = resource.getRemote().toString() + ".torrent";
+            Snark snark = new Snark(url, null, -1, null, null);
+            snark.command_interpreter = false;
+            _torrentmap.put(resource, snark);
+        }
+    }
+
+    @Override
+    protected long checkSize(Resource rsrc)
+        throws IOException
+    {
+        Snark snark = _torrentmap.get(rsrc);
+        long length = -1;
+        try {
+            snark.setupNetwork();
+            length = snark.meta.getTotalLength();
+        } catch (Exception e) {
+            // Unfortunately, the snark library does System.exit(-1) right now
+            // instead of properly passing the exception up the chain.
+            Log.warning("Bittorrent failed, falling back to HTTP");
+            snark.shutdown();
+            _fallback = new HTTPDownloader(_resources, _obs);
+            length = _fallback.checkSize(rsrc);
+        }
+        return length;
+    }
+
+    @Override
+    protected void doDownload(Resource rsrc)
+        throws IOException
+    {
+        if (_fallback != null) {
+            _fallback.doDownload(rsrc);
+            return;
+        }
+        Snark snark = _torrentmap.get(rsrc);
+        snark.collectPieces();
+        SnarkShutdown snarkhook = new SnarkShutdown(snark.storage,
+            snark.coordinator, snark.acceptor, snark.trackerclient, snark);
+        Runtime.getRuntime().addShutdownHook(snarkhook);
+        while (_currentSize != snark.meta.getTotalLength()) {
+            long now = System.currentTimeMillis();
+            if ((now - _lastUpdate) >= UPDATE_DELAY) {
+                _currentSize = snark.coordinator.getDownloaded();
+                if (_currentSize < SIZE_THRESHOLD &&
+                    (now - _start) >= TIME_THRESHOLD) {
+                    // The download isn't going as planned, abort;
+                    snark.shutdown();
+                    _fallback = new HTTPDownloader(_resources, _obs);
+                    _fallback.doDownload(rsrc);   
+                    return;
+                }
+            }
+            updateObserver();
+        }
+        snark.shutdown();
+    }
+
+    protected HashMap<Resource, Snark> _torrentmap =
+        new HashMap<Resource, Snark>();
+    protected HTTPDownloader _fallback = null;
+    protected static final long TIME_THRESHOLD = 60 * 1000l;
+    protected static final long SIZE_THRESHOLD = 4000l;
+}