Merge pull request #162 from vatsal22/concurrent1

Download files concurrently

core/src/main/java/com/threerings/getdown/net/Downloader.java

@@ -11,6 +11,9 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import com.threerings.getdown.data.Resource;
 
@@ -106,9 +109,21 @@ public abstract class Downloader extends Thread
             // make a note of the time at which we started the download
             _start = System.currentTimeMillis();
 
+            ExecutorService exec = Executors.newCachedThreadPool();
+
             // now actually download the files
             for (Resource resource : _resources) {
-                download(resource);
+                download(resource, exec);
+            }
+
+            exec.shutdown();
+
+            try{
+                // 70 years should be long enough
+                exec.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
+            }catch (InterruptedException ie) {
+                log.warning("Download was interrupted...");
+                exec.shutdownNow();
             }
 
             // finally report our download completion if we did not already do so when downloading
@@ -147,7 +162,7 @@ public abstract class Downloader extends Thread
     /**
      * Downloads the specified resource from its remote location to its local location.
      */
-    protected void download (Resource rsrc)
+    protected void download (final Resource rsrc, ExecutorService exec)
         throws IOException
     {
         // make sure the resource's target directory exists
@@ -156,7 +171,17 @@ public abstract class Downloader extends Thread
             log.warning("Failed to create target directory for resource '" + rsrc + "'. " +
                     "Download will certainly fail.");
         }
+
+        exec.execute(new Runnable() {
+            @Override
+            public void run() {
+                try {
                     doDownload(rsrc);
+                }catch (IOException ioe) {
+                    log.info("Error: " + rsrc.getPath() + " failed to download...");
+                }
+            }
+        });
     }
 
     /**

core/src/main/java/com/threerings/getdown/net/HTTPDownloader.java

@@ -5,11 +5,15 @@
 
 package com.threerings.getdown.net;
 
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.URL;
 import java.net.URLConnection;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.util.Collection;
 
 import com.threerings.getdown.data.Resource;
@@ -56,44 +60,19 @@ public class HTTPDownloader extends Downloader
     protected void doDownload (Resource rsrc)
         throws IOException
     {
-        // download the resource from the specified URL
-        URLConnection conn = ConnectionUtil.open(rsrc.getRemote(), 0, 0);
-        conn.connect();
 
-        // make sure we got a satisfactory response code
+        log.info("Downloading resource", "url", rsrc.getRemote(), "size", -1);
-        if (conn instanceof HttpURLConnection) {
+
-            HttpURLConnection hcon = (HttpURLConnection)conn;
+        URL remoteURL = rsrc.getRemote();
-            if (hcon.getResponseCode() != HttpURLConnection.HTTP_OK) {
+        File localNew = rsrc.getLocalNew();
-                throw new IOException("Unable to download resource " + rsrc.getRemote() + ": " +
+
-                                      hcon.getResponseCode());
+
-            }
+        try(ReadableByteChannel rbc = Channels.newChannel(remoteURL.openStream());
+            FileOutputStream fos = new FileOutputStream(localNew)) {
+
+            fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+            updateObserver(rsrc, localNew.length(), localNew.length());
         }
 
-        long actualSize = conn.getContentLength();
-        log.info("Downloading resource", "url", rsrc.getRemote(), "size", actualSize);
-        long currentSize = 0L;
-        try (InputStream in = conn.getInputStream();
-             FileOutputStream out = new FileOutputStream(rsrc.getLocalNew())) {
-
-            // TODO: look to see if we have a download info file
-            // containing info on potentially partially downloaded data;
-            // if so, use a "Range: bytes=HAVE-" header.
-
-            // read in the file data
-            int read;
-            while ((read = in.read(_buffer)) != -1) {
-                // write it out to our local copy
-                out.write(_buffer, 0, read);
-
-                // if we have no observer, then don't bother computing download statistics
-                if (_obs == null) {
-                    continue;
-                }
-
-                // note that we've downloaded some data
-                currentSize += read;
-                updateObserver(rsrc, currentSize, actualSize);
-            }
-        }
     }
 }