From b4dae3be33f3f9917c438256d9835fc860928a7c Mon Sep 17 00:00:00 2001 From: Michael Bayne Date: Thu, 23 Sep 2010 00:15:20 +0000 Subject: [PATCH] If we get valid values once we start downloading, use those to provide moderately more sane progress indication. It's still non-ideal, because, for example, we'll start out assuming everything is length 1. So we'd have a queue like (1, 1, 1, 1). Then we start downloading the first resource and see that we actually have (1500000, 1, 1, 1), so we climb up to 99% downloading that first resource, and then we find out that we have (1500000, 450000, 1, 1) and we pop back down to 70% or so, and so on as we discover in turn that the web server lied to us about each file. The alternative is to just sit at 25% for ages, then sit at 50% for ages, then 75%, etc. Maybe that's better... I guess I just like to see some sort of smooth upward progress. This is only a problem for me since I'm hosting a Getdown client on Github which lies about Content-size in a HEAD request. So OOO projects will continue to give accurate progress reports. --- .../java/com/threerings/getdown/net/Downloader.java | 11 +++++++++-- .../com/threerings/getdown/net/HTTPDownloader.java | 5 +++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/threerings/getdown/net/Downloader.java b/src/main/java/com/threerings/getdown/net/Downloader.java index c8a556b..00b5c6e 100644 --- a/src/main/java/com/threerings/getdown/net/Downloader.java +++ b/src/main/java/com/threerings/getdown/net/Downloader.java @@ -190,14 +190,21 @@ public abstract class Downloader extends Thread * * @param rsrc the resource currently being downloaded. * @param currentSize the number of bytes currently downloaded for said resource. + * @param actualSize the size reported for this resource now that we're actually downloading + * it. Some web servers lie about Content-length when doing a HEAD request, so by reporting + * updated sizes here we can recover from receiving bogus information in the earlier {@link + * #checkSize} phase. */ - protected void updateObserver (Resource rsrc, long currentSize) + protected void updateObserver (Resource rsrc, long currentSize, long actualSize) throws IOException { + // update the actual size for this resource (but don't let it shrink) + _sizes.put(rsrc, actualSize = Math.max(actualSize, _sizes.get(rsrc))); + // update the current downloaded size for said resource; don't allow the downloaded bytes // to exceed the original claimed size of the resource, otherwise our progress will get // booched and we'll end up back on the Daily WTF: http://tinyurl.com/29wt4oq - _downloaded.put(rsrc, Math.min(_sizes.get(rsrc), currentSize)); + _downloaded.put(rsrc, Math.min(actualSize, currentSize)); // notify the observer if it's been sufficiently long since our last notification long now = System.currentTimeMillis(); diff --git a/src/main/java/com/threerings/getdown/net/HTTPDownloader.java b/src/main/java/com/threerings/getdown/net/HTTPDownloader.java index fa1e4a9..df75499 100644 --- a/src/main/java/com/threerings/getdown/net/HTTPDownloader.java +++ b/src/main/java/com/threerings/getdown/net/HTTPDownloader.java @@ -89,7 +89,8 @@ public class HTTPDownloader extends Downloader } } - log.info("Downloading resource [url=" + rsrc.getRemote() + "]."); + long actualSize = conn.getContentLength(); + log.info("Downloading resource", "url", rsrc.getRemote(), "size", actualSize); InputStream in = null; FileOutputStream out = null; long currentSize = 0L; @@ -114,7 +115,7 @@ public class HTTPDownloader extends Downloader // note that we've downloaded some data currentSize += read; - updateObserver(rsrc, currentSize); + updateObserver(rsrc, currentSize, actualSize); } } finally { StreamUtil.close(in);