Create a separate digest.txt file for each "protocol version".

Right now there's only two versions of the digester protocol, so I just
always generate both of them. If we evolve the protocol further, we
could conceivably allow you to specify a "minimum" protocol version,
once you were confident that all of your clients had upgraded to a
version of Getdown that used that minimum version.

src/main/java/com/threerings/getdown/data/Application.java

@@ -904,7 +904,7 @@ public class Application
             // now re-download our control files; we download the digest first so that if it fails,
             // our config file will still reference the old version and re-running the updater will
             // start the whole process over again
-            downloadDigestFile();
+            downloadDigestFiles();
             downloadConfigFile();
 
         } catch (IOException ex) {
@@ -1171,7 +1171,7 @@ public class Application
             String olddig = (_digest == null) ? "" : _digest.getMetaDigest();
             try {
                 status.updateStatus("m.checking");
-                downloadDigestFile();
+                downloadDigestFiles();
                 _digest = new Digest(_appdir);
                 if (!olddig.equals(_digest.getMetaDigest())) {
                     log.info("Unversioned digest changed. Revalidating...");
@@ -1189,7 +1189,7 @@ public class Application
         // exceptions to propagate up to the caller as there is nothing else we can do
         if (_digest == null) {
             status.updateStatus("m.updating_metadata");
-            downloadDigestFile();
+            downloadDigestFiles();
             _digest = new Digest(_appdir);
         }
 
@@ -1200,7 +1200,7 @@ public class Application
             // attempt to redownload both of our metadata files; again we pass errors up to our
             // caller because there's nothing we can do to automatically recover
             downloadConfigFile();
-            downloadDigestFile();
+            downloadDigestFiles();
             _digest = new Digest(_appdir);
             // revalidate everything if we end up downloading new metadata
             clearValidationMarkers();
@@ -1459,13 +1459,15 @@ public class Application
     }
 
     /**
-     * Downloads a copy of Digest.DIGEST_FILE and validates its signature.
+     * Downloads the digest files and validates their signature.
      * @throws IOException
      */
-    protected void downloadDigestFile ()
+    protected void downloadDigestFiles ()
         throws IOException
     {
-        downloadControlFile(Digest.DIGEST_FILE, true);
+        for (int version = 1; version <= Digest.VERSION; version++) {
+            downloadControlFile(Digest.digestFile(version), true);
+        }
     }
 
     /**

src/main/java/com/threerings/getdown/data/Digest.java

@@ -30,21 +30,80 @@ import static com.threerings.getdown.Log.log;
  */
 public class Digest
 {
-    /** The name of our MD5 digest file. */
+    /** The current version of the digest protocol. */
-    public static final String DIGEST_FILE = "digest.txt";
+    public static final int VERSION = 2;
+
+    /**
+     * Returns the name of the digest file for the specified protocol version.
+     */
+    public static String digestFile (int version) {
+        String infix = version > 1 ? String.valueOf(version) : "";
+        return FILE_NAME + infix + FILE_SUFFIX;
+    }
+
+    /**
+     * Creates a digest file at the specified location using the supplied list of resources.
+     * @param version the version of the digest protocol to use.
+     */
+    public static void createDigest (int version, List<Resource> resources, File output)
+        throws IOException
+    {
+        MessageDigest md = getMessageDigest();
+        StringBuilder data = new StringBuilder();
+        PrintWriter pout = null;
+        try {
+            pout = new PrintWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
+
+            // compute and append the MD5 digest of each resource in the list
+            for (Resource rsrc : resources) {
+                String path = rsrc.getPath();
+                try {
+                    String digest = rsrc.computeDigest(version, md, null);
+                    note(data, path, digest);
+                    pout.println(path + " = " + digest);
+                } catch (Throwable t) {
+                    throw (IOException) new IOException(
+                        "Error computing digest for: " + rsrc).initCause(t);
+                }
+            }
+
+            // finally compute and append the digest for the file contents
+            md.reset();
+            byte[] contents = data.toString().getBytes("UTF-8");
+            String filename = digestFile(version);
+            pout.println(filename + " = " + StringUtil.hexlate(md.digest(contents)));
+
+        } finally {
+            StreamUtil.close(pout);
+        }
+    }
+
+    /**
+     * Obtains an appropriate message digest instance for use by the Getdown system.
+     */
+    public static MessageDigest getMessageDigest ()
+    {
+        try {
+            return MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException nsae) {
+            throw new RuntimeException("JVM does not support MD5. Gurp!");
+        }
+    }
 
     /**
      * Creates a digest instance which will parse and validate the <code>digest.txt</code> in the
      * supplied application directory.
+     * @param version the version of the digest protocol to use.
      */
     public Digest (File appdir)
         throws IOException
     {
         // parse and validate our digest file contents
+        String filename = digestFile(VERSION);
         StringBuilder data = new StringBuilder();
-        File dfile = new File(appdir, DIGEST_FILE);
+        File dfile = new File(appdir, filename);
         for (String[] pair : ConfigUtil.parsePairs(dfile, false)) {
-            if (pair[0].equals(DIGEST_FILE)) {
+            if (pair[0].equals(filename)) {
                 _metaDigest = pair[1];
                 break;
             }
@@ -80,7 +139,7 @@ public class Digest
     public boolean validateResource (Resource resource, ProgressObserver obs)
     {
         try {
-            String cmd5 = resource.computeDigest(getMessageDigest(), obs);
+            String cmd5 = resource.computeDigest(VERSION, getMessageDigest(), obs);
             String emd5 = _digests.get(resource.getPath());
             if (cmd5.equals(emd5)) {
                 return true;
@@ -101,53 +160,6 @@ public class Digest
         return _digests.get(resource.getPath());
     }
 
-    /**
-     * Creates a digest file at the specified location using the supplied list of resources.
-     */
-    public static void createDigest (List<Resource> resources, File output)
-        throws IOException
-    {
-        MessageDigest md = getMessageDigest();
-        StringBuilder data = new StringBuilder();
-        PrintWriter pout = null;
-        try {
-            pout = new PrintWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
-
-            // compute and append the MD5 digest of each resource in the list
-            for (Resource rsrc : resources) {
-                String path = rsrc.getPath();
-                try {
-                    String digest = rsrc.computeDigest(md, null);
-                    note(data, path, digest);
-                    pout.println(path + " = " + digest);
-                } catch (Throwable t) {
-                    throw (IOException) new IOException(
-                        "Error computing digest for: " + rsrc).initCause(t);
-                }
-            }
-
-            // finally compute and append the digest for the file contents
-            md.reset();
-            byte[] contents = data.toString().getBytes("UTF-8");
-            pout.println(DIGEST_FILE + " = " + StringUtil.hexlate(md.digest(contents)));
-
-        } finally {
-            StreamUtil.close(pout);
-        }
-    }
-
-    /**
-     * Obtains an appropriate message digest instance for use by the Getdown system.
-     */
-    public static MessageDigest getMessageDigest ()
-    {
-        try {
-            return MessageDigest.getInstance("MD5");
-        } catch (NoSuchAlgorithmException nsae) {
-            throw new RuntimeException("JVM does not support MD5. Gurp!");
-        }
-    }
-
     /** Used by {@link #createDigest} and {@link Digest}. */
     protected static void note (StringBuilder data, String path, String digest)
     {
@@ -156,4 +168,7 @@ public class Digest
 
     protected HashMap<String, String> _digests = new HashMap<String, String>();
     protected String _metaDigest = "";
+
+    protected static final String FILE_NAME = "digest";
+    protected static final String FILE_SUFFIX = ".txt";
 }

src/main/java/com/threerings/getdown/data/Resource.java

@@ -27,6 +27,91 @@ import static com.threerings.getdown.Log.log;
  */
 public class Resource
 {
+    /**
+     * Computes the MD5 hash of the supplied file.
+     * @param version the version of the digest protocol to use.
+     */
+    public static String computeDigest (int version, File target, MessageDigest md,
+                                        ProgressObserver obs)
+        throws IOException
+    {
+        md.reset();
+        byte[] buffer = new byte[DIGEST_BUFFER_SIZE];
+        int read;
+
+        boolean isJar = isJar(target.getPath());
+        boolean isPacked200Jar = isPacked200Jar(target.getPath());
+
+        // if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic
+        // manner to properly correlate jardiff patched jars with their unpatched originals
+        if (isJar || isPacked200Jar){
+            File tmpJarFile = null;
+            JarFile jar = null;
+            try {
+                // if this is a compressed jar file, uncompress it to compute the jar file digest
+                if (isPacked200Jar){
+                    tmpJarFile = new File(target.getPath() + ".tmp");
+                    FileUtil.unpackPacked200Jar(target, tmpJarFile);
+                    jar = new JarFile(tmpJarFile);
+                } else{
+                    jar = new JarFile(target);
+                }
+
+                List<JarEntry> entries = Collections.list(jar.entries());
+                Collections.sort(entries, ENTRY_COMP);
+
+                int eidx = 0;
+                for (JarEntry entry : entries) {
+                    // old versions of the digest code skipped metadata
+                    if (version < 2) {
+                        if (entry.getName().startsWith("META-INF")) {
+                            updateProgress(obs, eidx, entries.size());
+                            continue;
+                        }
+                    }
+
+                    InputStream in = null;
+                    try {
+                        in = jar.getInputStream(entry);
+                        while ((read = in.read(buffer)) != -1) {
+                            md.update(buffer, 0, read);
+                        }
+                    } finally {
+                        StreamUtil.close(in);
+                    }
+                    updateProgress(obs, eidx, entries.size());
+                }
+
+            } finally {
+                if (jar != null) {
+                    try {
+                        jar.close();
+                    } catch (IOException ioe) {
+                        log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe);
+                    }
+                }
+                if (tmpJarFile != null) {
+                    tmpJarFile.delete();
+                }
+            }
+
+        } else {
+            long totalSize = target.length(), position = 0L;
+            FileInputStream fin = null;
+            try {
+                fin = new FileInputStream(target);
+                while ((read = fin.read(buffer)) != -1) {
+                    md.update(buffer, 0, read);
+                    position += read;
+                    updateProgress(obs, position, totalSize);
+                }
+            } finally {
+                StreamUtil.close(fin);
+            }
+        }
+        return StringUtil.hexlate(md.digest());
+    }
+
     /**
      * Creates a resource with the supplied remote URL and local path.
      */
@@ -111,8 +196,10 @@ public class Resource
     /**
      * Computes the MD5 hash of this resource's underlying file.
      * <em>Note:</em> This is both CPU and I/O intensive.
+     * @param version the version of the digest protocol to use.
      */
-    public String computeDigest (MessageDigest md, ProgressObserver obs) throws IOException
+    public String computeDigest (int version, MessageDigest md, ProgressObserver obs)
+        throws IOException
     {
         File file;
         if (_local.toString().toLowerCase().endsWith(Application.CONFIG_FILE)) {
@@ -120,7 +207,7 @@ public class Resource
         } else {
             file = _localNew.exists() ? _localNew : _local;
         }
-        return computeDigest(file, md, obs);
+        return computeDigest(version, file, md, obs);
     }
 
     /**
@@ -213,81 +300,6 @@ public class Resource
         return _path;
     }
 
-    /**
-     * Computes the MD5 hash of the supplied file.
-     */
-    public static String computeDigest (File target, MessageDigest md, ProgressObserver obs)
-        throws IOException
-    {
-        md.reset();
-        byte[] buffer = new byte[DIGEST_BUFFER_SIZE];
-        int read;
-
-        boolean isJar = isJar(target.getPath());
-        boolean isPacked200Jar = isPacked200Jar(target.getPath());
-
-        // if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic
-        // manner to properly correlate jardiff patched jars with their unpatched originals
-        if (isJar || isPacked200Jar){
-            File tmpJarFile = null;
-            JarFile jar = null;
-            try {
-                // if this is a compressed jar file, uncompress it to compute the jar file digest
-                if (isPacked200Jar){
-                    tmpJarFile = new File(target.getPath() + ".tmp");
-                    FileUtil.unpackPacked200Jar(target, tmpJarFile);
-                    jar = new JarFile(tmpJarFile);
-                } else{
-                    jar = new JarFile(target);
-                }
-
-                List<JarEntry> entries = Collections.list(jar.entries());
-                Collections.sort(entries, ENTRY_COMP);
-
-                int eidx = 0;
-                for (JarEntry entry : entries) {
-                    InputStream in = null;
-                    try {
-                        in = jar.getInputStream(entry);
-                        while ((read = in.read(buffer)) != -1) {
-                            md.update(buffer, 0, read);
-                        }
-                    } finally {
-                        StreamUtil.close(in);
-                    }
-                    updateProgress(obs, eidx, entries.size());
-                }
-
-            } finally {
-                if (jar != null) {
-                    try {
-                        jar.close();
-                    } catch (IOException ioe) {
-                        log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe);
-                    }
-                }
-                if (tmpJarFile != null) {
-                    tmpJarFile.delete();
-                }
-            }
-
-        } else {
-            long totalSize = target.length(), position = 0L;
-            FileInputStream fin = null;
-            try {
-                fin = new FileInputStream(target);
-                while ((read = fin.read(buffer)) != -1) {
-                    md.update(buffer, 0, read);
-                    position += read;
-                    updateProgress(obs, position, totalSize);
-                }
-            } finally {
-                StreamUtil.close(fin);
-            }
-        }
-        return StringUtil.hexlate(md.digest());
-    }
-
     /** Helper function to simplify the process of reporting progress. */
     protected static void updateProgress (ProgressObserver obs, long pos, long total)
     {

src/main/java/com/threerings/getdown/tools/Differ.java

@@ -107,8 +107,9 @@ public class Differ
                 Resource orsrc = (oidx == -1) ? null : orsrcs.remove(oidx);
                 if (orsrc != null) {
                     // first see if they are the same
-                    String odig = orsrc.computeDigest(md, null);
+                    int version = Digest.VERSION;
-                    String ndig = rsrc.computeDigest(md, null);
+                    String odig = orsrc.computeDigest(version, md, null);
+                    String ndig = rsrc.computeDigest(version, md, null);
                     if (odig.equals(ndig)) {
                         if (verbose) {
                             System.out.println("Unchanged: " + rsrc.getPath());

src/main/java/com/threerings/getdown/tools/Digester.java

@@ -36,24 +36,40 @@ public class Digester
     public static void main (String[] args)
         throws IOException, GeneralSecurityException
     {
-        if (args.length != 1 && args.length != 4) {
+        switch (args.length) {
+        case 1:
+            createDigests(new File(args[0]), null, null, null);
+            break;
+        case 4:
+            createDigests(new File(args[0]), new File(args[1]), args[2], args[3]);
+            break;
+        default:
             System.err.println("Usage: Digester app_dir [keystore_path password alias]");
             System.exit(255);
         }
+    }
 
-        createDigest(new File(args[0]));
+    /**
-        if (args.length == 4) {
+     * Creates digest file(s) and optionally signs them if {@code keystore} is not null.
-            signDigest(new File(args[0]), new File(args[1]), args[2], args[3]);
+     */
+    public static void createDigests (File appdir, File keystore, String password, String alias)
+        throws IOException, GeneralSecurityException
+    {
+        for (int version = 1; version <= Digest.VERSION; version++) {
+            createDigest(version, appdir);
+            if (keystore != null) {
+                signDigest(version, appdir, keystore, password, alias);
+            }
         }
     }
 
     /**
      * Creates a digest file in the specified application directory.
      */
-    public static void createDigest (File appdir)
+    public static void createDigest (int version, File appdir)
         throws IOException
     {
-        File target = new File(appdir, Digest.DIGEST_FILE);
+        File target = new File(appdir, Digest.digestFile(version));
         System.out.println("Generating digest file '" + target + "'...");
 
         // create our application and instruct it to parse its business
@@ -70,17 +86,19 @@ public class Digester
         }
 
         // now generate the digest file
-        Digest.createDigest(rsrcs, target);
+        Digest.createDigest(version, rsrcs, target);
     }
 
     /**
      * Creates a digest file in the specified application directory.
      */
-    public static void signDigest (File appdir, File storePath, String storePass, String storeAlias)
+    public static void signDigest (int version, File appdir,
+                                   File storePath, String storePass, String storeAlias)
         throws IOException, GeneralSecurityException
     {
-        File inputFile = new File(appdir, Digest.DIGEST_FILE);
+        String filename = Digest.digestFile(version);
-        File signatureFile = new File(appdir, Digest.DIGEST_FILE + Application.SIGNATURE_SUFFIX);
+        File inputFile = new File(appdir, filename);
+        File signatureFile = new File(appdir, filename + Application.SIGNATURE_SUFFIX);
 
         FileInputStream storeInput = null, dataInput = null;
         FileOutputStream signatureOutput = null;

src/main/java/com/threerings/getdown/tools/DigesterTask.java

@@ -13,6 +13,8 @@ import java.security.GeneralSecurityException;
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.Task;
 
+import com.threerings.getdown.data.Digest;
+
 /**
  * An ant task used to create a <code>digest.txt</code> for a Getdown
  * application deployment.
@@ -70,10 +72,7 @@ public class DigesterTask extends Task
         }
 
         try {
-            Digester.createDigest(_appdir);
+            Digester.createDigests(_appdir, _storepath, _storepass, _storealias);
-            if (_storepath != null) {
-                Digester.signDigest(_appdir, _storepath, _storepass, _storealias);
-            }
         } catch (IOException ioe) {
             throw new BuildException("Error creating digest: " + ioe.getMessage(), ioe);
         } catch (GeneralSecurityException gse) {