From d75d5a7a69631a22e2de281d369bd4ba18fa23e5 Mon Sep 17 00:00:00 2001 From: Michael Bayne Date: Fri, 4 Nov 2016 18:51:17 -0700 Subject: [PATCH] Create a separate digest.txt file for each "protocol version". Right now there's only two versions of the digester protocol, so I just always generate both of them. If we evolve the protocol further, we could conceivably allow you to specify a "minimum" protocol version, once you were confident that all of your clients had upgraded to a version of Getdown that used that minimum version. --- .../threerings/getdown/data/Application.java | 16 +- .../com/threerings/getdown/data/Digest.java | 119 +++++++------ .../com/threerings/getdown/data/Resource.java | 166 ++++++++++-------- .../com/threerings/getdown/tools/Differ.java | 5 +- .../threerings/getdown/tools/Digester.java | 38 ++-- .../getdown/tools/DigesterTask.java | 7 +- 6 files changed, 199 insertions(+), 152 deletions(-) diff --git a/src/main/java/com/threerings/getdown/data/Application.java b/src/main/java/com/threerings/getdown/data/Application.java index ce72f50..0c56e15 100644 --- a/src/main/java/com/threerings/getdown/data/Application.java +++ b/src/main/java/com/threerings/getdown/data/Application.java @@ -904,7 +904,7 @@ public class Application // now re-download our control files; we download the digest first so that if it fails, // our config file will still reference the old version and re-running the updater will // start the whole process over again - downloadDigestFile(); + downloadDigestFiles(); downloadConfigFile(); } catch (IOException ex) { @@ -1171,7 +1171,7 @@ public class Application String olddig = (_digest == null) ? "" : _digest.getMetaDigest(); try { status.updateStatus("m.checking"); - downloadDigestFile(); + downloadDigestFiles(); _digest = new Digest(_appdir); if (!olddig.equals(_digest.getMetaDigest())) { log.info("Unversioned digest changed. Revalidating..."); @@ -1189,7 +1189,7 @@ public class Application // exceptions to propagate up to the caller as there is nothing else we can do if (_digest == null) { status.updateStatus("m.updating_metadata"); - downloadDigestFile(); + downloadDigestFiles(); _digest = new Digest(_appdir); } @@ -1200,7 +1200,7 @@ public class Application // attempt to redownload both of our metadata files; again we pass errors up to our // caller because there's nothing we can do to automatically recover downloadConfigFile(); - downloadDigestFile(); + downloadDigestFiles(); _digest = new Digest(_appdir); // revalidate everything if we end up downloading new metadata clearValidationMarkers(); @@ -1459,13 +1459,15 @@ public class Application } /** - * Downloads a copy of Digest.DIGEST_FILE and validates its signature. + * Downloads the digest files and validates their signature. * @throws IOException */ - protected void downloadDigestFile () + protected void downloadDigestFiles () throws IOException { - downloadControlFile(Digest.DIGEST_FILE, true); + for (int version = 1; version <= Digest.VERSION; version++) { + downloadControlFile(Digest.digestFile(version), true); + } } /** diff --git a/src/main/java/com/threerings/getdown/data/Digest.java b/src/main/java/com/threerings/getdown/data/Digest.java index 9d30c6f..0de8dc3 100644 --- a/src/main/java/com/threerings/getdown/data/Digest.java +++ b/src/main/java/com/threerings/getdown/data/Digest.java @@ -30,21 +30,80 @@ import static com.threerings.getdown.Log.log; */ public class Digest { - /** The name of our MD5 digest file. */ - public static final String DIGEST_FILE = "digest.txt"; + /** The current version of the digest protocol. */ + public static final int VERSION = 2; + + /** + * Returns the name of the digest file for the specified protocol version. + */ + public static String digestFile (int version) { + String infix = version > 1 ? String.valueOf(version) : ""; + return FILE_NAME + infix + FILE_SUFFIX; + } + + /** + * Creates a digest file at the specified location using the supplied list of resources. + * @param version the version of the digest protocol to use. + */ + public static void createDigest (int version, List resources, File output) + throws IOException + { + MessageDigest md = getMessageDigest(); + StringBuilder data = new StringBuilder(); + PrintWriter pout = null; + try { + pout = new PrintWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8")); + + // compute and append the MD5 digest of each resource in the list + for (Resource rsrc : resources) { + String path = rsrc.getPath(); + try { + String digest = rsrc.computeDigest(version, md, null); + note(data, path, digest); + pout.println(path + " = " + digest); + } catch (Throwable t) { + throw (IOException) new IOException( + "Error computing digest for: " + rsrc).initCause(t); + } + } + + // finally compute and append the digest for the file contents + md.reset(); + byte[] contents = data.toString().getBytes("UTF-8"); + String filename = digestFile(version); + pout.println(filename + " = " + StringUtil.hexlate(md.digest(contents))); + + } finally { + StreamUtil.close(pout); + } + } + + /** + * Obtains an appropriate message digest instance for use by the Getdown system. + */ + public static MessageDigest getMessageDigest () + { + try { + return MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException nsae) { + throw new RuntimeException("JVM does not support MD5. Gurp!"); + } + } /** * Creates a digest instance which will parse and validate the digest.txt in the * supplied application directory. + * @param version the version of the digest protocol to use. */ public Digest (File appdir) throws IOException { // parse and validate our digest file contents + String filename = digestFile(VERSION); StringBuilder data = new StringBuilder(); - File dfile = new File(appdir, DIGEST_FILE); + File dfile = new File(appdir, filename); for (String[] pair : ConfigUtil.parsePairs(dfile, false)) { - if (pair[0].equals(DIGEST_FILE)) { + if (pair[0].equals(filename)) { _metaDigest = pair[1]; break; } @@ -80,7 +139,7 @@ public class Digest public boolean validateResource (Resource resource, ProgressObserver obs) { try { - String cmd5 = resource.computeDigest(getMessageDigest(), obs); + String cmd5 = resource.computeDigest(VERSION, getMessageDigest(), obs); String emd5 = _digests.get(resource.getPath()); if (cmd5.equals(emd5)) { return true; @@ -101,53 +160,6 @@ public class Digest return _digests.get(resource.getPath()); } - /** - * Creates a digest file at the specified location using the supplied list of resources. - */ - public static void createDigest (List resources, File output) - throws IOException - { - MessageDigest md = getMessageDigest(); - StringBuilder data = new StringBuilder(); - PrintWriter pout = null; - try { - pout = new PrintWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8")); - - // compute and append the MD5 digest of each resource in the list - for (Resource rsrc : resources) { - String path = rsrc.getPath(); - try { - String digest = rsrc.computeDigest(md, null); - note(data, path, digest); - pout.println(path + " = " + digest); - } catch (Throwable t) { - throw (IOException) new IOException( - "Error computing digest for: " + rsrc).initCause(t); - } - } - - // finally compute and append the digest for the file contents - md.reset(); - byte[] contents = data.toString().getBytes("UTF-8"); - pout.println(DIGEST_FILE + " = " + StringUtil.hexlate(md.digest(contents))); - - } finally { - StreamUtil.close(pout); - } - } - - /** - * Obtains an appropriate message digest instance for use by the Getdown system. - */ - public static MessageDigest getMessageDigest () - { - try { - return MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException nsae) { - throw new RuntimeException("JVM does not support MD5. Gurp!"); - } - } - /** Used by {@link #createDigest} and {@link Digest}. */ protected static void note (StringBuilder data, String path, String digest) { @@ -156,4 +168,7 @@ public class Digest protected HashMap _digests = new HashMap(); protected String _metaDigest = ""; + + protected static final String FILE_NAME = "digest"; + protected static final String FILE_SUFFIX = ".txt"; } diff --git a/src/main/java/com/threerings/getdown/data/Resource.java b/src/main/java/com/threerings/getdown/data/Resource.java index 928d33e..df18c52 100644 --- a/src/main/java/com/threerings/getdown/data/Resource.java +++ b/src/main/java/com/threerings/getdown/data/Resource.java @@ -27,6 +27,91 @@ import static com.threerings.getdown.Log.log; */ public class Resource { + /** + * Computes the MD5 hash of the supplied file. + * @param version the version of the digest protocol to use. + */ + public static String computeDigest (int version, File target, MessageDigest md, + ProgressObserver obs) + throws IOException + { + md.reset(); + byte[] buffer = new byte[DIGEST_BUFFER_SIZE]; + int read; + + boolean isJar = isJar(target.getPath()); + boolean isPacked200Jar = isPacked200Jar(target.getPath()); + + // if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic + // manner to properly correlate jardiff patched jars with their unpatched originals + if (isJar || isPacked200Jar){ + File tmpJarFile = null; + JarFile jar = null; + try { + // if this is a compressed jar file, uncompress it to compute the jar file digest + if (isPacked200Jar){ + tmpJarFile = new File(target.getPath() + ".tmp"); + FileUtil.unpackPacked200Jar(target, tmpJarFile); + jar = new JarFile(tmpJarFile); + } else{ + jar = new JarFile(target); + } + + List entries = Collections.list(jar.entries()); + Collections.sort(entries, ENTRY_COMP); + + int eidx = 0; + for (JarEntry entry : entries) { + // old versions of the digest code skipped metadata + if (version < 2) { + if (entry.getName().startsWith("META-INF")) { + updateProgress(obs, eidx, entries.size()); + continue; + } + } + + InputStream in = null; + try { + in = jar.getInputStream(entry); + while ((read = in.read(buffer)) != -1) { + md.update(buffer, 0, read); + } + } finally { + StreamUtil.close(in); + } + updateProgress(obs, eidx, entries.size()); + } + + } finally { + if (jar != null) { + try { + jar.close(); + } catch (IOException ioe) { + log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe); + } + } + if (tmpJarFile != null) { + tmpJarFile.delete(); + } + } + + } else { + long totalSize = target.length(), position = 0L; + FileInputStream fin = null; + try { + fin = new FileInputStream(target); + while ((read = fin.read(buffer)) != -1) { + md.update(buffer, 0, read); + position += read; + updateProgress(obs, position, totalSize); + } + } finally { + StreamUtil.close(fin); + } + } + return StringUtil.hexlate(md.digest()); + } + /** * Creates a resource with the supplied remote URL and local path. */ @@ -111,8 +196,10 @@ public class Resource /** * Computes the MD5 hash of this resource's underlying file. * Note: This is both CPU and I/O intensive. + * @param version the version of the digest protocol to use. */ - public String computeDigest (MessageDigest md, ProgressObserver obs) throws IOException + public String computeDigest (int version, MessageDigest md, ProgressObserver obs) + throws IOException { File file; if (_local.toString().toLowerCase().endsWith(Application.CONFIG_FILE)) { @@ -120,7 +207,7 @@ public class Resource } else { file = _localNew.exists() ? _localNew : _local; } - return computeDigest(file, md, obs); + return computeDigest(version, file, md, obs); } /** @@ -213,81 +300,6 @@ public class Resource return _path; } - /** - * Computes the MD5 hash of the supplied file. - */ - public static String computeDigest (File target, MessageDigest md, ProgressObserver obs) - throws IOException - { - md.reset(); - byte[] buffer = new byte[DIGEST_BUFFER_SIZE]; - int read; - - boolean isJar = isJar(target.getPath()); - boolean isPacked200Jar = isPacked200Jar(target.getPath()); - - // if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic - // manner to properly correlate jardiff patched jars with their unpatched originals - if (isJar || isPacked200Jar){ - File tmpJarFile = null; - JarFile jar = null; - try { - // if this is a compressed jar file, uncompress it to compute the jar file digest - if (isPacked200Jar){ - tmpJarFile = new File(target.getPath() + ".tmp"); - FileUtil.unpackPacked200Jar(target, tmpJarFile); - jar = new JarFile(tmpJarFile); - } else{ - jar = new JarFile(target); - } - - List entries = Collections.list(jar.entries()); - Collections.sort(entries, ENTRY_COMP); - - int eidx = 0; - for (JarEntry entry : entries) { - InputStream in = null; - try { - in = jar.getInputStream(entry); - while ((read = in.read(buffer)) != -1) { - md.update(buffer, 0, read); - } - } finally { - StreamUtil.close(in); - } - updateProgress(obs, eidx, entries.size()); - } - - } finally { - if (jar != null) { - try { - jar.close(); - } catch (IOException ioe) { - log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe); - } - } - if (tmpJarFile != null) { - tmpJarFile.delete(); - } - } - - } else { - long totalSize = target.length(), position = 0L; - FileInputStream fin = null; - try { - fin = new FileInputStream(target); - while ((read = fin.read(buffer)) != -1) { - md.update(buffer, 0, read); - position += read; - updateProgress(obs, position, totalSize); - } - } finally { - StreamUtil.close(fin); - } - } - return StringUtil.hexlate(md.digest()); - } - /** Helper function to simplify the process of reporting progress. */ protected static void updateProgress (ProgressObserver obs, long pos, long total) { diff --git a/src/main/java/com/threerings/getdown/tools/Differ.java b/src/main/java/com/threerings/getdown/tools/Differ.java index 28fe715..9a34b72 100644 --- a/src/main/java/com/threerings/getdown/tools/Differ.java +++ b/src/main/java/com/threerings/getdown/tools/Differ.java @@ -107,8 +107,9 @@ public class Differ Resource orsrc = (oidx == -1) ? null : orsrcs.remove(oidx); if (orsrc != null) { // first see if they are the same - String odig = orsrc.computeDigest(md, null); - String ndig = rsrc.computeDigest(md, null); + int version = Digest.VERSION; + String odig = orsrc.computeDigest(version, md, null); + String ndig = rsrc.computeDigest(version, md, null); if (odig.equals(ndig)) { if (verbose) { System.out.println("Unchanged: " + rsrc.getPath()); diff --git a/src/main/java/com/threerings/getdown/tools/Digester.java b/src/main/java/com/threerings/getdown/tools/Digester.java index 8ec2411..661abf5 100644 --- a/src/main/java/com/threerings/getdown/tools/Digester.java +++ b/src/main/java/com/threerings/getdown/tools/Digester.java @@ -36,24 +36,40 @@ public class Digester public static void main (String[] args) throws IOException, GeneralSecurityException { - if (args.length != 1 && args.length != 4) { + switch (args.length) { + case 1: + createDigests(new File(args[0]), null, null, null); + break; + case 4: + createDigests(new File(args[0]), new File(args[1]), args[2], args[3]); + break; + default: System.err.println("Usage: Digester app_dir [keystore_path password alias]"); System.exit(255); } + } - createDigest(new File(args[0])); - if (args.length == 4) { - signDigest(new File(args[0]), new File(args[1]), args[2], args[3]); + /** + * Creates digest file(s) and optionally signs them if {@code keystore} is not null. + */ + public static void createDigests (File appdir, File keystore, String password, String alias) + throws IOException, GeneralSecurityException + { + for (int version = 1; version <= Digest.VERSION; version++) { + createDigest(version, appdir); + if (keystore != null) { + signDigest(version, appdir, keystore, password, alias); + } } } /** * Creates a digest file in the specified application directory. */ - public static void createDigest (File appdir) + public static void createDigest (int version, File appdir) throws IOException { - File target = new File(appdir, Digest.DIGEST_FILE); + File target = new File(appdir, Digest.digestFile(version)); System.out.println("Generating digest file '" + target + "'..."); // create our application and instruct it to parse its business @@ -70,17 +86,19 @@ public class Digester } // now generate the digest file - Digest.createDigest(rsrcs, target); + Digest.createDigest(version, rsrcs, target); } /** * Creates a digest file in the specified application directory. */ - public static void signDigest (File appdir, File storePath, String storePass, String storeAlias) + public static void signDigest (int version, File appdir, + File storePath, String storePass, String storeAlias) throws IOException, GeneralSecurityException { - File inputFile = new File(appdir, Digest.DIGEST_FILE); - File signatureFile = new File(appdir, Digest.DIGEST_FILE + Application.SIGNATURE_SUFFIX); + String filename = Digest.digestFile(version); + File inputFile = new File(appdir, filename); + File signatureFile = new File(appdir, filename + Application.SIGNATURE_SUFFIX); FileInputStream storeInput = null, dataInput = null; FileOutputStream signatureOutput = null; diff --git a/src/main/java/com/threerings/getdown/tools/DigesterTask.java b/src/main/java/com/threerings/getdown/tools/DigesterTask.java index a059250..18bcdda 100644 --- a/src/main/java/com/threerings/getdown/tools/DigesterTask.java +++ b/src/main/java/com/threerings/getdown/tools/DigesterTask.java @@ -13,6 +13,8 @@ import java.security.GeneralSecurityException; import org.apache.tools.ant.BuildException; import org.apache.tools.ant.Task; +import com.threerings.getdown.data.Digest; + /** * An ant task used to create a digest.txt for a Getdown * application deployment. @@ -70,10 +72,7 @@ public class DigesterTask extends Task } try { - Digester.createDigest(_appdir); - if (_storepath != null) { - Digester.signDigest(_appdir, _storepath, _storepass, _storealias); - } + Digester.createDigests(_appdir, _storepath, _storepass, _storealias); } catch (IOException ioe) { throw new BuildException("Error creating digest: " + ioe.getMessage(), ioe); } catch (GeneralSecurityException gse) {