AfflictionNetworks/getdown
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create a separate digest.txt file for each "protocol version".

Browse Source 
Right now there's only two versions of the digester protocol, so I just
always generate both of them. If we evolve the protocol further, we
could conceivably allow you to specify a "minimum" protocol version,
once you were confident that all of your clients had upgraded to a
version of Getdown that used that minimum version.
This commit is contained in:
Michael Bayne
2016-11-04 18:51:17 -07:00
parent 7c651fbd73
commit d75d5a7a69
6 changed files with 199 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/threerings/getdown/data/Application.java
+9 -7
View File
@@ -904,7 +904,7 @@ public class Application
// now re-download our control files; we download the digest first so that if it fails,
// our config file will still reference the old version and re-running the updater will
// start the whole process over again
downloadDigestFile();
downloadDigestFiles();
downloadConfigFile();
} catch (IOException ex) {
@@ -1171,7 +1171,7 @@ public class Application
String olddig = (_digest == null) ? "" : _digest.getMetaDigest();
try {
status.updateStatus("m.checking");
downloadDigestFile();
downloadDigestFiles();
_digest = new Digest(_appdir);
if (!olddig.equals(_digest.getMetaDigest())) {
log.info("Unversioned digest changed. Revalidating...");
@@ -1189,7 +1189,7 @@ public class Application
// exceptions to propagate up to the caller as there is nothing else we can do
if (_digest == null) {
status.updateStatus("m.updating_metadata");
downloadDigestFile();
downloadDigestFiles();
_digest = new Digest(_appdir);
}
@@ -1200,7 +1200,7 @@ public class Application
// attempt to redownload both of our metadata files; again we pass errors up to our
// caller because there's nothing we can do to automatically recover
downloadConfigFile();
downloadDigestFile();
downloadDigestFiles();
_digest = new Digest(_appdir);
// revalidate everything if we end up downloading new metadata
clearValidationMarkers();
@@ -1459,13 +1459,15 @@ public class Application
}
/**
* Downloads a copy of Digest.DIGEST_FILE and validates its signature.
* Downloads the digest files and validates their signature.
* @throws IOException
*/
protected void downloadDigestFile ()
protected void downloadDigestFiles ()
throws IOException
{
downloadControlFile(Digest.DIGEST_FILE, true);
for (int version = 1; version <= Digest.VERSION; version++) {
downloadControlFile(Digest.digestFile(version), true);
}
}
/**
src/main/java/com/threerings/getdown/data/Digest.java
+67 -52
View File
@@ -30,21 +30,80 @@ import static com.threerings.getdown.Log.log;
*/
public class Digest
{
/** The name of our MD5 digest file. */
public static final String DIGEST_FILE = "digest.txt";
/** The current version of the digest protocol. */
public static final int VERSION = 2;
/**
* Returns the name of the digest file for the specified protocol version.
*/
public static String digestFile (int version) {
String infix = version > 1 ? String.valueOf(version) : "";
return FILE_NAME + infix + FILE_SUFFIX;
}
/**
* Creates a digest file at the specified location using the supplied list of resources.
* @param version the version of the digest protocol to use.
*/
public static void createDigest (int version, List<Resource> resources, File output)
throws IOException
{
MessageDigest md = getMessageDigest();
StringBuilder data = new StringBuilder();
PrintWriter pout = null;
try {
pout = new PrintWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
// compute and append the MD5 digest of each resource in the list
for (Resource rsrc : resources) {
String path = rsrc.getPath();
try {
String digest = rsrc.computeDigest(version, md, null);
note(data, path, digest);
pout.println(path + " = " + digest);
} catch (Throwable t) {
throw (IOException) new IOException(
"Error computing digest for: " + rsrc).initCause(t);
}
}
// finally compute and append the digest for the file contents
md.reset();
byte[] contents = data.toString().getBytes("UTF-8");
String filename = digestFile(version);
pout.println(filename + " = " + StringUtil.hexlate(md.digest(contents)));
} finally {
StreamUtil.close(pout);
}
}
/**
* Obtains an appropriate message digest instance for use by the Getdown system.
*/
public static MessageDigest getMessageDigest ()
{
try {
return MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException nsae) {
throw new RuntimeException("JVM does not support MD5. Gurp!");
}
}
/**
* Creates a digest instance which will parse and validate the <code>digest.txt</code> in the
* supplied application directory.
* @param version the version of the digest protocol to use.
*/
public Digest (File appdir)
throws IOException
{
// parse and validate our digest file contents
String filename = digestFile(VERSION);
StringBuilder data = new StringBuilder();
File dfile = new File(appdir, DIGEST_FILE);
File dfile = new File(appdir, filename);
for (String[] pair : ConfigUtil.parsePairs(dfile, false)) {
if (pair[0].equals(DIGEST_FILE)) {
if (pair[0].equals(filename)) {
_metaDigest = pair[1];
break;
}
@@ -80,7 +139,7 @@ public class Digest
public boolean validateResource (Resource resource, ProgressObserver obs)
{
try {
String cmd5 = resource.computeDigest(getMessageDigest(), obs);
String cmd5 = resource.computeDigest(VERSION, getMessageDigest(), obs);
String emd5 = _digests.get(resource.getPath());
if (cmd5.equals(emd5)) {
return true;
@@ -101,53 +160,6 @@ public class Digest
return _digests.get(resource.getPath());
}
/**
* Creates a digest file at the specified location using the supplied list of resources.
*/
public static void createDigest (List<Resource> resources, File output)
throws IOException
{
MessageDigest md = getMessageDigest();
StringBuilder data = new StringBuilder();
PrintWriter pout = null;
try {
pout = new PrintWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
// compute and append the MD5 digest of each resource in the list
for (Resource rsrc : resources) {
String path = rsrc.getPath();
try {
String digest = rsrc.computeDigest(md, null);
note(data, path, digest);
pout.println(path + " = " + digest);
} catch (Throwable t) {
throw (IOException) new IOException(
"Error computing digest for: " + rsrc).initCause(t);
}
}
// finally compute and append the digest for the file contents
md.reset();
byte[] contents = data.toString().getBytes("UTF-8");
pout.println(DIGEST_FILE + " = " + StringUtil.hexlate(md.digest(contents)));
} finally {
StreamUtil.close(pout);
}
}
/**
* Obtains an appropriate message digest instance for use by the Getdown system.
*/
public static MessageDigest getMessageDigest ()
{
try {
return MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException nsae) {
throw new RuntimeException("JVM does not support MD5. Gurp!");
}
}
/** Used by {@link #createDigest} and {@link Digest}. */
protected static void note (StringBuilder data, String path, String digest)
{
@@ -156,4 +168,7 @@ public class Digest
protected HashMap<String, String> _digests = new HashMap<String, String>();
protected String _metaDigest = "";
protected static final String FILE_NAME = "digest";
protected static final String FILE_SUFFIX = ".txt";
}
src/main/java/com/threerings/getdown/data/Resource.java
+89 -77
View File
@@ -27,6 +27,91 @@ import static com.threerings.getdown.Log.log;
*/
public class Resource
{
/**
* Computes the MD5 hash of the supplied file.
* @param version the version of the digest protocol to use.
*/
public static String computeDigest (int version, File target, MessageDigest md,
ProgressObserver obs)
throws IOException
{
md.reset();
byte[] buffer = new byte[DIGEST_BUFFER_SIZE];
int read;
boolean isJar = isJar(target.getPath());
boolean isPacked200Jar = isPacked200Jar(target.getPath());
// if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic
// manner to properly correlate jardiff patched jars with their unpatched originals
if (isJar || isPacked200Jar){
File tmpJarFile = null;
JarFile jar = null;
try {
// if this is a compressed jar file, uncompress it to compute the jar file digest
if (isPacked200Jar){
tmpJarFile = new File(target.getPath() + ".tmp");
FileUtil.unpackPacked200Jar(target, tmpJarFile);
jar = new JarFile(tmpJarFile);
} else{
jar = new JarFile(target);
}
List<JarEntry> entries = Collections.list(jar.entries());
Collections.sort(entries, ENTRY_COMP);
int eidx = 0;
for (JarEntry entry : entries) {
// old versions of the digest code skipped metadata
if (version < 2) {
if (entry.getName().startsWith("META-INF")) {
updateProgress(obs, eidx, entries.size());
continue;
}
}
InputStream in = null;
try {
in = jar.getInputStream(entry);
while ((read = in.read(buffer)) != -1) {
md.update(buffer, 0, read);
}
} finally {
StreamUtil.close(in);
}
updateProgress(obs, eidx, entries.size());
}
} finally {
if (jar != null) {
try {
jar.close();
} catch (IOException ioe) {
log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe);
}
}
if (tmpJarFile != null) {
tmpJarFile.delete();
}
}
} else {
long totalSize = target.length(), position = 0L;
FileInputStream fin = null;
try {
fin = new FileInputStream(target);
while ((read = fin.read(buffer)) != -1) {
md.update(buffer, 0, read);
position += read;
updateProgress(obs, position, totalSize);
}
} finally {
StreamUtil.close(fin);
}
}
return StringUtil.hexlate(md.digest());
}
/**
* Creates a resource with the supplied remote URL and local path.
*/
@@ -111,8 +196,10 @@ public class Resource
/**
* Computes the MD5 hash of this resource's underlying file.
* <em>Note:</em> This is both CPU and I/O intensive.
* @param version the version of the digest protocol to use.
*/
public String computeDigest (MessageDigest md, ProgressObserver obs) throws IOException
public String computeDigest (int version, MessageDigest md, ProgressObserver obs)
throws IOException
{
File file;
if (_local.toString().toLowerCase().endsWith(Application.CONFIG_FILE)) {
@@ -120,7 +207,7 @@ public class Resource
} else {
file = _localNew.exists() ? _localNew : _local;
}
return computeDigest(file, md, obs);
return computeDigest(version, file, md, obs);
}
/**
@@ -213,81 +300,6 @@ public class Resource
return _path;
}
/**
* Computes the MD5 hash of the supplied file.
*/
public static String computeDigest (File target, MessageDigest md, ProgressObserver obs)
throws IOException
{
md.reset();
byte[] buffer = new byte[DIGEST_BUFFER_SIZE];
int read;
boolean isJar = isJar(target.getPath());
boolean isPacked200Jar = isPacked200Jar(target.getPath());
// if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic
// manner to properly correlate jardiff patched jars with their unpatched originals
if (isJar || isPacked200Jar){
File tmpJarFile = null;
JarFile jar = null;
try {
// if this is a compressed jar file, uncompress it to compute the jar file digest
if (isPacked200Jar){
tmpJarFile = new File(target.getPath() + ".tmp");
FileUtil.unpackPacked200Jar(target, tmpJarFile);
jar = new JarFile(tmpJarFile);
} else{
jar = new JarFile(target);
}
List<JarEntry> entries = Collections.list(jar.entries());
Collections.sort(entries, ENTRY_COMP);
int eidx = 0;
for (JarEntry entry : entries) {
InputStream in = null;
try {
in = jar.getInputStream(entry);
while ((read = in.read(buffer)) != -1) {
md.update(buffer, 0, read);
}
} finally {
StreamUtil.close(in);
}
updateProgress(obs, eidx, entries.size());
}
} finally {
if (jar != null) {
try {
jar.close();
} catch (IOException ioe) {
log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe);
}
}
if (tmpJarFile != null) {
tmpJarFile.delete();
}
}
} else {
long totalSize = target.length(), position = 0L;
FileInputStream fin = null;
try {
fin = new FileInputStream(target);
while ((read = fin.read(buffer)) != -1) {
md.update(buffer, 0, read);
position += read;
updateProgress(obs, position, totalSize);
}
} finally {
StreamUtil.close(fin);
}
}
return StringUtil.hexlate(md.digest());
}
/** Helper function to simplify the process of reporting progress. */
protected static void updateProgress (ProgressObserver obs, long pos, long total)
{
src/main/java/com/threerings/getdown/tools/Differ.java
+3 -2
View File
@@ -107,8 +107,9 @@ public class Differ
Resource orsrc = (oidx == -1) ? null : orsrcs.remove(oidx);
if (orsrc != null) {
// first see if they are the same
String odig = orsrc.computeDigest(md, null);
String ndig = rsrc.computeDigest(md, null);
int version = Digest.VERSION;
String odig = orsrc.computeDigest(version, md, null);
String ndig = rsrc.computeDigest(version, md, null);
if (odig.equals(ndig)) {
if (verbose) {
System.out.println("Unchanged: " + rsrc.getPath());
src/main/java/com/threerings/getdown/tools/Digester.java
+28 -10
View File
@@ -36,24 +36,40 @@ public class Digester
public static void main (String[] args)
throws IOException, GeneralSecurityException
{
if (args.length != 1 && args.length != 4) {
switch (args.length) {
case 1:
createDigests(new File(args[0]), null, null, null);
break;
case 4:
createDigests(new File(args[0]), new File(args[1]), args[2], args[3]);
break;
default:
System.err.println("Usage: Digester app_dir [keystore_path password alias]");
System.exit(255);
}
}
createDigest(new File(args[0]));
if (args.length == 4) {
signDigest(new File(args[0]), new File(args[1]), args[2], args[3]);
/**
* Creates digest file(s) and optionally signs them if {@code keystore} is not null.
*/
public static void createDigests (File appdir, File keystore, String password, String alias)
throws IOException, GeneralSecurityException
{
for (int version = 1; version <= Digest.VERSION; version++) {
createDigest(version, appdir);
if (keystore != null) {
signDigest(version, appdir, keystore, password, alias);
}
}
}
/**
* Creates a digest file in the specified application directory.
*/
public static void createDigest (File appdir)
public static void createDigest (int version, File appdir)
throws IOException
{
File target = new File(appdir, Digest.DIGEST_FILE);
File target = new File(appdir, Digest.digestFile(version));
System.out.println("Generating digest file '" + target + "'...");
// create our application and instruct it to parse its business
@@ -70,17 +86,19 @@ public class Digester
}
// now generate the digest file
Digest.createDigest(rsrcs, target);
Digest.createDigest(version, rsrcs, target);
}
/**
* Creates a digest file in the specified application directory.
*/
public static void signDigest (File appdir, File storePath, String storePass, String storeAlias)
public static void signDigest (int version, File appdir,
File storePath, String storePass, String storeAlias)
throws IOException, GeneralSecurityException
{
File inputFile = new File(appdir, Digest.DIGEST_FILE);
File signatureFile = new File(appdir, Digest.DIGEST_FILE + Application.SIGNATURE_SUFFIX);
String filename = Digest.digestFile(version);
File inputFile = new File(appdir, filename);
File signatureFile = new File(appdir, filename + Application.SIGNATURE_SUFFIX);
FileInputStream storeInput = null, dataInput = null;
FileOutputStream signatureOutput = null;
src/main/java/com/threerings/getdown/tools/DigesterTask.java
+3 -4
View File
@@ -13,6 +13,8 @@ import java.security.GeneralSecurityException;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Task;
import com.threerings.getdown.data.Digest;
/**
* An ant task used to create a <code>digest.txt</code> for a Getdown
* application deployment.
@@ -70,10 +72,7 @@ public class DigesterTask extends Task
}
try {
Digester.createDigest(_appdir);
if (_storepath != null) {
Digester.signDigest(_appdir, _storepath, _storepass, _storealias);
}
Digester.createDigests(_appdir, _storepath, _storepass, _storealias);
} catch (IOException ioe) {
throw new BuildException("Error creating digest: " + ioe.getMessage(), ioe);
} catch (GeneralSecurityException gse) {