Merge pull request #210 from sergiorussia/feature/zip-files-support

zip files supported as resources
This commit is contained in:
Michael Bayne
2019-05-09 09:48:10 -07:00
committed by GitHub
10 changed files with 205 additions and 152 deletions
@@ -6,6 +6,8 @@
package com.threerings.getdown.cache; package com.threerings.getdown.cache;
import java.io.File; import java.io.File;
import com.threerings.getdown.data.Resource;
import com.threerings.getdown.util.FileUtil; import com.threerings.getdown.util.FileUtil;
/** /**
@@ -55,9 +57,9 @@ public class GarbageCollector
if (subdirs != null) { if (subdirs != null) {
for (File dir : subdirs) { for (File dir : subdirs) {
if (dir.isDirectory()) { if (dir.isDirectory()) {
// Get all the native jars in the directory (there should only be one) // Get all the native jars or zips in the directory (there should only be one)
for (File file : dir.listFiles()) { for (File file : dir.listFiles()) {
if (!file.getName().endsWith(".jar")) { if (!Resource.isJar(file) && !Resource.isZip(file)) {
continue; continue;
} }
File cachedFile = getCachedFile(file); File cachedFile = getCachedFile(file);
@@ -10,7 +10,7 @@ import java.io.IOException;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.jar.JarFile; import java.util.zip.ZipFile;
import com.threerings.getdown.cache.GarbageCollector; import com.threerings.getdown.cache.GarbageCollector;
import com.threerings.getdown.cache.ResourceCache; import com.threerings.getdown.cache.ResourceCache;
@@ -112,7 +112,7 @@ public class PathBuilder
if (!unpackedIndicator.exists()) { if (!unpackedIndicator.exists()) {
try { try {
FileUtil.unpackJar(new JarFile(cachedFile), cachedParent, false); FileUtil.unpackJar(new ZipFile(cachedFile), cachedParent, false);
unpackedIndicator.createNewFile(); unpackedIndicator.createNewFile();
} catch (IOException ioe) { } catch (IOException ioe) {
log.warning("Failed to unpack native jar", log.warning("Failed to unpack native jar",
@@ -5,7 +5,10 @@
package com.threerings.getdown.data; package com.threerings.getdown.data;
import java.io.*; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL; import java.net.URL;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.util.Collections; import java.util.Collections;
@@ -13,13 +16,12 @@ import java.util.Comparator;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.jar.JarEntry; import java.util.zip.ZipEntry;
import java.util.jar.JarFile; import java.util.zip.ZipFile;
import com.threerings.getdown.util.FileUtil; import com.threerings.getdown.util.FileUtil;
import com.threerings.getdown.util.ProgressObserver; import com.threerings.getdown.util.ProgressObserver;
import com.threerings.getdown.util.StringUtil; import com.threerings.getdown.util.StringUtil;
import static com.threerings.getdown.Log.log; import static com.threerings.getdown.Log.log;
/** /**
@@ -60,29 +62,29 @@ public class Resource implements Comparable<Resource>
byte[] buffer = new byte[DIGEST_BUFFER_SIZE]; byte[] buffer = new byte[DIGEST_BUFFER_SIZE];
int read; int read;
boolean isJar = isJar(target.getPath()); boolean isZip = isJar(target) || isZip(target); // jar is a zip too
boolean isPacked200Jar = isPacked200Jar(target.getPath()); boolean isPacked200Jar = isPacked200Jar(target);
// if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic // if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic
// manner to properly correlate jardiff patched jars with their unpatched originals // manner to properly correlate jardiff patched jars with their unpatched originals
if (isJar || isPacked200Jar){ if (isPacked200Jar || isZip){
File tmpJarFile = null; File tmpJarFile = null;
JarFile jar = null; ZipFile zip = null;
try { try {
// if this is a compressed jar file, uncompress it to compute the jar file digest // if this is a compressed zip file, uncompress it to compute the zip file digest
if (isPacked200Jar){ if (isPacked200Jar){
tmpJarFile = new File(target.getPath() + ".tmp"); tmpJarFile = new File(target.getPath() + ".tmp");
FileUtil.unpackPacked200Jar(target, tmpJarFile); tmpJarFile.deleteOnExit();
jar = new JarFile(tmpJarFile); zip = FileUtil.unpackPacked200Jar(target, tmpJarFile);
} else{ } else{
jar = new JarFile(target); zip = new ZipFile(target);
} }
List<JarEntry> entries = Collections.list(jar.entries()); List<? extends ZipEntry> entries = Collections.list(zip.entries());
Collections.sort(entries, ENTRY_COMP); Collections.sort(entries, ENTRY_COMP);
int eidx = 0; int eidx = 0;
for (JarEntry entry : entries) { for (ZipEntry entry : entries) {
// old versions of the digest code skipped metadata // old versions of the digest code skipped metadata
if (version < 2) { if (version < 2) {
if (entry.getName().startsWith("META-INF")) { if (entry.getName().startsWith("META-INF")) {
@@ -91,7 +93,7 @@ public class Resource implements Comparable<Resource>
} }
} }
try (InputStream in = jar.getInputStream(entry)) { try (InputStream in = zip.getInputStream(entry)) {
while ((read = in.read(buffer)) != -1) { while ((read = in.read(buffer)) != -1) {
md.update(buffer, 0, read); md.update(buffer, 0, read);
} }
@@ -101,11 +103,11 @@ public class Resource implements Comparable<Resource>
} }
} finally { } finally {
if (jar != null) { if (zip != null) {
try { try {
jar.close(); zip.close();
} catch (IOException ioe) { } catch (IOException ioe) {
log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe); log.warning("Error closing zip", "path", target, "zip", zip, "error", ioe);
} }
} }
if (tmpJarFile != null) { if (tmpJarFile != null) {
@@ -135,14 +137,13 @@ public class Resource implements Comparable<Resource>
_remote = remote; _remote = remote;
_local = local; _local = local;
_localNew = new File(local.toString() + "_new"); _localNew = new File(local.toString() + "_new");
String lpath = _local.getPath(); _marker = new File(_local.getPath() + "v");
_marker = new File(lpath + "v");
_attrs = attrs; _attrs = attrs;
_isJar = isJar(lpath); _isZip = isJar(local) || isZip(local);
_isPacked200Jar = isPacked200Jar(lpath); _isPacked200Jar = isPacked200Jar(local);
boolean unpack = attrs.contains(Attr.UNPACK); boolean unpack = attrs.contains(Attr.UNPACK);
if (unpack && _isJar) { if (unpack && _isZip) {
_unpacked = _local.getParentFile(); _unpacked = _local.getParentFile();
} else if(unpack && _isPacked200Jar) { } else if(unpack && _isPacked200Jar) {
String dotJar = ".jar", lname = _local.getName(); String dotJar = ".jar", lname = _local.getName();
@@ -298,11 +299,11 @@ public class Resource implements Comparable<Resource>
public void unpack () throws IOException public void unpack () throws IOException
{ {
// sanity check // sanity check
if (!_isJar && !_isPacked200Jar) { if (!_isZip && !_isPacked200Jar) {
throw new IOException("Requested to unpack non-jar file '" + _local + "'."); throw new IOException("Requested to unpack non-jar file '" + _local + "'.");
} }
if (_isJar) { if (_isZip) {
try (JarFile jar = new JarFile(_local)) { try (ZipFile jar = new ZipFile(_local)) {
FileUtil.unpackJar(jar, _unpacked, _attrs.contains(Attr.CLEAN)); FileUtil.unpackJar(jar, _unpacked, _attrs.contains(Attr.CLEAN));
} }
} else { } else {
@@ -366,26 +367,34 @@ public class Resource implements Comparable<Resource>
} }
} }
protected static boolean isJar (String path) public static boolean isJar (File file)
{ {
String path = file.getName();
return path.endsWith(".jar") || path.endsWith(".jar_new"); return path.endsWith(".jar") || path.endsWith(".jar_new");
} }
protected static boolean isPacked200Jar (String path) public static boolean isPacked200Jar (File file)
{ {
String path = file.getName();
return path.endsWith(".jar.pack") || path.endsWith(".jar.pack_new") || return path.endsWith(".jar.pack") || path.endsWith(".jar.pack_new") ||
path.endsWith(".jar.pack.gz")|| path.endsWith(".jar.pack.gz_new"); path.endsWith(".jar.pack.gz")|| path.endsWith(".jar.pack.gz_new");
} }
public static boolean isZip (File file)
{
String path = file.getName();
return path.endsWith(".zip") || path.endsWith(".zip_new");
}
protected String _path; protected String _path;
protected URL _remote; protected URL _remote;
protected File _local, _localNew, _marker, _unpacked; protected File _local, _localNew, _marker, _unpacked;
protected EnumSet<Attr> _attrs; protected EnumSet<Attr> _attrs;
protected boolean _isJar, _isPacked200Jar; protected boolean _isZip, _isPacked200Jar;
/** Used to sort the entries in a jar file. */ /** Used to sort the entries in a jar file. */
protected static final Comparator<JarEntry> ENTRY_COMP = new Comparator<JarEntry>() { protected static final Comparator<ZipEntry> ENTRY_COMP = new Comparator<ZipEntry>() {
@Override public int compare (JarEntry e1, JarEntry e2) { @Override public int compare (ZipEntry e1, ZipEntry e2) {
return e1.getName().compareTo(e2.getName()); return e1.getName().compareTo(e2.getName());
} }
}; };
@@ -11,15 +11,12 @@ import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.security.MessageDigest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.security.MessageDigest; import java.util.zip.ZipOutputStream;
import com.threerings.getdown.data.Application; import com.threerings.getdown.data.Application;
import com.threerings.getdown.data.Digest; import com.threerings.getdown.data.Digest;
@@ -99,7 +96,7 @@ public class Differ
MessageDigest md = Digest.getMessageDigest(version); MessageDigest md = Digest.getMessageDigest(version);
try (FileOutputStream fos = new FileOutputStream(patch); try (FileOutputStream fos = new FileOutputStream(patch);
BufferedOutputStream buffered = new BufferedOutputStream(fos); BufferedOutputStream buffered = new BufferedOutputStream(fos);
JarOutputStream jout = new JarOutputStream(buffered)) { ZipOutputStream jout = new ZipOutputStream(buffered)) {
// for each file in the new application, it either already exists // for each file in the new application, it either already exists
// in the old application, or it is new // in the old application, or it is new
@@ -172,13 +169,13 @@ public class Differ
throws IOException throws IOException
{ {
File temp = File.createTempFile("differ", "jar"); File temp = File.createTempFile("differ", "jar");
try (JarFile jar = new JarFile(target); try (ZipFile jar = new ZipFile(target);
FileOutputStream tempFos = new FileOutputStream(temp); FileOutputStream tempFos = new FileOutputStream(temp);
BufferedOutputStream tempBos = new BufferedOutputStream(tempFos); BufferedOutputStream tempBos = new BufferedOutputStream(tempFos);
JarOutputStream jout = new JarOutputStream(tempBos)) { ZipOutputStream jout = new ZipOutputStream(tempBos)) {
byte[] buffer = new byte[4096]; byte[] buffer = new byte[4096];
for (Enumeration< JarEntry > iter = jar.entries(); iter.hasMoreElements();) { for (Enumeration<? extends ZipEntry> iter = jar.entries(); iter.hasMoreElements();) {
JarEntry entry = iter.nextElement(); ZipEntry entry = iter.nextElement();
entry.setCompressedSize(-1); entry.setCompressedSize(-1);
jout.putNextEntry(entry); jout.putNextEntry(entry);
try (InputStream in = jar.getInputStream(entry)) { try (InputStream in = jar.getInputStream(entry)) {
@@ -193,7 +190,7 @@ public class Differ
return temp; return temp;
} }
protected void jarDiff (File ofile, File nfile, JarOutputStream jout) protected void jarDiff (File ofile, File nfile, ZipOutputStream jout)
throws IOException throws IOException
{ {
JarDiff.createPatch(ofile.getPath(), nfile.getPath(), jout, false); JarDiff.createPatch(ofile.getPath(), nfile.getPath(), jout, false);
@@ -222,8 +219,7 @@ public class Differ
} }
} }
protected static void pipe (File file, JarOutputStream jout) protected static void pipe (File file, ZipOutputStream jout) throws IOException
throws IOException
{ {
try (FileInputStream fin = new FileInputStream(file)) { try (FileInputStream fin = new FileInputStream(file)) {
StreamUtil.copy(fin, jout); StreamUtil.copy(fin, jout);
@@ -41,9 +41,17 @@
package com.threerings.getdown.tools; package com.threerings.getdown.tools;
import java.io.*; import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.io.Writer;
import java.util.*; import java.util.*;
import java.util.jar.*; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
@@ -86,7 +94,7 @@ public class JarDiff implements JarDiffCodes
// and new.jar ) // and new.jar )
// and for files that cannot be implicitly moved, we will either // and for files that cannot be implicitly moved, we will either
// find out whether it is moved or new (modified) // find out whether it is moved or new (modified)
for (JarEntry newEntry : newJar) { for (ZipEntry newEntry : newJar) {
String newname = newEntry.getName(); String newname = newEntry.getName();
// Return best match of contents, will return a name match if possible // Return best match of contents, will return a name match if possible
@@ -154,7 +162,7 @@ public class JarDiff implements JarDiffCodes
// SECOND PASS: <deleted files> = <oldjarnames> - <implicitmoves> - // SECOND PASS: <deleted files> = <oldjarnames> - <implicitmoves> -
// <source of move commands> - <new or modified entries> // <source of move commands> - <new or modified entries>
ArrayList<String> deleted = new ArrayList<>(); ArrayList<String> deleted = new ArrayList<>();
for (JarEntry oldEntry : oldJar) { for (ZipEntry oldEntry : oldJar) {
String oldName = oldEntry.getName(); String oldName = oldEntry.getName();
if (!implicit.contains(oldName) && !moveSrc.contains(oldName) if (!implicit.contains(oldName) && !moveSrc.contains(oldName)
&& !newEntries.contains(oldName)) { && !newEntries.contains(oldName)) {
@@ -180,7 +188,7 @@ public class JarDiff implements JarDiffCodes
} }
} }
JarOutputStream jos = new JarOutputStream(os); ZipOutputStream jos = new ZipOutputStream(os);
// Write out all the MOVEs and REMOVEs // Write out all the MOVEs and REMOVEs
createIndex(jos, deleted, moved); createIndex(jos, deleted, moved);
@@ -203,7 +211,7 @@ public class JarDiff implements JarDiffCodes
* <code>oldEntries</code> gives the names of the files that were removed, * <code>oldEntries</code> gives the names of the files that were removed,
* <code>movedMap</code> maps from the new name to the old name. * <code>movedMap</code> maps from the new name to the old name.
*/ */
private static void createIndex (JarOutputStream jos, List<String> oldEntries, private static void createIndex (ZipOutputStream jos, List<String> oldEntries,
Map<String,String> movedMap) Map<String,String> movedMap)
throws IOException throws IOException
{ {
@@ -220,17 +228,17 @@ public class JarDiff implements JarDiffCodes
} }
// And those that have moved // And those that have moved
for (String newName : movedMap.keySet()) { for (Map.Entry<String, String> entry : movedMap.entrySet()) {
String oldName = movedMap.get(newName); String oldName = entry.getValue();
writer.write(MOVE_COMMAND); writer.write(MOVE_COMMAND);
writer.write(" "); writer.write(" ");
writeEscapedString(writer, oldName); writeEscapedString(writer, oldName);
writer.write(" "); writer.write(" ");
writeEscapedString(writer, newName); writeEscapedString(writer, entry.getKey());
writer.write("\r\n"); writer.write("\r\n");
} }
jos.putNextEntry(new JarEntry(INDEX_NAME)); jos.putNextEntry(new ZipEntry(INDEX_NAME));
byte[] bytes = writer.toString().getBytes(UTF_8); byte[] bytes = writer.toString().getBytes(UTF_8);
jos.write(bytes, 0, bytes.length); jos.write(bytes, 0, bytes.length);
} }
@@ -264,7 +272,7 @@ public class JarDiff implements JarDiffCodes
return writer; return writer;
} }
private static void writeEntry (JarOutputStream jos, JarEntry entry, JarFile2 file) private static void writeEntry (ZipOutputStream jos, ZipEntry entry, JarFile2 file)
throws IOException throws IOException
{ {
try (InputStream data = file.getJarFile().getInputStream(entry)) { try (InputStream data = file.getJarFile().getInputStream(entry)) {
@@ -278,31 +286,31 @@ public class JarDiff implements JarDiffCodes
} }
/** /**
* JarFile2 wraps a JarFile providing some convenience methods. * JarFile2 wraps a ZipFile providing some convenience methods.
*/ */
private static class JarFile2 implements Iterable<JarEntry>, Closeable private static class JarFile2 implements Iterable<ZipEntry>, Closeable
{ {
private JarFile _jar; private ZipFile _jar;
private List<JarEntry> _entries; private List<ZipEntry> _entries;
private HashMap<String,JarEntry> _nameToEntryMap; private HashMap<String,ZipEntry> _nameToEntryMap;
private HashMap<Long,LinkedList<JarEntry>> _crcToEntryMap; private HashMap<Long,LinkedList<ZipEntry>> _crcToEntryMap;
public JarFile2 (String path) throws IOException { public JarFile2 (String path) throws IOException {
_jar = new JarFile(new File(path)); _jar = new ZipFile(new File(path));
index(); index();
} }
public JarFile getJarFile () { public ZipFile getJarFile () {
return _jar; return _jar;
} }
// from interface Iterable<JarEntry> // from interface Iterable<ZipEntry>
@Override @Override
public Iterator<JarEntry> iterator () { public Iterator<ZipEntry> iterator () {
return _entries.iterator(); return _entries.iterator();
} }
public JarEntry getEntryByName (String name) { public ZipEntry getEntryByName (String name) {
return _nameToEntryMap.get(name); return _nameToEntryMap.get(name);
} }
@@ -350,7 +358,7 @@ public class JarDiff implements JarDiffCodes
return retVal; return retVal;
} }
public String getBestMatch (JarFile2 file, JarEntry entry) throws IOException { public String getBestMatch (JarFile2 file, ZipEntry entry) throws IOException {
// check for same name and same content, return name if found // check for same name and same content, return name if found
if (contains(file, entry)) { if (contains(file, entry)) {
return (entry.getName()); return (entry.getName());
@@ -360,9 +368,8 @@ public class JarDiff implements JarDiffCodes
return (hasSameContent(file,entry)); return (hasSameContent(file,entry));
} }
public boolean contains (JarFile2 f, JarEntry e) throws IOException { public boolean contains (JarFile2 f, ZipEntry e) throws IOException {
ZipEntry thisEntry = getEntryByName(e.getName());
JarEntry thisEntry = getEntryByName(e.getName());
// Look up name in 'this' Jar2File - if not exist return false // Look up name in 'this' Jar2File - if not exist return false
if (thisEntry == null) if (thisEntry == null)
@@ -379,17 +386,17 @@ public class JarDiff implements JarDiffCodes
} }
} }
public String hasSameContent (JarFile2 file, JarEntry entry) throws IOException { public String hasSameContent (JarFile2 file, ZipEntry entry) throws IOException {
String thisName = null; String thisName = null;
Long crcL = Long.valueOf(entry.getCrc()); Long crcL = Long.valueOf(entry.getCrc());
// check if this jar contains files with the passed in entry's crc // check if this jar contains files with the passed in entry's crc
if (_crcToEntryMap.containsKey(crcL)) { if (_crcToEntryMap.containsKey(crcL)) {
// get the Linked List with files with the crc // get the Linked List with files with the crc
LinkedList<JarEntry> ll = _crcToEntryMap.get(crcL); LinkedList<ZipEntry> ll = _crcToEntryMap.get(crcL);
// go through the list and check for content match // go through the list and check for content match
ListIterator<JarEntry> li = ll.listIterator(0); ListIterator<ZipEntry> li = ll.listIterator(0);
while (li.hasNext()) { while (li.hasNext()) {
JarEntry thisEntry = li.next(); ZipEntry thisEntry = li.next();
// check for content match // check for content match
try (InputStream oldIS = getJarFile().getInputStream(thisEntry); try (InputStream oldIS = getJarFile().getInputStream(thisEntry);
InputStream newIS = file.getJarFile().getInputStream(entry)) { InputStream newIS = file.getJarFile().getInputStream(entry)) {
@@ -404,7 +411,7 @@ public class JarDiff implements JarDiffCodes
} }
private void index () throws IOException { private void index () throws IOException {
Enumeration<JarEntry> entries = _jar.entries(); Enumeration<? extends ZipEntry> entries = _jar.entries();
_nameToEntryMap = new HashMap<>(); _nameToEntryMap = new HashMap<>();
_crcToEntryMap = new HashMap<>(); _crcToEntryMap = new HashMap<>();
@@ -414,7 +421,7 @@ public class JarDiff implements JarDiffCodes
} }
if (entries != null) { if (entries != null) {
while (entries.hasMoreElements()) { while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement(); ZipEntry entry = entries.nextElement();
long crc = entry.getCrc(); long crc = entry.getCrc();
Long crcL = Long.valueOf(crc); Long crcL = Long.valueOf(crc);
if (_debug) { if (_debug) {
@@ -427,13 +434,13 @@ public class JarDiff implements JarDiffCodes
// generate the CRC to entries map // generate the CRC to entries map
if (_crcToEntryMap.containsKey(crcL)) { if (_crcToEntryMap.containsKey(crcL)) {
// key exist, add the entry to the correcponding linked list // key exist, add the entry to the correcponding linked list
LinkedList<JarEntry> ll = _crcToEntryMap.get(crcL); LinkedList<ZipEntry> ll = _crcToEntryMap.get(crcL);
ll.add(entry); ll.add(entry);
_crcToEntryMap.put(crcL, ll); _crcToEntryMap.put(crcL, ll);
} else { } else {
// create a new entry in the hashmap for the new key // create a new entry in the hashmap for the new key
LinkedList<JarEntry> ll = new LinkedList<JarEntry>(); LinkedList<ZipEntry> ll = new LinkedList<>();
ll.add(entry); ll.add(entry);
_crcToEntryMap.put(crcL, ll); _crcToEntryMap.put(crcL, ll);
} }
@@ -11,22 +11,18 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.LineNumberReader; import java.io.LineNumberReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.jar.JarEntry; import java.util.zip.ZipFile;
import java.util.jar.JarFile; import java.util.zip.ZipOutputStream;
import java.util.jar.JarOutputStream;
import com.threerings.getdown.util.ProgressObserver; import com.threerings.getdown.util.ProgressObserver;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
/** /**
@@ -49,11 +45,9 @@ public class JarDiffPatcher implements JarDiffCodes
throws IOException throws IOException
{ {
File oldFile = new File(jarPath), diffFile = new File(diffPath); File oldFile = new File(jarPath), diffFile = new File(diffPath);
try (ZipFile oldJar = new ZipFile(oldFile);
try (JarFile oldJar = new JarFile(oldFile); ZipFile jarDiff = new ZipFile(diffFile);
JarFile jarDiff = new JarFile(diffFile); ZipOutputStream jos = new ZipOutputStream(new FileOutputStream(target))) {
JarOutputStream jos = new JarOutputStream(new FileOutputStream(target))) {
Set<String> ignoreSet = new HashSet<>(); Set<String> ignoreSet = new HashSet<>();
Map<String, String> renameMap = new HashMap<>(); Map<String, String> renameMap = new HashMap<>();
determineNameMapping(jarDiff, ignoreSet, renameMap); determineNameMapping(jarDiff, ignoreSet, renameMap);
@@ -63,7 +57,7 @@ public class JarDiffPatcher implements JarDiffCodes
// Files to implicit move // Files to implicit move
Set<String> oldjarNames = new HashSet<>(); Set<String> oldjarNames = new HashSet<>();
Enumeration<JarEntry> oldEntries = oldJar.entries(); Enumeration<? extends ZipEntry> oldEntries = oldJar.entries();
if (oldEntries != null) { if (oldEntries != null) {
while (oldEntries.hasMoreElements()) { while (oldEntries.hasMoreElements()) {
oldjarNames.add(oldEntries.nextElement().getName()); oldjarNames.add(oldEntries.nextElement().getName());
@@ -83,10 +77,10 @@ public class JarDiffPatcher implements JarDiffCodes
size -= ignoreSet.size(); size -= ignoreSet.size();
// Add content from JARDiff // Add content from JARDiff
Enumeration<JarEntry> entries = jarDiff.entries(); Enumeration<? extends ZipEntry> entries = jarDiff.entries();
if (entries != null) { if (entries != null) {
while (entries.hasMoreElements()) { while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement(); ZipEntry entry = entries.nextElement();
if (!INDEX_NAME.equals(entry.getName())) { if (!INDEX_NAME.equals(entry.getName())) {
updateObserver(observer, currentEntry, size); updateObserver(observer, currentEntry, size);
currentEntry++; currentEntry++;
@@ -114,15 +108,15 @@ public class JarDiffPatcher implements JarDiffCodes
// Apply move <oldName> <newName> command // Apply move <oldName> <newName> command
String oldName = renameMap.get(newName); String oldName = renameMap.get(newName);
// Get source JarEntry // Get source ZipEntry
JarEntry oldEntry = oldJar.getJarEntry(oldName); ZipEntry oldEntry = oldJar.getEntry(oldName);
if (oldEntry == null) { if (oldEntry == null) {
String moveCmd = MOVE_COMMAND + oldName + " " + newName; String moveCmd = MOVE_COMMAND + oldName + " " + newName;
throw new IOException("error.badmove: " + moveCmd); throw new IOException("error.badmove: " + moveCmd);
} }
// Create dest JarEntry // Create dest ZipEntry
JarEntry newEntry = new JarEntry(newName); ZipEntry newEntry = new ZipEntry(newName);
newEntry.setTime(oldEntry.getTime()); newEntry.setTime(oldEntry.getTime());
newEntry.setSize(oldEntry.getSize()); newEntry.setSize(oldEntry.getSize());
newEntry.setCompressedSize(oldEntry.getCompressedSize()); newEntry.setCompressedSize(oldEntry.getCompressedSize());
@@ -149,19 +143,15 @@ public class JarDiffPatcher implements JarDiffCodes
} }
// implicit move // implicit move
Iterator<String> iEntries = oldjarNames.iterator(); for (String name : oldjarNames) {
if (iEntries != null) { ZipEntry entry = oldJar.getEntry(name);
while (iEntries.hasNext()) { if (entry == null) {
String name = iEntries.next(); // names originally retrieved from the JAR, so this should never happen
JarEntry entry = oldJar.getJarEntry(name); throw new AssertionError("JAR entry not found: " + name);
if (entry == null) {
// names originally retrieved from the JAR, so this should never happen
throw new AssertionError("JAR entry not found: " + name);
}
updateObserver(observer, currentEntry, size);
currentEntry++;
writeEntry(jos, entry, oldJar);
} }
updateObserver(observer, currentEntry, size);
currentEntry++;
writeEntry(jos, entry, oldJar);
} }
updateObserver(observer, currentEntry, size); updateObserver(observer, currentEntry, size);
} }
@@ -175,7 +165,7 @@ public class JarDiffPatcher implements JarDiffCodes
} }
protected void determineNameMapping ( protected void determineNameMapping (
JarFile jarDiff, Set<String> ignoreSet, Map<String, String> renameMap) ZipFile jarDiff, Set<String> ignoreSet, Map<String, String> renameMap)
throws IOException throws IOException
{ {
InputStream is = jarDiff.getInputStream(jarDiff.getEntry(INDEX_NAME)); InputStream is = jarDiff.getInputStream(jarDiff.getEntry(INDEX_NAME));
@@ -264,7 +254,7 @@ public class JarDiffPatcher implements JarDiffCodes
return sub; return sub;
} }
protected void writeEntry (JarOutputStream jos, JarEntry entry, JarFile file) protected void writeEntry (ZipOutputStream jos, ZipEntry entry, ZipFile file)
throws IOException throws IOException
{ {
try (InputStream data = file.getInputStream(entry)) { try (InputStream data = file.getInputStream(entry)) {
@@ -272,10 +262,10 @@ public class JarDiffPatcher implements JarDiffCodes
} }
} }
protected void writeEntry (JarOutputStream jos, JarEntry entry, InputStream data) protected void writeEntry (ZipOutputStream jos, ZipEntry entry, InputStream data)
throws IOException throws IOException
{ {
jos.putNextEntry(new JarEntry(entry.getName())); jos.putNextEntry(new ZipEntry(entry.getName()));
// Read the entry // Read the entry
int size = data.read(newBytes); int size = data.read(newBytes);
@@ -9,16 +9,13 @@ import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import com.threerings.getdown.util.FileUtil; import com.threerings.getdown.util.FileUtil;
import com.threerings.getdown.util.ProgressObserver; import com.threerings.getdown.util.ProgressObserver;
import com.threerings.getdown.util.StreamUtil; import com.threerings.getdown.util.StreamUtil;
import static com.threerings.getdown.Log.log; import static com.threerings.getdown.Log.log;
/** /**
@@ -55,10 +52,10 @@ public class Patcher
_obs = obs; _obs = obs;
_plength = patch.length(); _plength = patch.length();
try (JarFile file = new JarFile(patch)) { try (ZipFile file = new ZipFile(patch)) {
Enumeration<JarEntry> entries = file.entries(); // old skool! Enumeration<? extends ZipEntry> entries = file.entries();
while (entries.hasMoreElements()) { while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement(); ZipEntry entry = entries.nextElement();
String path = entry.getName(); String path = entry.getName();
long elength = entry.getCompressedSize(); long elength = entry.getCompressedSize();
@@ -96,7 +93,7 @@ public class Patcher
return path.substring(0, path.length() - suffix.length()); return path.substring(0, path.length() - suffix.length());
} }
protected void createFile (JarFile file, ZipEntry entry, File target) protected void createFile (ZipFile file, ZipEntry entry, File target)
{ {
// create our copy buffer if necessary // create our copy buffer if necessary
if (_buffer == null) { if (_buffer == null) {
@@ -124,7 +121,7 @@ public class Patcher
} }
} }
protected void patchFile (JarFile file, ZipEntry entry, protected void patchFile (ZipFile file, ZipEntry entry,
File appdir, String path) File appdir, String path)
{ {
File target = new File(appdir, path); File target = new File(appdir, path);
@@ -5,10 +5,26 @@
package com.threerings.getdown.util; package com.threerings.getdown.util;
import java.io.*; import java.io.BufferedOutputStream;
import java.util.*; import java.io.BufferedReader;
import java.util.jar.*; import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.Enumeration;
import java.util.List;
import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;
import java.util.jar.Pack200;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import com.threerings.getdown.Log; import com.threerings.getdown.Log;
import static com.threerings.getdown.Log.log; import static com.threerings.getdown.Log.log;
@@ -16,7 +32,7 @@ import static com.threerings.getdown.Log.log;
/** /**
* File related utilities. * File related utilities.
*/ */
public class FileUtil public final class FileUtil
{ {
/** /**
* Gets the specified source file to the specified destination file by hook or crook. Windows * Gets the specified source file to the specified destination file by hook or crook. Windows
@@ -114,13 +130,12 @@ public class FileUtil
* @param cleanExistingDirs if true, all files in all directories contained in {@code jar} will * @param cleanExistingDirs if true, all files in all directories contained in {@code jar} will
* be deleted prior to unpacking the jar. * be deleted prior to unpacking the jar.
*/ */
public static void unpackJar (JarFile jar, File target, boolean cleanExistingDirs) public static void unpackJar (ZipFile jar, File target, boolean cleanExistingDirs) throws IOException
throws IOException
{ {
if (cleanExistingDirs) { if (cleanExistingDirs) {
Enumeration<?> entries = jar.entries(); Enumeration<? extends ZipEntry> entries = jar.entries();
while (entries.hasMoreElements()) { while (entries.hasMoreElements()) {
JarEntry entry = (JarEntry)entries.nextElement(); ZipEntry entry = entries.nextElement();
if (entry.isDirectory()) { if (entry.isDirectory()) {
File efile = new File(target, entry.getName()); File efile = new File(target, entry.getName());
if (efile.exists()) { if (efile.exists()) {
@@ -133,9 +148,9 @@ public class FileUtil
} }
} }
Enumeration<?> entries = jar.entries(); Enumeration<? extends ZipEntry> entries = jar.entries();
while (entries.hasMoreElements()) { while (entries.hasMoreElements()) {
JarEntry entry = (JarEntry)entries.nextElement(); ZipEntry entry = entries.nextElement();
File efile = new File(target, entry.getName()); File efile = new File(target, entry.getName());
// if we're unpacking a normal jar file, it will have special path // if we're unpacking a normal jar file, it will have special path
@@ -169,10 +184,10 @@ public class FileUtil
* Unpacks a pack200 packed jar file from {@code packedJar} into {@code target}. If {@code * Unpacks a pack200 packed jar file from {@code packedJar} into {@code target}. If {@code
* packedJar} has a {@code .gz} extension, it will be gunzipped first. * packedJar} has a {@code .gz} extension, it will be gunzipped first.
*/ */
public static void unpackPacked200Jar (File packedJar, File target) throws IOException public static JarFile unpackPacked200Jar (File packedJar, File target) throws IOException
{ {
try (InputStream packJarIn = new FileInputStream(packedJar); try (InputStream packJarIn = new FileInputStream(packedJar);
JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(target))) { JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(target))) {
boolean gz = (packedJar.getName().endsWith(".gz") || boolean gz = (packedJar.getName().endsWith(".gz") ||
packedJar.getName().endsWith(".gz_new")); packedJar.getName().endsWith(".gz_new"));
try (InputStream packJarIn2 = (gz ? new GZIPInputStream(packJarIn) : packJarIn)) { try (InputStream packJarIn2 = (gz ? new GZIPInputStream(packJarIn) : packJarIn)) {
@@ -180,6 +195,7 @@ public class FileUtil
unpacker.unpack(packJarIn2, jarOut); unpacker.unpack(packJarIn2, jarOut);
} }
} }
return new JarFile(target);
} }
/** /**
@@ -7,23 +7,41 @@ package com.threerings.getdown.cache;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.junit.*; import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder; import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import static org.junit.Assert.*; import org.junit.runners.Parameterized;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue; import static org.junit.Assume.assumeTrue;
/** /**
* Validates that cache garbage is collected and deleted correctly. * Validates that cache garbage is collected and deleted correctly.
*/ */
@RunWith(Parameterized.class)
public class GarbageCollectorTest public class GarbageCollectorTest
{ {
@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{ ".jar" },
{ ".zip" }
});
}
@Parameterized.Parameter
public String extension;
@Before public void setupFiles () throws IOException @Before public void setupFiles () throws IOException
{ {
_cachedFile = _folder.newFile("abc123.jar"); _cachedFile = _folder.newFile("abc123" + extension);
_lastAccessedFile = _folder.newFile("abc123.jar" + ResourceCache.LAST_ACCESSED_FILE_SUFFIX); _lastAccessedFile = _folder.newFile("abc123" + extension + ResourceCache.LAST_ACCESSED_FILE_SUFFIX);
} }
@Test public void shouldDeleteCacheEntryIfRetentionPeriodIsReached () @Test public void shouldDeleteCacheEntryIfRetentionPeriodIsReached ()
@@ -7,26 +7,44 @@ package com.threerings.getdown.cache;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.junit.*; import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder; import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import static org.junit.Assert.*; import org.junit.runners.Parameterized;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/** /**
* Asserts the correct functionality of the {@link ResourceCache}. * Asserts the correct functionality of the {@link ResourceCache}.
*/ */
@RunWith(Parameterized.class)
public class ResourceCacheTest public class ResourceCacheTest
{ {
@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{ ".jar" },
{ ".zip" }
});
}
@Parameterized.Parameter
public String extension;
@Before public void setupCache () throws IOException { @Before public void setupCache () throws IOException {
_fileToCache = _folder.newFile("filetocache.jar"); _fileToCache = _folder.newFile("filetocache" + extension);
_cache = new ResourceCache(_folder.newFolder(".cache")); _cache = new ResourceCache(_folder.newFolder(".cache"));
} }
@Test public void shouldCacheFile () throws IOException @Test public void shouldCacheFile () throws IOException
{ {
assertEquals("abc123.jar", cacheFile().getName()); assertEquals("abc123" + extension, cacheFile().getName());
} }
private File cacheFile() throws IOException private File cacheFile() throws IOException
@@ -36,7 +54,7 @@ public class ResourceCacheTest
@Test public void shouldTrackFileUsage () throws IOException @Test public void shouldTrackFileUsage () throws IOException
{ {
String name = "abc123.jar" + ResourceCache.LAST_ACCESSED_FILE_SUFFIX; String name = "abc123" + extension + ResourceCache.LAST_ACCESSED_FILE_SUFFIX;
File lastAccessedFile = new File(cacheFile().getParentFile(), name); File lastAccessedFile = new File(cacheFile().getParentFile(), name);
assertTrue(lastAccessedFile.exists()); assertTrue(lastAccessedFile.exists());
} }