diff --git a/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java b/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java index a0dcf7b..05eb487 100644 --- a/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java +++ b/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java @@ -17,7 +17,6 @@ import com.threerings.getdown.data.Resource; public class ClassPaths { - static final String CACHE_DIR = ".cache"; /** * Builds either a default or cached classpath based on {@code app}'s configuration. @@ -48,23 +47,20 @@ public class ClassPaths */ public static ClassPath buildCachedClassPath (Application app) throws IOException { - File cacheDir = new File(app.getAppDir(), CACHE_DIR); + File codeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/code"); + // a negative value of code_cache_retention_days allows to clean up the cache forcefully - if (app.getCodeCacheRetentionDays() <= 0) { - runGarbageCollection(app, cacheDir); + if (app.getCodeCacheRetentionDays() != 0) { + runGarbageCollection(app, codeCacheDir); } - ResourceCache cache = new ResourceCache(cacheDir); - LinkedHashSet classPathEntries = new LinkedHashSet(); + ResourceCache cache = new ResourceCache(codeCacheDir); + LinkedHashSet classPathEntries = new LinkedHashSet<>(); for (Resource resource: app.getActiveCodeResources()) { File entry = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource)); classPathEntries.add(entry); } - if (app.getCodeCacheRetentionDays() > 0) { - runGarbageCollection(app, cacheDir); - } - return new ClassPath(classPathEntries); } diff --git a/core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java b/core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java new file mode 100644 index 0000000..9c72468 --- /dev/null +++ b/core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java @@ -0,0 +1,83 @@ +package com.threerings.getdown.classpath; + +import com.threerings.getdown.classpath.cache.GarbageCollector; +import com.threerings.getdown.classpath.cache.ResourceCache; +import com.threerings.getdown.data.Application; +import com.threerings.getdown.data.Resource; +import com.threerings.getdown.util.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.jar.JarFile; +import static com.threerings.getdown.Log.log; + +/** + * Similar to {@link ClassPath}, except used to represent directories containing native libraries instead. + */ +public class NativeLibPath extends ClassPath { + + public NativeLibPath(LinkedHashSet entries) { + super(entries); + } + + /** + * Builds a {@link NativeLibPath} instance by first caching all native jars (indicated by + * nresource=[native jar]), unpacking them, and referencing the locations of each of the unpacked files. + * Also performs garbage collection similar to {@link ClassPaths#buildCachedClassPath} + * + * @param app used to determine native jars and related information + * @param addCurrentLibraryPath if true, it adds the locations referenced by + * {@code System.getProperty("java.library.path")} as well + */ + public static NativeLibPath buildLibsPath(Application app, boolean addCurrentLibraryPath) throws IOException { + List resources = app.getNativeJars(); + String[] curPaths = System.getProperty("java.library.path").split(File.pathSeparator); + LinkedHashSet nativedirs = new LinkedHashSet<>(); + + File nativeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/native"); + ResourceCache cache = new ResourceCache(nativeCacheDir); + + // negative value forces total garbage collection, 0 avoids garbage collection at all + if (app.getCodeCacheRetentionDays() != 0) { + runGarbageCollection(app, nativeCacheDir); + } + + for (Resource resource : resources) { + // Use untruncated directory names because in the off chance that two native jars share a directory AND + // contain files with the same names, we'll get overwriting issues when unpacking + File cachedFile = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource), true); + + if (!getUnpackedIndicator(cachedFile).exists()) { + try { + FileUtil.unpackJar(new JarFile(cachedFile), cachedFile.getParentFile(), false); + getUnpackedIndicator(cachedFile).createNewFile(); + } catch (IOException ioe) { + log.warning("Failed to unpack native jar", "File", cachedFile.getAbsolutePath(), ioe); + // Keep going and unpack the other jars... + } + } + + nativedirs.add(cachedFile.getParentFile()); + } + + if (addCurrentLibraryPath) { + for (String path : curPaths) { + nativedirs.add(new File(path)); + } + } + + return new NativeLibPath(nativedirs); + } + + private static void runGarbageCollection(Application app, File nativeCacheDir) { + long retainMillis = TimeUnit.DAYS.toMillis(app.getCodeCacheRetentionDays()); + GarbageCollector.collectNative(nativeCacheDir, retainMillis); + } + + private static File getUnpackedIndicator(File cachedFile) { + return new File(cachedFile.getParent(), cachedFile.getName() + ".unpacked"); + } +} diff --git a/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java b/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java index d4d3e46..c4acd6d 100644 --- a/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java +++ b/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java @@ -1,6 +1,8 @@ package com.threerings.getdown.classpath.cache; import java.io.File; +import java.io.FileFilter; + import com.threerings.getdown.util.FileUtil; /** @@ -40,6 +42,42 @@ public class GarbageCollector }); } + /** + * Collect and delete garbage in the native cache. It tries to find a jar file with a matching + * lastmodified file, and deletes the entire directory accordingly. + */ + public static void collectNative (File cacheDir, final long retentionPeriodMillis) { + FileUtil.walkDirectChildren(cacheDir, new FileUtil.Visitor() { + @Override + public void visit(File file) { + if (file.isDirectory()) { + // Get all the native jars in the directory (there should only be one though) + File[] nativejars = file.listFiles(new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname.getAbsolutePath().endsWith(".jar"); + } + }); + for (File nativejar : nativejars) { + if (nativejar==null || !nativejar.exists()) { + continue; + } + + File cachedFile = getCachedFile(nativejar); + File lastAccessedFile = getLastAccessedFile(nativejar); + + if (!cachedFile.exists() || !lastAccessedFile.exists() || shouldDelete(lastAccessedFile, retentionPeriodMillis)) { + FileUtil.deleteDirHarder(file); + } + } + } else { + // @TODO There shouldn't be any loose files in native/ but if there are then what? Delete them? + // file.delete(); + } + } + }); + } + private static boolean shouldDelete (File lastAccessedFile, long retentionMillis) { return System.currentTimeMillis() - lastAccessedFile.lastModified() > retentionMillis; diff --git a/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java b/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java index 6e35b3f..d517841 100644 --- a/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java +++ b/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java @@ -26,13 +26,34 @@ public class ResourceCache } /** - * Caches the given file under it's {@code digest}. - * + * Caches the given file under it's {@code digest}. Same as calling {@link #cacheFile(File, String, boolean)} + * with {@code false} * @return the cached file */ public File cacheFile (File fileToCache, String digest) throws IOException { - File cacheLocation = new File(_cacheDir, digest.substring(0, 2)); + return cacheFile(fileToCache, digest, false); + } + + /** + * Caches the given file under it's {@code digest}. + * @param fileToCache file to cache + * @param digest used to determine the name of the directory to store file in + * @param useFullHashName if true, the name of the cache directory will not be truncated. + * This is useful if you need to avoid the possibility of two files + * sharing a directory. + * @return the cached file + */ + public File cacheFile (File fileToCache, String digest, boolean useFullHashName) throws IOException + { + File cacheLocation; + + if (useFullHashName) { + cacheLocation = new File(_cacheDir, digest); + } else { + cacheLocation = new File(_cacheDir, digest.substring(0, 2)); + + } createDirectoryIfNecessary(cacheLocation); File cachedFile = new File(cacheLocation, digest + getFileSuffix(fileToCache)); diff --git a/core/src/main/java/com/threerings/getdown/data/Application.java b/core/src/main/java/com/threerings/getdown/data/Application.java index 5fca7fb..66bffbe 100644 --- a/core/src/main/java/com/threerings/getdown/data/Application.java +++ b/core/src/main/java/com/threerings/getdown/data/Application.java @@ -25,6 +25,7 @@ import java.util.zip.GZIPInputStream; import com.threerings.getdown.classpath.ClassPaths; import com.threerings.getdown.classpath.ClassPath; +import com.threerings.getdown.classpath.NativeLibPath; import com.threerings.getdown.util.*; // avoid ambiguity with java.util.Base64 which we can't use as it's 1.8+ import com.threerings.getdown.util.Base64; @@ -54,6 +55,9 @@ public class Application /** A special classname that means 'use -jar code.jar' instead of a classname. */ public static final String MANIFEST_CLASS = "manifest"; + /** Name of directory to store cached files in. */ + public static final String CACHE_DIR = ".cache"; + /** Used to communicate information about the UI displayed when updating the application. */ public static final class UpdateInterface { @@ -374,6 +378,22 @@ public class Application return codes; } + /** + * Returns all jar resources indicated to contain native library files (.dll, .so etc.) + */ + public List getNativeJars() { + + List nativeJars = new ArrayList<>(); + + for (Resource resource: _resources) { + if (resource.isNativeJar()) { + nativeJars.add(resource); + } + } + + return nativeJars; + } + /** * Returns all non-code resources and all resources from active auxiliary resource groups. */ @@ -673,6 +693,7 @@ public class Application parseResources(config, "uresource", Resource.UNPACK, _resources); parseResources(config, "xresource", Resource.EXEC, _resources); parseResources(config, "presource", Resource.PRELOAD, _resources); + parseResources(config, "nresource", Resource.NATIVE, _resources); // parse our auxiliary resource groups for (String auxgroup : config.getList("auxgroups")) { @@ -915,6 +936,10 @@ public class Application // add the -classpath arguments if we're not in -jar mode ClassPath classPath = ClassPaths.buildClassPath(this); + // get the -Djava.library.path value to pass + // @TODO optional getdown.txt parameter to set addCurrentLibraryPath to true or false? + NativeLibPath nativeLibPaths = NativeLibPath.buildLibsPath(this, true); + if (!dashJarMode) { args.add("-classpath"); args.add(classPath.asArgumentString()); @@ -938,6 +963,9 @@ public class Application // add the marker indicating the app is running in getdown args.add("-D" + Properties.GETDOWN + "=true"); + // set the native library path + args.add("-Djava.library.path=" + nativeLibPaths.asArgumentString()); + // pass along any pass-through arguments for (Map.Entry entry : System.getProperties().entrySet()) { String key = (String)entry.getKey(); diff --git a/core/src/main/java/com/threerings/getdown/data/Resource.java b/core/src/main/java/com/threerings/getdown/data/Resource.java index 5e426cf..9d6a040 100644 --- a/core/src/main/java/com/threerings/getdown/data/Resource.java +++ b/core/src/main/java/com/threerings/getdown/data/Resource.java @@ -37,13 +37,16 @@ public class Resource implements Comparable /** Indicates that the resource should be marked executable. */ EXEC, /** Indicates that the resource should be downloaded before a UI is displayed. */ - PRELOAD + PRELOAD, + /** Indicates that the resource is a jar containing native libs. */ + NATIVE }; public static final EnumSet NORMAL = EnumSet.noneOf(Attr.class); public static final EnumSet UNPACK = EnumSet.of(Attr.UNPACK); public static final EnumSet EXEC = EnumSet.of(Attr.EXEC); public static final EnumSet PRELOAD = EnumSet.of(Attr.PRELOAD); + public static final EnumSet NATIVE = EnumSet.of(Attr.NATIVE); /** * Computes the MD5 hash of the supplied file. @@ -212,6 +215,15 @@ public class Resource implements Comparable return _attrs.contains(Attr.PRELOAD); } + /** + * Returns true if this resource is a native lib jar. + */ + public boolean isNativeJar () + { + return _attrs.contains(Attr.NATIVE); + } + + /** * Computes the MD5 hash of this resource's underlying file. * Note: This is both CPU and I/O intensive. diff --git a/core/src/main/java/com/threerings/getdown/util/FileUtil.java b/core/src/main/java/com/threerings/getdown/util/FileUtil.java index 378f701..d3325af 100644 --- a/core/src/main/java/com/threerings/getdown/util/FileUtil.java +++ b/core/src/main/java/com/threerings/getdown/util/FileUtil.java @@ -77,6 +77,27 @@ public class FileUtil return deleted; } + /** + * Force deletes {@code file} and all of its children recursively using {@link #deleteHarder}. + * Note that some children may still be deleted even if {@code false} is returned. Also, since {@link #deleteHarder} + * is used, the {@code file} could be deleted once the jvm exits even if {@code false} is returned. + * + * @param file file to delete + * @return true iff {@code file} was successfully deleted. + */ + public static boolean deleteDirHarder(File file) { + if (file.isDirectory()) { + for (File child : file.listFiles()) { + if (child.isDirectory()) { + deleteDirHarder(child); + } else { + deleteHarder(child); + } + } + } + return deleteHarder(file); + } + /** * Reads the contents of the supplied input stream into a list of lines. Closes the reader on * successful or failed completion. @@ -190,7 +211,7 @@ public class FileUtil } /** - * Used by {@link #walkTree}. + * Used by {@link #walkTree} and {@link #walkDirectChildren}. */ public interface Visitor { @@ -218,4 +239,21 @@ public class FileUtil } } } + + /** + * Walks all direct sub-files of {@code root}, calling {@code visitor} on each file. + */ + public static void walkDirectChildren (final File root, Visitor visitor) { + File[] children = root.listFiles(); + + if (children == null) return; + Deque stack = new ArrayDeque<>(Arrays.asList(children)); + while (!stack.isEmpty()) { + File currentFile = stack.pop(); + if (currentFile.exists()) { + visitor.visit(currentFile); + } + } + } + } diff --git a/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java b/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java index be6b32b..05b47a1 100644 --- a/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java +++ b/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java @@ -46,10 +46,10 @@ public class ClassPathsTest when(_application.getCodeCacheRetentionDays()).thenReturn(1); Path firstCachedJarFile = _appdir.getRoot().toPath(). - resolve(ClassPaths.CACHE_DIR).resolve("fi").resolve("first.jar"); + resolve(Application.CACHE_DIR + "/code").resolve("fi").resolve("first.jar"); Path secondCachedJarFile = _appdir.getRoot().toPath(). - resolve(ClassPaths.CACHE_DIR).resolve("se").resolve("second.jar"); + resolve(Application.CACHE_DIR + "/code").resolve("se").resolve("second.jar"); String expectedClassPath = firstCachedJarFile.toAbsolutePath() + File.pathSeparator + secondCachedJarFile.toAbsolutePath();