From 86723193e6f66b56a61925bcd4896a644c3c2ac9 Mon Sep 17 00:00:00 2001 From: vsolanki Date: Wed, 5 Dec 2018 11:45:48 -0500 Subject: [PATCH] Use nresource=[native jar] to add contents of resource to java.library.path This feature makes use of the current caching system for "code" jars. Resources marked by nresource are copied to specific cache directories, unpacked, and then the caches are added to the applications java.library.path. This "native" cache is cleaned according to "cache_retention_days", similar to the code cache. --- .../getdown/classpath/ClassPaths.java | 16 ++-- .../getdown/classpath/NativeLibPath.java | 83 +++++++++++++++++++ .../classpath/cache/GarbageCollector.java | 38 +++++++++ .../classpath/cache/ResourceCache.java | 27 +++++- .../threerings/getdown/data/Application.java | 28 +++++++ .../com/threerings/getdown/data/Resource.java | 14 +++- .../com/threerings/getdown/util/FileUtil.java | 40 ++++++++- .../getdown/classpath/ClassPathsTest.java | 4 +- 8 files changed, 233 insertions(+), 17 deletions(-) create mode 100644 core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java diff --git a/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java b/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java index a0dcf7b..05eb487 100644 --- a/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java +++ b/core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java @@ -17,7 +17,6 @@ import com.threerings.getdown.data.Resource; public class ClassPaths { - static final String CACHE_DIR = ".cache"; /** * Builds either a default or cached classpath based on {@code app}'s configuration. @@ -48,23 +47,20 @@ public class ClassPaths */ public static ClassPath buildCachedClassPath (Application app) throws IOException { - File cacheDir = new File(app.getAppDir(), CACHE_DIR); + File codeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/code"); + // a negative value of code_cache_retention_days allows to clean up the cache forcefully - if (app.getCodeCacheRetentionDays() <= 0) { - runGarbageCollection(app, cacheDir); + if (app.getCodeCacheRetentionDays() != 0) { + runGarbageCollection(app, codeCacheDir); } - ResourceCache cache = new ResourceCache(cacheDir); - LinkedHashSet classPathEntries = new LinkedHashSet(); + ResourceCache cache = new ResourceCache(codeCacheDir); + LinkedHashSet classPathEntries = new LinkedHashSet<>(); for (Resource resource: app.getActiveCodeResources()) { File entry = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource)); classPathEntries.add(entry); } - if (app.getCodeCacheRetentionDays() > 0) { - runGarbageCollection(app, cacheDir); - } - return new ClassPath(classPathEntries); } diff --git a/core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java b/core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java new file mode 100644 index 0000000..9c72468 --- /dev/null +++ b/core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java @@ -0,0 +1,83 @@ +package com.threerings.getdown.classpath; + +import com.threerings.getdown.classpath.cache.GarbageCollector; +import com.threerings.getdown.classpath.cache.ResourceCache; +import com.threerings.getdown.data.Application; +import com.threerings.getdown.data.Resource; +import com.threerings.getdown.util.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.jar.JarFile; +import static com.threerings.getdown.Log.log; + +/** + * Similar to {@link ClassPath}, except used to represent directories containing native libraries instead. + */ +public class NativeLibPath extends ClassPath { + + public NativeLibPath(LinkedHashSet entries) { + super(entries); + } + + /** + * Builds a {@link NativeLibPath} instance by first caching all native jars (indicated by + * nresource=[native jar]), unpacking them, and referencing the locations of each of the unpacked files. + * Also performs garbage collection similar to {@link ClassPaths#buildCachedClassPath} + * + * @param app used to determine native jars and related information + * @param addCurrentLibraryPath if true, it adds the locations referenced by + * {@code System.getProperty("java.library.path")} as well + */ + public static NativeLibPath buildLibsPath(Application app, boolean addCurrentLibraryPath) throws IOException { + List resources = app.getNativeJars(); + String[] curPaths = System.getProperty("java.library.path").split(File.pathSeparator); + LinkedHashSet nativedirs = new LinkedHashSet<>(); + + File nativeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/native"); + ResourceCache cache = new ResourceCache(nativeCacheDir); + + // negative value forces total garbage collection, 0 avoids garbage collection at all + if (app.getCodeCacheRetentionDays() != 0) { + runGarbageCollection(app, nativeCacheDir); + } + + for (Resource resource : resources) { + // Use untruncated directory names because in the off chance that two native jars share a directory AND + // contain files with the same names, we'll get overwriting issues when unpacking + File cachedFile = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource), true); + + if (!getUnpackedIndicator(cachedFile).exists()) { + try { + FileUtil.unpackJar(new JarFile(cachedFile), cachedFile.getParentFile(), false); + getUnpackedIndicator(cachedFile).createNewFile(); + } catch (IOException ioe) { + log.warning("Failed to unpack native jar", "File", cachedFile.getAbsolutePath(), ioe); + // Keep going and unpack the other jars... + } + } + + nativedirs.add(cachedFile.getParentFile()); + } + + if (addCurrentLibraryPath) { + for (String path : curPaths) { + nativedirs.add(new File(path)); + } + } + + return new NativeLibPath(nativedirs); + } + + private static void runGarbageCollection(Application app, File nativeCacheDir) { + long retainMillis = TimeUnit.DAYS.toMillis(app.getCodeCacheRetentionDays()); + GarbageCollector.collectNative(nativeCacheDir, retainMillis); + } + + private static File getUnpackedIndicator(File cachedFile) { + return new File(cachedFile.getParent(), cachedFile.getName() + ".unpacked"); + } +} diff --git a/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java b/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java index d4d3e46..c4acd6d 100644 --- a/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java +++ b/core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java @@ -1,6 +1,8 @@ package com.threerings.getdown.classpath.cache; import java.io.File; +import java.io.FileFilter; + import com.threerings.getdown.util.FileUtil; /** @@ -40,6 +42,42 @@ public class GarbageCollector }); } + /** + * Collect and delete garbage in the native cache. It tries to find a jar file with a matching + * lastmodified file, and deletes the entire directory accordingly. + */ + public static void collectNative (File cacheDir, final long retentionPeriodMillis) { + FileUtil.walkDirectChildren(cacheDir, new FileUtil.Visitor() { + @Override + public void visit(File file) { + if (file.isDirectory()) { + // Get all the native jars in the directory (there should only be one though) + File[] nativejars = file.listFiles(new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname.getAbsolutePath().endsWith(".jar"); + } + }); + for (File nativejar : nativejars) { + if (nativejar==null || !nativejar.exists()) { + continue; + } + + File cachedFile = getCachedFile(nativejar); + File lastAccessedFile = getLastAccessedFile(nativejar); + + if (!cachedFile.exists() || !lastAccessedFile.exists() || shouldDelete(lastAccessedFile, retentionPeriodMillis)) { + FileUtil.deleteDirHarder(file); + } + } + } else { + // @TODO There shouldn't be any loose files in native/ but if there are then what? Delete them? + // file.delete(); + } + } + }); + } + private static boolean shouldDelete (File lastAccessedFile, long retentionMillis) { return System.currentTimeMillis() - lastAccessedFile.lastModified() > retentionMillis; diff --git a/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java b/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java index 6e35b3f..d517841 100644 --- a/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java +++ b/core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java @@ -26,13 +26,34 @@ public class ResourceCache } /** - * Caches the given file under it's {@code digest}. - * + * Caches the given file under it's {@code digest}. Same as calling {@link #cacheFile(File, String, boolean)} + * with {@code false} * @return the cached file */ public File cacheFile (File fileToCache, String digest) throws IOException { - File cacheLocation = new File(_cacheDir, digest.substring(0, 2)); + return cacheFile(fileToCache, digest, false); + } + + /** + * Caches the given file under it's {@code digest}. + * @param fileToCache file to cache + * @param digest used to determine the name of the directory to store file in + * @param useFullHashName if true, the name of the cache directory will not be truncated. + * This is useful if you need to avoid the possibility of two files + * sharing a directory. + * @return the cached file + */ + public File cacheFile (File fileToCache, String digest, boolean useFullHashName) throws IOException + { + File cacheLocation; + + if (useFullHashName) { + cacheLocation = new File(_cacheDir, digest); + } else { + cacheLocation = new File(_cacheDir, digest.substring(0, 2)); + + } createDirectoryIfNecessary(cacheLocation); File cachedFile = new File(cacheLocation, digest + getFileSuffix(fileToCache)); diff --git a/core/src/main/java/com/threerings/getdown/data/Application.java b/core/src/main/java/com/threerings/getdown/data/Application.java index 5fca7fb..66bffbe 100644 --- a/core/src/main/java/com/threerings/getdown/data/Application.java +++ b/core/src/main/java/com/threerings/getdown/data/Application.java @@ -25,6 +25,7 @@ import java.util.zip.GZIPInputStream; import com.threerings.getdown.classpath.ClassPaths; import com.threerings.getdown.classpath.ClassPath; +import com.threerings.getdown.classpath.NativeLibPath; import com.threerings.getdown.util.*; // avoid ambiguity with java.util.Base64 which we can't use as it's 1.8+ import com.threerings.getdown.util.Base64; @@ -54,6 +55,9 @@ public class Application /** A special classname that means 'use -jar code.jar' instead of a classname. */ public static final String MANIFEST_CLASS = "manifest"; + /** Name of directory to store cached files in. */ + public static final String CACHE_DIR = ".cache"; + /** Used to communicate information about the UI displayed when updating the application. */ public static final class UpdateInterface { @@ -374,6 +378,22 @@ public class Application return codes; } + /** + * Returns all jar resources indicated to contain native library files (.dll, .so etc.) + */ + public List getNativeJars() { + + List nativeJars = new ArrayList<>(); + + for (Resource resource: _resources) { + if (resource.isNativeJar()) { + nativeJars.add(resource); + } + } + + return nativeJars; + } + /** * Returns all non-code resources and all resources from active auxiliary resource groups. */ @@ -673,6 +693,7 @@ public class Application parseResources(config, "uresource", Resource.UNPACK, _resources); parseResources(config, "xresource", Resource.EXEC, _resources); parseResources(config, "presource", Resource.PRELOAD, _resources); + parseResources(config, "nresource", Resource.NATIVE, _resources); // parse our auxiliary resource groups for (String auxgroup : config.getList("auxgroups")) { @@ -915,6 +936,10 @@ public class Application // add the -classpath arguments if we're not in -jar mode ClassPath classPath = ClassPaths.buildClassPath(this); + // get the -Djava.library.path value to pass + // @TODO optional getdown.txt parameter to set addCurrentLibraryPath to true or false? + NativeLibPath nativeLibPaths = NativeLibPath.buildLibsPath(this, true); + if (!dashJarMode) { args.add("-classpath"); args.add(classPath.asArgumentString()); @@ -938,6 +963,9 @@ public class Application // add the marker indicating the app is running in getdown args.add("-D" + Properties.GETDOWN + "=true"); + // set the native library path + args.add("-Djava.library.path=" + nativeLibPaths.asArgumentString()); + // pass along any pass-through arguments for (Map.Entry entry : System.getProperties().entrySet()) { String key = (String)entry.getKey(); diff --git a/core/src/main/java/com/threerings/getdown/data/Resource.java b/core/src/main/java/com/threerings/getdown/data/Resource.java index 5e426cf..9d6a040 100644 --- a/core/src/main/java/com/threerings/getdown/data/Resource.java +++ b/core/src/main/java/com/threerings/getdown/data/Resource.java @@ -37,13 +37,16 @@ public class Resource implements Comparable /** Indicates that the resource should be marked executable. */ EXEC, /** Indicates that the resource should be downloaded before a UI is displayed. */ - PRELOAD + PRELOAD, + /** Indicates that the resource is a jar containing native libs. */ + NATIVE }; public static final EnumSet NORMAL = EnumSet.noneOf(Attr.class); public static final EnumSet UNPACK = EnumSet.of(Attr.UNPACK); public static final EnumSet EXEC = EnumSet.of(Attr.EXEC); public static final EnumSet PRELOAD = EnumSet.of(Attr.PRELOAD); + public static final EnumSet NATIVE = EnumSet.of(Attr.NATIVE); /** * Computes the MD5 hash of the supplied file. @@ -212,6 +215,15 @@ public class Resource implements Comparable return _attrs.contains(Attr.PRELOAD); } + /** + * Returns true if this resource is a native lib jar. + */ + public boolean isNativeJar () + { + return _attrs.contains(Attr.NATIVE); + } + + /** * Computes the MD5 hash of this resource's underlying file. * Note: This is both CPU and I/O intensive. diff --git a/core/src/main/java/com/threerings/getdown/util/FileUtil.java b/core/src/main/java/com/threerings/getdown/util/FileUtil.java index 378f701..d3325af 100644 --- a/core/src/main/java/com/threerings/getdown/util/FileUtil.java +++ b/core/src/main/java/com/threerings/getdown/util/FileUtil.java @@ -77,6 +77,27 @@ public class FileUtil return deleted; } + /** + * Force deletes {@code file} and all of its children recursively using {@link #deleteHarder}. + * Note that some children may still be deleted even if {@code false} is returned. Also, since {@link #deleteHarder} + * is used, the {@code file} could be deleted once the jvm exits even if {@code false} is returned. + * + * @param file file to delete + * @return true iff {@code file} was successfully deleted. + */ + public static boolean deleteDirHarder(File file) { + if (file.isDirectory()) { + for (File child : file.listFiles()) { + if (child.isDirectory()) { + deleteDirHarder(child); + } else { + deleteHarder(child); + } + } + } + return deleteHarder(file); + } + /** * Reads the contents of the supplied input stream into a list of lines. Closes the reader on * successful or failed completion. @@ -190,7 +211,7 @@ public class FileUtil } /** - * Used by {@link #walkTree}. + * Used by {@link #walkTree} and {@link #walkDirectChildren}. */ public interface Visitor { @@ -218,4 +239,21 @@ public class FileUtil } } } + + /** + * Walks all direct sub-files of {@code root}, calling {@code visitor} on each file. + */ + public static void walkDirectChildren (final File root, Visitor visitor) { + File[] children = root.listFiles(); + + if (children == null) return; + Deque stack = new ArrayDeque<>(Arrays.asList(children)); + while (!stack.isEmpty()) { + File currentFile = stack.pop(); + if (currentFile.exists()) { + visitor.visit(currentFile); + } + } + } + } diff --git a/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java b/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java index be6b32b..05b47a1 100644 --- a/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java +++ b/core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java @@ -46,10 +46,10 @@ public class ClassPathsTest when(_application.getCodeCacheRetentionDays()).thenReturn(1); Path firstCachedJarFile = _appdir.getRoot().toPath(). - resolve(ClassPaths.CACHE_DIR).resolve("fi").resolve("first.jar"); + resolve(Application.CACHE_DIR + "/code").resolve("fi").resolve("first.jar"); Path secondCachedJarFile = _appdir.getRoot().toPath(). - resolve(ClassPaths.CACHE_DIR).resolve("se").resolve("second.jar"); + resolve(Application.CACHE_DIR + "/code").resolve("se").resolve("second.jar"); String expectedClassPath = firstCachedJarFile.toAbsolutePath() + File.pathSeparator + secondCachedJarFile.toAbsolutePath();