Use nresource=[native jar] to add contents of resource to java.library.path

This feature makes use of the current caching system for "code" jars. Resources marked by nresource are copied to specific cache directories, unpacked, and then the caches are added to the applications java.library.path. This "native" cache is cleaned according to "cache_retention_days", similar to the code cache.

core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java

@@ -17,7 +17,6 @@ import com.threerings.getdown.data.Resource;
 
 public class ClassPaths
 {
-    static final String CACHE_DIR = ".cache";
 
     /**
      * Builds either a default or cached classpath based on {@code app}'s configuration.
@@ -48,23 +47,20 @@ public class ClassPaths
      */
     public static ClassPath buildCachedClassPath (Application app) throws IOException
     {
-        File cacheDir = new File(app.getAppDir(), CACHE_DIR);
+        File codeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/code");
+
         // a negative value of code_cache_retention_days allows to clean up the cache forcefully
-        if (app.getCodeCacheRetentionDays() <= 0) {
-            runGarbageCollection(app, cacheDir);
+        if (app.getCodeCacheRetentionDays() != 0) {
+            runGarbageCollection(app, codeCacheDir);
         }
 
-        ResourceCache cache = new ResourceCache(cacheDir);
-        LinkedHashSet<File> classPathEntries = new LinkedHashSet<File>();
+        ResourceCache cache = new ResourceCache(codeCacheDir);
+        LinkedHashSet<File> classPathEntries = new LinkedHashSet<>();
         for (Resource resource: app.getActiveCodeResources()) {
             File entry = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource));
             classPathEntries.add(entry);
         }
 
-        if (app.getCodeCacheRetentionDays() > 0) {
-            runGarbageCollection(app, cacheDir);
-        }
-
         return new ClassPath(classPathEntries);
     }
 

core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java

@@ -0,0 +1,83 @@
+package com.threerings.getdown.classpath;
+
+import com.threerings.getdown.classpath.cache.GarbageCollector;
+import com.threerings.getdown.classpath.cache.ResourceCache;
+import com.threerings.getdown.data.Application;
+import com.threerings.getdown.data.Resource;
+import com.threerings.getdown.util.FileUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.jar.JarFile;
+import static com.threerings.getdown.Log.log;
+
+/**
+ * Similar to {@link ClassPath}, except used to represent directories containing native libraries instead.
+ */
+public class NativeLibPath extends ClassPath {
+
+    public NativeLibPath(LinkedHashSet<File> entries) {
+        super(entries);
+    }
+
+    /**
+     * Builds a {@link NativeLibPath} instance by first caching all native jars (indicated by
+     * nresource=[native jar]), unpacking them, and referencing the locations of each of the unpacked files.
+     * Also performs garbage collection similar to {@link ClassPaths#buildCachedClassPath}
+     *
+     * @param app                   used to determine native jars and related information
+     * @param addCurrentLibraryPath if true, it adds the locations referenced by
+     *                              {@code System.getProperty("java.library.path")} as well
+     */
+    public static NativeLibPath buildLibsPath(Application app, boolean addCurrentLibraryPath) throws IOException {
+        List<Resource> resources = app.getNativeJars();
+        String[] curPaths = System.getProperty("java.library.path").split(File.pathSeparator);
+        LinkedHashSet<File> nativedirs = new LinkedHashSet<>();
+
+        File nativeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/native");
+        ResourceCache cache = new ResourceCache(nativeCacheDir);
+
+        // negative value forces total garbage collection, 0 avoids garbage collection at all
+        if (app.getCodeCacheRetentionDays() != 0) {
+            runGarbageCollection(app, nativeCacheDir);
+        }
+
+        for (Resource resource : resources) {
+            // Use untruncated directory names because in the off chance that two native jars share a directory AND
+            // contain files with the same names, we'll get overwriting issues when unpacking
+            File cachedFile = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource), true);
+
+            if (!getUnpackedIndicator(cachedFile).exists()) {
+                try {
+                    FileUtil.unpackJar(new JarFile(cachedFile), cachedFile.getParentFile(), false);
+                    getUnpackedIndicator(cachedFile).createNewFile();
+                } catch (IOException ioe) {
+                    log.warning("Failed to unpack native jar", "File", cachedFile.getAbsolutePath(), ioe);
+                    // Keep going and unpack the other jars...
+                }
+            }
+
+            nativedirs.add(cachedFile.getParentFile());
+        }
+
+        if (addCurrentLibraryPath) {
+            for (String path : curPaths) {
+                nativedirs.add(new File(path));
+            }
+        }
+
+        return new NativeLibPath(nativedirs);
+    }
+
+    private static void runGarbageCollection(Application app, File nativeCacheDir) {
+        long retainMillis = TimeUnit.DAYS.toMillis(app.getCodeCacheRetentionDays());
+        GarbageCollector.collectNative(nativeCacheDir, retainMillis);
+    }
+
+    private static File getUnpackedIndicator(File cachedFile) {
+        return new File(cachedFile.getParent(), cachedFile.getName() + ".unpacked");
+    }
+}

core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java

@@ -1,6 +1,8 @@
 package com.threerings.getdown.classpath.cache;
 
 import java.io.File;
+import java.io.FileFilter;
+
 import com.threerings.getdown.util.FileUtil;
 
 /**
@@ -40,6 +42,42 @@ public class GarbageCollector
         });
     }
 
+    /**
+     * Collect and delete garbage in the native cache. It tries to find a jar file with a matching
+     * lastmodified file, and deletes the entire directory accordingly.
+     */
+    public static void collectNative (File cacheDir, final long retentionPeriodMillis) {
+        FileUtil.walkDirectChildren(cacheDir, new FileUtil.Visitor() {
+            @Override
+            public void visit(File file) {
+                if (file.isDirectory()) {
+                    // Get all the native jars in the directory (there should only be one though)
+                    File[] nativejars = file.listFiles(new FileFilter() {
+                        @Override
+                        public boolean accept(File pathname) {
+                            return pathname.getAbsolutePath().endsWith(".jar");
+                        }
+                    });
+                    for (File nativejar : nativejars) {
+                        if (nativejar==null || !nativejar.exists()) {
+                            continue;
+                        }
+
+                        File cachedFile = getCachedFile(nativejar);
+                        File lastAccessedFile = getLastAccessedFile(nativejar);
+
+                        if (!cachedFile.exists() || !lastAccessedFile.exists() || shouldDelete(lastAccessedFile, retentionPeriodMillis)) {
+                           FileUtil.deleteDirHarder(file);
+                        }
+                    }
+                } else {
+                    // @TODO There shouldn't be any loose files in native/ but if there are then what? Delete them?
+                    //  file.delete();
+                }
+            }
+        });
+    }
+
     private static boolean shouldDelete (File lastAccessedFile, long retentionMillis)
     {
         return System.currentTimeMillis() - lastAccessedFile.lastModified() > retentionMillis;

core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java

@@ -26,13 +26,34 @@ public class ResourceCache
     }
 
     /**
-     * Caches the given file under it's {@code digest}.
-     *
+     * Caches the given file under it's {@code digest}. Same as calling {@link #cacheFile(File, String, boolean)}
+     * with {@code false}
      * @return the cached file
      */
     public File cacheFile (File fileToCache, String digest) throws IOException
     {
-        File cacheLocation = new File(_cacheDir, digest.substring(0, 2));
+        return cacheFile(fileToCache, digest, false);
+    }
+
+    /**
+     * Caches the given file under it's {@code digest}.
+     * @param fileToCache file to cache
+     * @param digest used to determine the name of the directory to store file in
+     * @param useFullHashName if true, the name of the cache directory will not be truncated.
+     *                        This is useful if you need to avoid the possibility of two files
+     *                        sharing a directory.
+     * @return the cached file
+     */
+    public File cacheFile (File fileToCache, String digest, boolean useFullHashName) throws IOException
+    {
+        File cacheLocation;
+
+        if (useFullHashName) {
+            cacheLocation = new File(_cacheDir, digest);
+        } else {
+            cacheLocation = new File(_cacheDir, digest.substring(0, 2));
+
+        }
         createDirectoryIfNecessary(cacheLocation);
 
         File cachedFile = new File(cacheLocation, digest + getFileSuffix(fileToCache));

core/src/main/java/com/threerings/getdown/data/Application.java

@@ -25,6 +25,7 @@ import java.util.zip.GZIPInputStream;
 
 import com.threerings.getdown.classpath.ClassPaths;
 import com.threerings.getdown.classpath.ClassPath;
+import com.threerings.getdown.classpath.NativeLibPath;
 import com.threerings.getdown.util.*;
 // avoid ambiguity with java.util.Base64 which we can't use as it's 1.8+
 import com.threerings.getdown.util.Base64;
@@ -54,6 +55,9 @@ public class Application
     /** A special classname that means 'use -jar code.jar' instead of a classname. */
     public static final String MANIFEST_CLASS = "manifest";
 
+    /** Name of directory to store cached files in. */
+    public static final String CACHE_DIR = ".cache";
+
     /** Used to communicate information about the UI displayed when updating the application. */
     public static final class UpdateInterface
     {
@@ -374,6 +378,22 @@ public class Application
         return codes;
     }
 
+    /**
+     * Returns all jar resources indicated to contain native library files (.dll, .so etc.)
+     */
+    public List<Resource> getNativeJars() {
+
+        List<Resource> nativeJars = new ArrayList<>();
+
+        for (Resource resource: _resources) {
+            if (resource.isNativeJar()) {
+                nativeJars.add(resource);
+            }
+        }
+
+        return nativeJars;
+    }
+
     /**
      * Returns all non-code resources and all resources from active auxiliary resource groups.
      */
@@ -673,6 +693,7 @@ public class Application
         parseResources(config, "uresource", Resource.UNPACK, _resources);
         parseResources(config, "xresource", Resource.EXEC, _resources);
         parseResources(config, "presource", Resource.PRELOAD, _resources);
+        parseResources(config, "nresource", Resource.NATIVE, _resources);
 
         // parse our auxiliary resource groups
         for (String auxgroup : config.getList("auxgroups")) {
@@ -915,6 +936,10 @@ public class Application
         // add the -classpath arguments if we're not in -jar mode
         ClassPath classPath = ClassPaths.buildClassPath(this);
 
+        // get the -Djava.library.path value to pass
+        // @TODO optional getdown.txt parameter to set addCurrentLibraryPath to true or false?
+        NativeLibPath nativeLibPaths = NativeLibPath.buildLibsPath(this, true);
+
         if (!dashJarMode) {
             args.add("-classpath");
             args.add(classPath.asArgumentString());
@@ -938,6 +963,9 @@ public class Application
         // add the marker indicating the app is running in getdown
         args.add("-D" + Properties.GETDOWN + "=true");
 
+        // set the native library path
+        args.add("-Djava.library.path=" + nativeLibPaths.asArgumentString());
+
         // pass along any pass-through arguments
         for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
             String key = (String)entry.getKey();

core/src/main/java/com/threerings/getdown/data/Resource.java

@@ -37,13 +37,16 @@ public class Resource implements Comparable<Resource>
         /** Indicates that the resource should be marked executable. */
         EXEC,
         /** Indicates that the resource should be downloaded before a UI is displayed. */
-        PRELOAD
+        PRELOAD,
+        /** Indicates that the resource is a jar containing native libs. */
+        NATIVE
     };
 
     public static final EnumSet<Attr> NORMAL  = EnumSet.noneOf(Attr.class);
     public static final EnumSet<Attr> UNPACK  = EnumSet.of(Attr.UNPACK);
     public static final EnumSet<Attr> EXEC    = EnumSet.of(Attr.EXEC);
     public static final EnumSet<Attr> PRELOAD = EnumSet.of(Attr.PRELOAD);
+    public static final EnumSet<Attr> NATIVE = EnumSet.of(Attr.NATIVE);
 
     /**
      * Computes the MD5 hash of the supplied file.
@@ -212,6 +215,15 @@ public class Resource implements Comparable<Resource>
         return _attrs.contains(Attr.PRELOAD);
     }
 
+    /**
+     * Returns true if this resource is a native lib jar.
+     */
+    public boolean isNativeJar ()
+    {
+        return _attrs.contains(Attr.NATIVE);
+    }
+
+
     /**
      * Computes the MD5 hash of this resource's underlying file.
      * <em>Note:</em> This is both CPU and I/O intensive.

core/src/main/java/com/threerings/getdown/util/FileUtil.java

@@ -77,6 +77,27 @@ public class FileUtil
         return deleted;
     }
 
+    /**
+     * Force deletes {@code file} and all of its children recursively using {@link #deleteHarder}.
+     * Note that some children may still be deleted even if {@code false} is returned. Also, since {@link #deleteHarder}
+     * is used, the {@code file} could be deleted once the jvm exits even if {@code false} is returned.
+     *
+     * @param file file to delete
+     * @return true iff {@code file} was successfully deleted.
+     */
+    public static boolean deleteDirHarder(File file) {
+        if (file.isDirectory()) {
+            for (File child : file.listFiles()) {
+                if (child.isDirectory()) {
+                    deleteDirHarder(child);
+                } else {
+                    deleteHarder(child);
+                }
+            }
+        }
+        return deleteHarder(file);
+    }
+
     /**
      * Reads the contents of the supplied input stream into a list of lines. Closes the reader on
      * successful or failed completion.
@@ -190,7 +211,7 @@ public class FileUtil
     }
 
     /**
-     * Used by {@link #walkTree}.
+     * Used by {@link #walkTree} and {@link #walkDirectChildren}.
      */
     public interface Visitor
     {
@@ -218,4 +239,21 @@ public class FileUtil
             }
         }
     }
+
+    /**
+     * Walks all direct sub-files of {@code root}, calling {@code visitor} on each file.
+     */
+    public static void walkDirectChildren (final File root, Visitor visitor) {
+        File[] children = root.listFiles();
+
+        if (children == null) return;
+        Deque<File> stack = new ArrayDeque<>(Arrays.asList(children));
+        while (!stack.isEmpty()) {
+            File currentFile = stack.pop();
+            if (currentFile.exists()) {
+                visitor.visit(currentFile);
+            }
+        }
+    }
+
 }

core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java

@@ -46,10 +46,10 @@ public class ClassPathsTest
         when(_application.getCodeCacheRetentionDays()).thenReturn(1);
 
         Path firstCachedJarFile = _appdir.getRoot().toPath().
-            resolve(ClassPaths.CACHE_DIR).resolve("fi").resolve("first.jar");
+            resolve(Application.CACHE_DIR + "/code").resolve("fi").resolve("first.jar");
 
         Path secondCachedJarFile = _appdir.getRoot().toPath().
-            resolve(ClassPaths.CACHE_DIR).resolve("se").resolve("second.jar");
+            resolve(Application.CACHE_DIR + "/code").resolve("se").resolve("second.jar");
 
         String expectedClassPath = firstCachedJarFile.toAbsolutePath() + File.pathSeparator +
             secondCachedJarFile.toAbsolutePath();