Merge pull request #173 from vatsal22/native_cache

Use nresource=[native jar] to use native jars

core/src/main/java/com/threerings/getdown/classpath/ClassPaths.java

@@ -17,7 +17,6 @@ import com.threerings.getdown.data.Resource;
 
 public class ClassPaths
 {
-    static final String CACHE_DIR = ".cache";
 
     /**
      * Builds either a default or cached classpath based on {@code app}'s configuration.
@@ -48,23 +47,20 @@ public class ClassPaths
      */
     public static ClassPath buildCachedClassPath (Application app) throws IOException
     {
-        File cacheDir = new File(app.getAppDir(), CACHE_DIR);
+        File codeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/code");
+
         // a negative value of code_cache_retention_days allows to clean up the cache forcefully
-        if (app.getCodeCacheRetentionDays() <= 0) {
-            runGarbageCollection(app, cacheDir);
+        if (app.getCodeCacheRetentionDays() != 0) {
+            runGarbageCollection(app, codeCacheDir);
         }
 
-        ResourceCache cache = new ResourceCache(cacheDir);
-        LinkedHashSet<File> classPathEntries = new LinkedHashSet<File>();
+        ResourceCache cache = new ResourceCache(codeCacheDir);
+        LinkedHashSet<File> classPathEntries = new LinkedHashSet<>();
         for (Resource resource: app.getActiveCodeResources()) {
             File entry = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource));
             classPathEntries.add(entry);
         }
 
-        if (app.getCodeCacheRetentionDays() > 0) {
-            runGarbageCollection(app, cacheDir);
-        }
-
         return new ClassPath(classPathEntries);
     }
 

core/src/main/java/com/threerings/getdown/classpath/NativeLibPath.java

@@ -0,0 +1,83 @@
+package com.threerings.getdown.classpath;
+
+import com.threerings.getdown.classpath.cache.GarbageCollector;
+import com.threerings.getdown.classpath.cache.ResourceCache;
+import com.threerings.getdown.data.Application;
+import com.threerings.getdown.data.Resource;
+import com.threerings.getdown.util.FileUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.jar.JarFile;
+import static com.threerings.getdown.Log.log;
+
+/**
+ * Similar to {@link ClassPath}, except used to represent directories containing native libraries instead.
+ */
+public class NativeLibPath extends ClassPath {
+
+    public NativeLibPath(LinkedHashSet<File> entries) {
+        super(entries);
+    }
+
+    /**
+     * Builds a {@link NativeLibPath} instance by first caching all native jars (indicated by
+     * nresource=[native jar]), unpacking them, and referencing the locations of each of the unpacked files.
+     * Also performs garbage collection similar to {@link ClassPaths#buildCachedClassPath}
+     *
+     * @param app                   used to determine native jars and related information
+     * @param addCurrentLibraryPath if true, it adds the locations referenced by
+     *                              {@code System.getProperty("java.library.path")} as well
+     */
+    public static NativeLibPath buildLibsPath(Application app, boolean addCurrentLibraryPath) throws IOException {
+        List<Resource> resources = app.getNativeJars();
+        String[] curPaths = System.getProperty("java.library.path").split(File.pathSeparator);
+        LinkedHashSet<File> nativedirs = new LinkedHashSet<>();
+
+        File nativeCacheDir = new File(app.getAppDir(), Application.CACHE_DIR + "/native");
+        ResourceCache cache = new ResourceCache(nativeCacheDir);
+
+        // negative value forces total garbage collection, 0 avoids garbage collection at all
+        if (app.getCodeCacheRetentionDays() != 0) {
+            runGarbageCollection(app, nativeCacheDir);
+        }
+
+        for (Resource resource : resources) {
+            // Use untruncated directory names because in the off chance that two native jars share a directory AND
+            // contain files with the same names, we'll get overwriting issues when unpacking
+            File cachedFile = cache.cacheFile(resource.getFinalTarget(), app.getDigest(resource), true);
+
+            if (!getUnpackedIndicator(cachedFile).exists()) {
+                try {
+                    FileUtil.unpackJar(new JarFile(cachedFile), cachedFile.getParentFile(), false);
+                    getUnpackedIndicator(cachedFile).createNewFile();
+                } catch (IOException ioe) {
+                    log.warning("Failed to unpack native jar", "File", cachedFile.getAbsolutePath(), ioe);
+                    // Keep going and unpack the other jars...
+                }
+            }
+
+            nativedirs.add(cachedFile.getParentFile());
+        }
+
+        if (addCurrentLibraryPath) {
+            for (String path : curPaths) {
+                nativedirs.add(new File(path));
+            }
+        }
+
+        return new NativeLibPath(nativedirs);
+    }
+
+    private static void runGarbageCollection(Application app, File nativeCacheDir) {
+        long retainMillis = TimeUnit.DAYS.toMillis(app.getCodeCacheRetentionDays());
+        GarbageCollector.collectNative(nativeCacheDir, retainMillis);
+    }
+
+    private static File getUnpackedIndicator(File cachedFile) {
+        return new File(cachedFile.getParent(), cachedFile.getName() + ".unpacked");
+    }
+}

core/src/main/java/com/threerings/getdown/classpath/cache/GarbageCollector.java

@@ -1,6 +1,8 @@
 package com.threerings.getdown.classpath.cache;
 
 import java.io.File;
+import java.io.FileFilter;
+
 import com.threerings.getdown.util.FileUtil;
 
 /**
@@ -40,6 +42,42 @@ public class GarbageCollector
         });
     }
 
+    /**
+     * Collect and delete garbage in the native cache. It tries to find a jar file with a matching
+     * lastmodified file, and deletes the entire directory accordingly.
+     */
+    public static void collectNative (File cacheDir, final long retentionPeriodMillis) {
+        FileUtil.walkDirectChildren(cacheDir, new FileUtil.Visitor() {
+            @Override
+            public void visit(File file) {
+                if (file.isDirectory()) {
+                    // Get all the native jars in the directory (there should only be one though)
+                    File[] nativejars = file.listFiles(new FileFilter() {
+                        @Override
+                        public boolean accept(File pathname) {
+                            return pathname.getAbsolutePath().endsWith(".jar");
+                        }
+                    });
+                    for (File nativejar : nativejars) {
+                        if (nativejar==null || !nativejar.exists()) {
+                            continue;
+                        }
+
+                        File cachedFile = getCachedFile(nativejar);
+                        File lastAccessedFile = getLastAccessedFile(nativejar);
+
+                        if (!cachedFile.exists() || !lastAccessedFile.exists() || shouldDelete(lastAccessedFile, retentionPeriodMillis)) {
+                           FileUtil.deleteDirHarder(file);
+                        }
+                    }
+                } else {
+                    // @TODO There shouldn't be any loose files in native/ but if there are then what? Delete them?
+                    //  file.delete();
+                }
+            }
+        });
+    }
+
     private static boolean shouldDelete (File lastAccessedFile, long retentionMillis)
     {
         return System.currentTimeMillis() - lastAccessedFile.lastModified() > retentionMillis;

core/src/main/java/com/threerings/getdown/classpath/cache/ResourceCache.java

@@ -26,13 +26,34 @@ public class ResourceCache
     }
 
     /**
-     * Caches the given file under it's {@code digest}.
-     *
+     * Caches the given file under it's {@code digest}. Same as calling {@link #cacheFile(File, String, boolean)}
+     * with {@code false}
      * @return the cached file
      */
     public File cacheFile (File fileToCache, String digest) throws IOException
     {
-        File cacheLocation = new File(_cacheDir, digest.substring(0, 2));
+        return cacheFile(fileToCache, digest, false);
+    }
+
+    /**
+     * Caches the given file under it's {@code digest}.
+     * @param fileToCache file to cache
+     * @param digest used to determine the name of the directory to store file in
+     * @param useFullHashName if true, the name of the cache directory will not be truncated.
+     *                        This is useful if you need to avoid the possibility of two files
+     *                        sharing a directory.
+     * @return the cached file
+     */
+    public File cacheFile (File fileToCache, String digest, boolean useFullHashName) throws IOException
+    {
+        File cacheLocation;
+
+        if (useFullHashName) {
+            cacheLocation = new File(_cacheDir, digest);
+        } else {
+            cacheLocation = new File(_cacheDir, digest.substring(0, 2));
+
+        }
         createDirectoryIfNecessary(cacheLocation);
 
         File cachedFile = new File(cacheLocation, digest + getFileSuffix(fileToCache));

core/src/main/java/com/threerings/getdown/data/Application.java

@@ -25,6 +25,7 @@ import java.util.zip.GZIPInputStream;
 
 import com.threerings.getdown.classpath.ClassPaths;
 import com.threerings.getdown.classpath.ClassPath;
+import com.threerings.getdown.classpath.NativeLibPath;
 import com.threerings.getdown.util.*;
 // avoid ambiguity with java.util.Base64 which we can't use as it's 1.8+
 import com.threerings.getdown.util.Base64;
@@ -54,6 +55,9 @@ public class Application
     /** A special classname that means 'use -jar code.jar' instead of a classname. */
     public static final String MANIFEST_CLASS = "manifest";
 
+    /** Name of directory to store cached files in. */
+    public static final String CACHE_DIR = ".cache";
+
     /** Used to communicate information about the UI displayed when updating the application. */
     public static final class UpdateInterface
     {
@@ -374,6 +378,22 @@ public class Application
         return codes;
     }
 
+    /**
+     * Returns all jar resources indicated to contain native library files (.dll, .so etc.)
+     */
+    public List<Resource> getNativeJars() {
+
+        List<Resource> nativeJars = new ArrayList<>();
+
+        for (Resource resource: _resources) {
+            if (resource.isNativeJar()) {
+                nativeJars.add(resource);
+            }
+        }
+
+        return nativeJars;
+    }
+
     /**
      * Returns all non-code resources and all resources from active auxiliary resource groups.
      */
@@ -673,6 +693,7 @@ public class Application
         parseResources(config, "uresource", Resource.UNPACK, _resources);
         parseResources(config, "xresource", Resource.EXEC, _resources);
         parseResources(config, "presource", Resource.PRELOAD, _resources);
+        parseResources(config, "nresource", Resource.NATIVE, _resources);
 
         // parse our auxiliary resource groups
         for (String auxgroup : config.getList("auxgroups")) {
@@ -915,6 +936,10 @@ public class Application
         // add the -classpath arguments if we're not in -jar mode
         ClassPath classPath = ClassPaths.buildClassPath(this);
 
+        // get the -Djava.library.path value to pass
+        // @TODO optional getdown.txt parameter to set addCurrentLibraryPath to true or false?
+        NativeLibPath nativeLibPaths = NativeLibPath.buildLibsPath(this, true);
+
         if (!dashJarMode) {
             args.add("-classpath");
             args.add(classPath.asArgumentString());
@@ -938,6 +963,9 @@ public class Application
         // add the marker indicating the app is running in getdown
         args.add("-D" + Properties.GETDOWN + "=true");
 
+        // set the native library path
+        args.add("-Djava.library.path=" + nativeLibPaths.asArgumentString());
+
         // pass along any pass-through arguments
         for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
             String key = (String)entry.getKey();

core/src/main/java/com/threerings/getdown/data/Resource.java

@@ -37,13 +37,16 @@ public class Resource implements Comparable<Resource>
         /** Indicates that the resource should be marked executable. */
         EXEC,
         /** Indicates that the resource should be downloaded before a UI is displayed. */
-        PRELOAD
+        PRELOAD,
+        /** Indicates that the resource is a jar containing native libs. */
+        NATIVE
     };
 
     public static final EnumSet<Attr> NORMAL  = EnumSet.noneOf(Attr.class);
     public static final EnumSet<Attr> UNPACK  = EnumSet.of(Attr.UNPACK);
     public static final EnumSet<Attr> EXEC    = EnumSet.of(Attr.EXEC);
     public static final EnumSet<Attr> PRELOAD = EnumSet.of(Attr.PRELOAD);
+    public static final EnumSet<Attr> NATIVE = EnumSet.of(Attr.NATIVE);
 
     /**
      * Computes the MD5 hash of the supplied file.
@@ -212,6 +215,15 @@ public class Resource implements Comparable<Resource>
         return _attrs.contains(Attr.PRELOAD);
     }
 
+    /**
+     * Returns true if this resource is a native lib jar.
+     */
+    public boolean isNativeJar ()
+    {
+        return _attrs.contains(Attr.NATIVE);
+    }
+
+
     /**
      * Computes the MD5 hash of this resource's underlying file.
      * <em>Note:</em> This is both CPU and I/O intensive.

core/src/main/java/com/threerings/getdown/util/FileUtil.java

@@ -77,6 +77,27 @@ public class FileUtil
         return deleted;
     }
 
+    /**
+     * Force deletes {@code file} and all of its children recursively using {@link #deleteHarder}.
+     * Note that some children may still be deleted even if {@code false} is returned. Also, since {@link #deleteHarder}
+     * is used, the {@code file} could be deleted once the jvm exits even if {@code false} is returned.
+     *
+     * @param file file to delete
+     * @return true iff {@code file} was successfully deleted.
+     */
+    public static boolean deleteDirHarder(File file) {
+        if (file.isDirectory()) {
+            for (File child : file.listFiles()) {
+                if (child.isDirectory()) {
+                    deleteDirHarder(child);
+                } else {
+                    deleteHarder(child);
+                }
+            }
+        }
+        return deleteHarder(file);
+    }
+
     /**
      * Reads the contents of the supplied input stream into a list of lines. Closes the reader on
      * successful or failed completion.
@@ -190,7 +211,7 @@ public class FileUtil
     }
 
     /**
-     * Used by {@link #walkTree}.
+     * Used by {@link #walkTree} and {@link #walkDirectChildren}.
      */
     public interface Visitor
     {
@@ -218,4 +239,21 @@ public class FileUtil
             }
         }
     }
+
+    /**
+     * Walks all direct sub-files of {@code root}, calling {@code visitor} on each file.
+     */
+    public static void walkDirectChildren (final File root, Visitor visitor) {
+        File[] children = root.listFiles();
+
+        if (children == null) return;
+        Deque<File> stack = new ArrayDeque<>(Arrays.asList(children));
+        while (!stack.isEmpty()) {
+            File currentFile = stack.pop();
+            if (currentFile.exists()) {
+                visitor.visit(currentFile);
+            }
+        }
+    }
+
 }

core/src/test/java/com/threerings/getdown/classpath/ClassPathsTest.java

@@ -46,10 +46,10 @@ public class ClassPathsTest
         when(_application.getCodeCacheRetentionDays()).thenReturn(1);
 
         Path firstCachedJarFile = _appdir.getRoot().toPath().
-            resolve(ClassPaths.CACHE_DIR).resolve("fi").resolve("first.jar");
+            resolve(Application.CACHE_DIR + "/code").resolve("fi").resolve("first.jar");
 
         Path secondCachedJarFile = _appdir.getRoot().toPath().
-            resolve(ClassPaths.CACHE_DIR).resolve("se").resolve("second.jar");
+            resolve(Application.CACHE_DIR + "/code").resolve("se").resolve("second.jar");
 
         String expectedClassPath = firstCachedJarFile.toAbsolutePath() + File.pathSeparator +
             secondCachedJarFile.toAbsolutePath();