Added support for data migrations, which are run after all schema migrations

have been run and are also coordinated between disparate processes by an
in-database lock system.

src/java/com/samskivert/jdbc/depot/DataMigration.java

@@ -0,0 +1,73 @@
+//
+// $Id$
+//
+// samskivert library - useful routines for java programs
+// Copyright (C) 2001-2008 Michael Bayne
+// 
+// This library is free software; you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published
+// by the Free Software Foundation; either version 2.1 of the License, or
+// (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package com.samskivert.jdbc.depot;
+
+/**
+ * Encapsulates a migration of data between entities that should be run only once and using the
+ * same safeguards applied to entity migrations. Note: this should not be used for schema
+ * migrations, use {@link SchemaMigration} for that. Data migrations are registered on a specific
+ * repository via {@link DepotRepository#registerMigration} and should be registered in the
+ * repository's constructor as they will be invoked (if appropriate) in the repository's {@link
+ * DepotRepository#init} method.
+ *
+ * <p> In general one will register an anonymous inner class in a repository's constructor and can
+ * then access methods in the repository directly:
+ *
+ * <pre>
+ * public class FooRepository extends DepotRepository {
+ *     public FooRepository (PersistenceContext ctx) {
+ *         super(ctx);
+ *         registerMigration(new DataMigration("2008_09_25_referral_to_tracking_id") {
+ *             public void invoke () throws DatabaseException {
+ *                 // feel free to use load() findAll(), update(), etc.
+ *             }
+ *         });
+ *     }
+ * </pre>
+ */
+public abstract class DataMigration
+{
+    /**
+     * Creates a data migration with the specified unique identifier. The identifier must be unique
+     * across all users of the database and for all time, so be careful. Best to include the date
+     * and pertintent information, e.g. "2008_09_25_referral_to_tracking_id".
+     */
+    public DataMigration (String ident)
+    {
+        _ident = ident;
+    }
+
+    /**
+     * Returns the identifier of this migration.
+     */
+    public String getIdent ()
+    {
+        return _ident;
+    }
+
+    /**
+     * Effects the data migration.
+     */
+    public abstract void invoke () throws DatabaseException;
+
+    /** The unique identifier for this migration. */
+    protected String _ident;
+}

src/java/com/samskivert/jdbc/depot/DepotMarshaller.java

@@ -630,10 +630,10 @@ public class DepotMarshaller<T extends PersistentRecord>
             // we didn't get the lock, so wait 5 seconds and then check to see if the other process
             // finished the update or failed in which case we'll try to grab the lock ourselves
             try {
-                log.info("Waiting for migration lock for " + _pClass.getName() + ".");
+                log.info("Waiting on migration lock for " + _pClass.getName() + ".");
                 Thread.sleep(5000);
             } catch (InterruptedException ie) {
-                throw new DatabaseException("Interrupted while waiting for migration lock.");
+                throw new DatabaseException("Interrupted while waiting on migration lock.");
             }
         }
 

src/java/com/samskivert/jdbc/depot/DepotMigrationHistoryRecord.java

@@ -0,0 +1,73 @@
+//
+// $Id$
+//
+// samskivert library - useful routines for java programs
+// Copyright (C) 2001-2008 Michael Bayne
+// 
+// This library is free software; you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published
+// by the Free Software Foundation; either version 2.1 of the License, or
+// (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package com.samskivert.jdbc.depot;
+
+import java.sql.Timestamp;
+
+import com.samskivert.jdbc.depot.annotation.Column;
+import com.samskivert.jdbc.depot.annotation.Id;
+import com.samskivert.jdbc.depot.expression.ColumnExp;
+
+/**
+ * Maintains a record of all successfully invoked data migrations.
+ */
+public class DepotMigrationHistoryRecord extends PersistentRecord
+{
+    // AUTO-GENERATED: FIELDS START
+    /** The column identifier for the {@link #ident} field. */
+    public static final String IDENT = "ident";
+
+    /** The qualified column identifier for the {@link #ident} field. */
+    public static final ColumnExp IDENT_C =
+        new ColumnExp(DepotMigrationHistoryRecord.class, IDENT);
+
+    /** The column identifier for the {@link #whenCompleted} field. */
+    public static final String WHEN_COMPLETED = "whenCompleted";
+
+    /** The qualified column identifier for the {@link #whenCompleted} field. */
+    public static final ColumnExp WHEN_COMPLETED_C =
+        new ColumnExp(DepotMigrationHistoryRecord.class, WHEN_COMPLETED);
+    // AUTO-GENERATED: FIELDS END
+
+    /** Our schema version. Probably not likely to change. */
+    public static final int SCHEMA_VERSION = 1;
+
+    /** The unique identifier for this migration. */
+    @Id public String ident;
+
+    /** The time at which the migration was completed. */
+    @Column(nullable=true)
+    public Timestamp whenCompleted;
+
+    // AUTO-GENERATED: METHODS START
+    /**
+     * Create and return a primary {@link Key} to identify a {@link DepotMigrationHistoryRecord}
+     * with the supplied key values.
+     */
+    public static Key<DepotMigrationHistoryRecord> getKey (String ident)
+    {
+        return new Key<DepotMigrationHistoryRecord>(
+                DepotMigrationHistoryRecord.class,
+                new String[] { IDENT },
+                new Comparable[] { ident });
+    }
+    // AUTO-GENERATED: METHODS END
+}

src/java/com/samskivert/jdbc/depot/DepotRepository.java

@@ -24,6 +24,7 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.sql.Timestamp;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -50,6 +51,8 @@ import com.samskivert.jdbc.depot.clause.UpdateClause;
 import com.samskivert.jdbc.depot.expression.SQLExpression;
 import com.samskivert.jdbc.depot.expression.ValueExp;
 
+import static com.samskivert.jdbc.depot.Log.log;
+
 /**
  * Provides a base for classes that provide access to persistent objects. Also defines the
  * mechanism by which all persistent queries and updates are routed through the distributed cache.
@@ -82,11 +85,11 @@ public abstract class DepotRepository
     }
 
     /**
-     * Provides a place where a repository can perform any initialization that requires database
-     * operations. The default implementation resolves all records managed by this repository so
-     * that their schema migrations are run.
+     * Resolves all persistent records registered to this repository (via {@link
+     * #getManagedRecords}. This will be done before the repository is initialized via {@link
+     * #init}.
      */
-    protected void init ()
+    protected void resolveRecords ()
         throws DatabaseException
     {
         Set<Class<? extends PersistentRecord>> classes =
@@ -97,6 +100,41 @@ public abstract class DepotRepository
         }
     }
 
+    /**
+     * Provides a place where a repository can perform any initialization that requires database
+     * operations.
+     */
+    protected void init ()
+        throws DatabaseException
+    {
+        // run any registered data migrations
+        for (DataMigration migration : _dataMigs) {
+            runMigration(migration);
+        }
+        _dataMigs = null; // note that we've been initialized
+    }
+
+    /**
+     * Registers a data migration for this repository. This migration will only be run once and its
+     * unique identifier will be stored persistently to ensure that it is never run again on the
+     * same database. Nonetheless, migrations should strive to be idempotent because someone might
+     * come along and create a brand new system installation and all registered migrations will be
+     * run once on the freshly created database. As with all database migrations, understand
+     * clearly how the process works and think about edge cases when creating a migration.
+     *
+     * <p> See {@link PersistenceContext#registerMigration} for details on how schema migrations
+     * operate and how they might interact with data migrations.
+     */
+    protected void registerMigration (DataMigration migration)
+    {
+        if (_dataMigs == null) {
+            // we've already been initialized, so we have to run this migration immediately
+            runMigration(migration);
+        } else {
+            _dataMigs.add(migration);
+        }
+    }
+
     /**
      * Adds the persistent classes used by this repository to the supplied set.
      */
@@ -889,5 +927,67 @@ public abstract class DepotRepository
         }
     }
 
+    /**
+     * If the supplied migration has not already been run, it will be run and if it completes, we
+     * will note in the DepotMigrationHistory table that it has been run.
+     */
+    protected void runMigration (DataMigration migration)
+        throws DatabaseException
+    {
+        // attempt to get a lock to run this migration (or detect that it has already been run)
+        DepotMigrationHistoryRecord record;
+        while (true) {
+            // check to see if the migration has already been completed
+            record = load(DepotMigrationHistoryRecord.class, migration.getIdent());
+            if (record != null && record.whenCompleted != null) {
+                return; // great, no need to do anything
+            }
+
+            // if no record exists at all, try to insert one and thereby obtain the migration lock
+            if (record == null) {
+                try {
+                    record = new DepotMigrationHistoryRecord();
+                    record.ident = migration.getIdent();
+                    insert(record);
+                    break; // we got the lock, break out of this loop and run the migration
+                } catch (DuplicateKeyException dke) {
+                    // someone beat us to the punch, so we have to wait for them to finish
+                }
+            }
+
+            // we didn't get the lock, so wait 5 seconds and then check to see if the other process
+            // finished the update or failed in which case we'll try to grab the lock ourselves
+            try {
+                log.info("Waiting on migration lock for " + migration.getIdent() + ".");
+                Thread.sleep(5000);
+            } catch (InterruptedException ie) {
+                throw new DatabaseException("Interrupted while waiting on migration lock.");
+            }
+        }
+
+        log.info("Running data migration", "ident", migration.getIdent());
+        try {
+            // run the migration
+            migration.invoke();
+
+            // report to the world that we've done so
+            record.whenCompleted = new Timestamp(System.currentTimeMillis());
+            update(record);
+
+        } finally {
+            // clear out our migration history record if we failed to get the job done
+            if (record.whenCompleted == null) {
+                try {
+                    delete(record);
+                } catch (Throwable dt) {
+                    log.warning("Oh noez! Failed to delete history record for failed migration. " +
+                                "All clients will loop forever waiting for the lock.",
+                                "ident", migration.getIdent(), dt);
+                }
+            }
+        }
+    }
+
     protected PersistenceContext _ctx;
+    protected List<DataMigration> _dataMigs = new ArrayList<DataMigration>();
 }

src/java/com/samskivert/jdbc/depot/PersistenceContext.java

@@ -212,6 +212,12 @@ public class PersistenceContext
      * <p> If you want a completely predictable migration process, never use the default migrations
      * and register a pre-migration for every single schema migration and they will then be
      * guaranteed to be run in registration order and with predictable pre- and post-conditions.
+     *
+     * <p> Note that if {@link PersistenceContext#initializeRepositories} is used, then all schema
+     * migrations for all known repositories will be run and then all data migrations for all known
+     * repositories will be run. This is recommeneded because schema migrations may fail and it is
+     * generally better to have not yet done the data migration rather than having schema and data
+     * migrations interleaved and potentially leaving the database in a strange state.
      */
     public <T extends PersistentRecord> void registerMigration (
         Class<T> type, SchemaMigration migration)
@@ -468,14 +474,18 @@ public class PersistenceContext
     public void initializeRepositories (boolean warnOnLazyInit)
         throws DatabaseException
     {
-        // initialize our repositories
+        // resolve all persistent records and trigger all schema migrations
+        for (DepotRepository repo : _repositories) {
+            repo.resolveRecords();
+        }
+        // then run all repository initialization methods, triggering all data migrations
         for (DepotRepository repo : _repositories) {
             repo.init();
         }
-        // note that we've now been initialized
-        _repositories = null;
-        // now issue a warning if we lazily initialize any other persistent record
+        // now potentially issue a warning if we lazily initialize any other persistent record
         _warnOnLazyInit = warnOnLazyInit;
+        // finally note that we've now been initialized
+        _repositories = null;
     }
 
     /**
@@ -489,6 +499,7 @@ public class PersistenceContext
             if (_warnOnLazyInit) {
                 log.warning("Repository created lazily: " + repo.getClass().getName());
             }
+            repo.resolveRecords();
             repo.init();
         } else {
             _repositories.add(repo);

src/java/com/samskivert/jdbc/depot/SchemaMigration.java

@@ -30,10 +30,14 @@ import com.samskivert.jdbc.DatabaseLiaison;
 import static com.samskivert.jdbc.depot.Log.log;
 
 /**
- * These can be registered with the {@link PersistenceContext} to effect hand-coded migrations
- * between entity versions. The modifier should override {@link #invoke} to perform its
- * migrations. See {@link PersistenceContext#registerPreMigration} and {@link
- * PersistenceContext#registerPostMigration}.
+ * Encapsulates the migration of a persistent record's database schema. These can be registered
+ * with the {@link PersistenceContext} to effect hand-coded migrations between entity versions. The
+ * modifier should override {@link #invoke} to perform its migrations. See {@link
+ * PersistenceContext#registerPreMigration} for details on the migration process.
+ *
+ * <p> Note: these should only be used for actual schema changes (column additions, removals,
+ * renames, retypes, etc.). It should not be used for data migration, use {@link DataMigration} for
+ * that.
  */
 public abstract class SchemaMigration extends Modifier
 {