From fee6be6c2d99808425ff0142b759bd05592ae241 Mon Sep 17 00:00:00 2001 From: Michael Bayne Date: Thu, 25 Sep 2008 23:40:01 +0000 Subject: [PATCH] Added support for data migrations, which are run after all schema migrations have been run and are also coordinated between disparate processes by an in-database lock system. --- .../samskivert/jdbc/depot/DataMigration.java | 73 ++++++++++++ .../jdbc/depot/DepotMarshaller.java | 4 +- .../depot/DepotMigrationHistoryRecord.java | 73 ++++++++++++ .../jdbc/depot/DepotRepository.java | 108 +++++++++++++++++- .../jdbc/depot/PersistenceContext.java | 19 ++- .../jdbc/depot/SchemaMigration.java | 12 +- 6 files changed, 275 insertions(+), 14 deletions(-) create mode 100644 src/java/com/samskivert/jdbc/depot/DataMigration.java create mode 100644 src/java/com/samskivert/jdbc/depot/DepotMigrationHistoryRecord.java diff --git a/src/java/com/samskivert/jdbc/depot/DataMigration.java b/src/java/com/samskivert/jdbc/depot/DataMigration.java new file mode 100644 index 0000000..c05548f --- /dev/null +++ b/src/java/com/samskivert/jdbc/depot/DataMigration.java @@ -0,0 +1,73 @@ +// +// $Id$ +// +// samskivert library - useful routines for java programs +// Copyright (C) 2001-2008 Michael Bayne +// +// This library is free software; you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published +// by the Free Software Foundation; either version 2.1 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package com.samskivert.jdbc.depot; + +/** + * Encapsulates a migration of data between entities that should be run only once and using the + * same safeguards applied to entity migrations. Note: this should not be used for schema + * migrations, use {@link SchemaMigration} for that. Data migrations are registered on a specific + * repository via {@link DepotRepository#registerMigration} and should be registered in the + * repository's constructor as they will be invoked (if appropriate) in the repository's {@link + * DepotRepository#init} method. + * + *

In general one will register an anonymous inner class in a repository's constructor and can + * then access methods in the repository directly: + * + * 

+ * public class FooRepository extends DepotRepository {
+ *     public FooRepository (PersistenceContext ctx) {
+ *         super(ctx);
+ *         registerMigration(new DataMigration("2008_09_25_referral_to_tracking_id") {
+ *             public void invoke () throws DatabaseException {
+ *                 // feel free to use load() findAll(), update(), etc.
+ *             }
+ *         });
+ *     }
+ *
+ */ +public abstract class DataMigration +{ + /** + * Creates a data migration with the specified unique identifier. The identifier must be unique + * across all users of the database and for all time, so be careful. Best to include the date + * and pertintent information, e.g. "2008_09_25_referral_to_tracking_id". + */ + public DataMigration (String ident) + { + _ident = ident; + } + + /** + * Returns the identifier of this migration. + */ + public String getIdent () + { + return _ident; + } + + /** + * Effects the data migration. + */ + public abstract void invoke () throws DatabaseException; + + /** The unique identifier for this migration. */ + protected String _ident; +} diff --git a/src/java/com/samskivert/jdbc/depot/DepotMarshaller.java b/src/java/com/samskivert/jdbc/depot/DepotMarshaller.java index 5f024de..ac517de 100644 --- a/src/java/com/samskivert/jdbc/depot/DepotMarshaller.java +++ b/src/java/com/samskivert/jdbc/depot/DepotMarshaller.java @@ -630,10 +630,10 @@ public class DepotMarshaller // we didn't get the lock, so wait 5 seconds and then check to see if the other process // finished the update or failed in which case we'll try to grab the lock ourselves try { - log.info("Waiting for migration lock for " + _pClass.getName() + "."); + log.info("Waiting on migration lock for " + _pClass.getName() + "."); Thread.sleep(5000); } catch (InterruptedException ie) { - throw new DatabaseException("Interrupted while waiting for migration lock."); + throw new DatabaseException("Interrupted while waiting on migration lock."); } } diff --git a/src/java/com/samskivert/jdbc/depot/DepotMigrationHistoryRecord.java b/src/java/com/samskivert/jdbc/depot/DepotMigrationHistoryRecord.java new file mode 100644 index 0000000..f3d5a63 --- /dev/null +++ b/src/java/com/samskivert/jdbc/depot/DepotMigrationHistoryRecord.java @@ -0,0 +1,73 @@ +// +// $Id$ +// +// samskivert library - useful routines for java programs +// Copyright (C) 2001-2008 Michael Bayne +// +// This library is free software; you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published +// by the Free Software Foundation; either version 2.1 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package com.samskivert.jdbc.depot; + +import java.sql.Timestamp; + +import com.samskivert.jdbc.depot.annotation.Column; +import com.samskivert.jdbc.depot.annotation.Id; +import com.samskivert.jdbc.depot.expression.ColumnExp; + +/** + * Maintains a record of all successfully invoked data migrations. + */ +public class DepotMigrationHistoryRecord extends PersistentRecord +{ + // AUTO-GENERATED: FIELDS START + /** The column identifier for the {@link #ident} field. */ + public static final String IDENT = "ident"; + + /** The qualified column identifier for the {@link #ident} field. */ + public static final ColumnExp IDENT_C = + new ColumnExp(DepotMigrationHistoryRecord.class, IDENT); + + /** The column identifier for the {@link #whenCompleted} field. */ + public static final String WHEN_COMPLETED = "whenCompleted"; + + /** The qualified column identifier for the {@link #whenCompleted} field. */ + public static final ColumnExp WHEN_COMPLETED_C = + new ColumnExp(DepotMigrationHistoryRecord.class, WHEN_COMPLETED); + // AUTO-GENERATED: FIELDS END + + /** Our schema version. Probably not likely to change. */ + public static final int SCHEMA_VERSION = 1; + + /** The unique identifier for this migration. */ + @Id public String ident; + + /** The time at which the migration was completed. */ + @Column(nullable=true) + public Timestamp whenCompleted; + + // AUTO-GENERATED: METHODS START + /** + * Create and return a primary {@link Key} to identify a {@link DepotMigrationHistoryRecord} + * with the supplied key values. + */ + public static Key getKey (String ident) + { + return new Key( + DepotMigrationHistoryRecord.class, + new String[] { IDENT }, + new Comparable[] { ident }); + } + // AUTO-GENERATED: METHODS END +} diff --git a/src/java/com/samskivert/jdbc/depot/DepotRepository.java b/src/java/com/samskivert/jdbc/depot/DepotRepository.java index 7b9dad4..8bc7ddc 100644 --- a/src/java/com/samskivert/jdbc/depot/DepotRepository.java +++ b/src/java/com/samskivert/jdbc/depot/DepotRepository.java @@ -24,6 +24,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; @@ -50,6 +51,8 @@ import com.samskivert.jdbc.depot.clause.UpdateClause; import com.samskivert.jdbc.depot.expression.SQLExpression; import com.samskivert.jdbc.depot.expression.ValueExp; +import static com.samskivert.jdbc.depot.Log.log; + /** * Provides a base for classes that provide access to persistent objects. Also defines the * mechanism by which all persistent queries and updates are routed through the distributed cache. @@ -82,11 +85,11 @@ public abstract class DepotRepository } /** - * Provides a place where a repository can perform any initialization that requires database - * operations. The default implementation resolves all records managed by this repository so - * that their schema migrations are run. + * Resolves all persistent records registered to this repository (via {@link + * #getManagedRecords}. This will be done before the repository is initialized via {@link + * #init}. */ - protected void init () + protected void resolveRecords () throws DatabaseException { Set> classes = @@ -97,6 +100,41 @@ public abstract class DepotRepository } } + /** + * Provides a place where a repository can perform any initialization that requires database + * operations. + */ + protected void init () + throws DatabaseException + { + // run any registered data migrations + for (DataMigration migration : _dataMigs) { + runMigration(migration); + } + _dataMigs = null; // note that we've been initialized + } + + /** + * Registers a data migration for this repository. This migration will only be run once and its + * unique identifier will be stored persistently to ensure that it is never run again on the + * same database. Nonetheless, migrations should strive to be idempotent because someone might + * come along and create a brand new system installation and all registered migrations will be + * run once on the freshly created database. As with all database migrations, understand + * clearly how the process works and think about edge cases when creating a migration. + * + *

See {@link PersistenceContext#registerMigration} for details on how schema migrations + * operate and how they might interact with data migrations. + */ + protected void registerMigration (DataMigration migration) + { + if (_dataMigs == null) { + // we've already been initialized, so we have to run this migration immediately + runMigration(migration); + } else { + _dataMigs.add(migration); + } + } + /** * Adds the persistent classes used by this repository to the supplied set. */ @@ -889,5 +927,67 @@ public abstract class DepotRepository } } + /** + * If the supplied migration has not already been run, it will be run and if it completes, we + * will note in the DepotMigrationHistory table that it has been run. + */ + protected void runMigration (DataMigration migration) + throws DatabaseException + { + // attempt to get a lock to run this migration (or detect that it has already been run) + DepotMigrationHistoryRecord record; + while (true) { + // check to see if the migration has already been completed + record = load(DepotMigrationHistoryRecord.class, migration.getIdent()); + if (record != null && record.whenCompleted != null) { + return; // great, no need to do anything + } + + // if no record exists at all, try to insert one and thereby obtain the migration lock + if (record == null) { + try { + record = new DepotMigrationHistoryRecord(); + record.ident = migration.getIdent(); + insert(record); + break; // we got the lock, break out of this loop and run the migration + } catch (DuplicateKeyException dke) { + // someone beat us to the punch, so we have to wait for them to finish + } + } + + // we didn't get the lock, so wait 5 seconds and then check to see if the other process + // finished the update or failed in which case we'll try to grab the lock ourselves + try { + log.info("Waiting on migration lock for " + migration.getIdent() + "."); + Thread.sleep(5000); + } catch (InterruptedException ie) { + throw new DatabaseException("Interrupted while waiting on migration lock."); + } + } + + log.info("Running data migration", "ident", migration.getIdent()); + try { + // run the migration + migration.invoke(); + + // report to the world that we've done so + record.whenCompleted = new Timestamp(System.currentTimeMillis()); + update(record); + + } finally { + // clear out our migration history record if we failed to get the job done + if (record.whenCompleted == null) { + try { + delete(record); + } catch (Throwable dt) { + log.warning("Oh noez! Failed to delete history record for failed migration. " + + "All clients will loop forever waiting for the lock.", + "ident", migration.getIdent(), dt); + } + } + } + } + protected PersistenceContext _ctx; + protected List _dataMigs = new ArrayList(); } diff --git a/src/java/com/samskivert/jdbc/depot/PersistenceContext.java b/src/java/com/samskivert/jdbc/depot/PersistenceContext.java index d51a0d5..810444c 100644 --- a/src/java/com/samskivert/jdbc/depot/PersistenceContext.java +++ b/src/java/com/samskivert/jdbc/depot/PersistenceContext.java @@ -212,6 +212,12 @@ public class PersistenceContext *

If you want a completely predictable migration process, never use the default migrations * and register a pre-migration for every single schema migration and they will then be * guaranteed to be run in registration order and with predictable pre- and post-conditions. + * + *

Note that if {@link PersistenceContext#initializeRepositories} is used, then all schema + * migrations for all known repositories will be run and then all data migrations for all known + * repositories will be run. This is recommeneded because schema migrations may fail and it is + * generally better to have not yet done the data migration rather than having schema and data + * migrations interleaved and potentially leaving the database in a strange state. */ public void registerMigration ( Class type, SchemaMigration migration) @@ -468,14 +474,18 @@ public class PersistenceContext public void initializeRepositories (boolean warnOnLazyInit) throws DatabaseException { - // initialize our repositories + // resolve all persistent records and trigger all schema migrations + for (DepotRepository repo : _repositories) { + repo.resolveRecords(); + } + // then run all repository initialization methods, triggering all data migrations for (DepotRepository repo : _repositories) { repo.init(); } - // note that we've now been initialized - _repositories = null; - // now issue a warning if we lazily initialize any other persistent record + // now potentially issue a warning if we lazily initialize any other persistent record _warnOnLazyInit = warnOnLazyInit; + // finally note that we've now been initialized + _repositories = null; } /** @@ -489,6 +499,7 @@ public class PersistenceContext if (_warnOnLazyInit) { log.warning("Repository created lazily: " + repo.getClass().getName()); } + repo.resolveRecords(); repo.init(); } else { _repositories.add(repo); diff --git a/src/java/com/samskivert/jdbc/depot/SchemaMigration.java b/src/java/com/samskivert/jdbc/depot/SchemaMigration.java index 7c220d9..a5b85a7 100644 --- a/src/java/com/samskivert/jdbc/depot/SchemaMigration.java +++ b/src/java/com/samskivert/jdbc/depot/SchemaMigration.java @@ -30,10 +30,14 @@ import com.samskivert.jdbc.DatabaseLiaison; import static com.samskivert.jdbc.depot.Log.log; /** - * These can be registered with the {@link PersistenceContext} to effect hand-coded migrations - * between entity versions. The modifier should override {@link #invoke} to perform its - * migrations. See {@link PersistenceContext#registerPreMigration} and {@link - * PersistenceContext#registerPostMigration}. + * Encapsulates the migration of a persistent record's database schema. These can be registered + * with the {@link PersistenceContext} to effect hand-coded migrations between entity versions. The + * modifier should override {@link #invoke} to perform its migrations. See {@link + * PersistenceContext#registerPreMigration} for details on the migration process. + * + *

Note: these should only be used for actual schema changes (column additions, removals, + * renames, retypes, etc.). It should not be used for data migration, use {@link DataMigration} for + * that. */ public abstract class SchemaMigration extends Modifier {