@kwaeri/migration/src/migration.ts

The @kwaeri/migration component of the @kwaer/node-kit application platform.

/*-----------------------------------------------------------------------------
 * @package:    node-kit migration
 * @author:     Richard B Winters
 * @copyright:  2015-2019 Massively Modified, Inc.
 * @license:    Apache-2.0
 * @version:    0.1.2
 *---------------------------------------------------------------------------*/
'use strict'


// INCLUDES
import { kdt } from '@kwaeri/developer-tools';
import { Filesystem } from '@kwaeri/filesystem';
import { Configuration } from '@kwaeri/configuration';
import { Driver } from '@kwaeri/driver/build/src/driver';
import { MySQLDriver } from '@kwaeri/driver/build/src/mysql';
import { PostgreSQLDriver } from '@kwaeri/driver/build/src/postgres';
import { Console } from '@kwaeri/console';
import debug from 'debug';


// DEFINES
let _       = new kdt(),
    output  = new Console( { color: false,  background: false, decor: [] } );

export type MigrationRecord =
{
    id: number;
    date: string;
    name: string;
    sequel: string;
    applied: boolean;
};

export type MigrationPromise =
{
    version?: string;
    result: boolean;
    migrations: MigrationRecord[];
};

export type MigratorConfiguration =
{
    version: string;
    type?: string;
    table?: string;
    environment?: string;
};

export type MigrationMap =
{
    timestamp: string;
    name: string;
    path: string;
};

/* Configure Debug module support */
let DEBUG = debug( 'nkm:migrator' );


/**
 * The Migrator Class
 *
 * The { Migrator } class handles everything that has to do with the migration
 * system, including; Setup, Management, and Execution of migrations.
 */
export class Migration extends Filesystem
{
    /**
     * @var { string }
     */
    path = 'data/migrations';


    /**
     * @var { string }
     */
    file = 'migrations.json';


    /**
     * @var {string }
     */
    version = '0.1.10';


    /**
     * @var { string }
     */
    type = 'mysql';


    /**
     * @var { string }
     */
    table = 'nodekit_migrations';


    /**
     * @var { Configuration }
     */
    configuration: Configuration;


    /**
     * @var { Configuration }
     */
    databaseConfiguration: Configuration;


    /**
     * @var { MySQLDriver|PostgreSQLDriver }
     */
    dbo: MySQLDriver|PostgreSQLDriver;


    /**
     * Class constructor
     *
     * @since 0.1.10
     */
    constructor( configuration?: MigratorConfiguration )
    {
        super();

        // Organize all of the uncertainty:
        this.version     = ( configuration && configuration.version ) ? configuration.version : null,
        this.type        = ( configuration && configuration.type ) ? configuration.type : "mysql",
        this.table       = ( configuration && configuration.table ) ? configuration.table : "nodekit_migrations";

        // Get the current environment:
        let environment = ( configuration && configuration.environment ) ? configuration.environment : "default";

        // Instantiate a new configuration object to wrap the migrations configuration:
        this.configuration = new Configuration( "data/migrations", "migrations.json", "migrations" );

        // Grab the database configuration pre-emptively as well, if it exists:
        this.databaseConfiguration = new Configuration( 'conf', `database.${environment}.json`, 'database' );
    }


    /**
     * Checks if the migration system has been installed
     *
     * @pram { void }
     *
     * @return { Promise<boolean> } A promise with a boolean value indicating whether the migration system is installed.
     *
     * @since 0.1.10
     */
    async checkInstall<T extends MigrationPromise>(): Promise<boolean>
    {
        // Check that the migration configuration exists @ ./data/migrations/:
        let installed = await this.configuration.get();

        // If not -> return false (It will generate the migration configuration):
        if( !installed.success )
        {
            return Promise.resolve( false );
        }

        // Next, grab the database configuration:
        let dbConf = await this.databaseConfiguration.get();

        // If it doesn't exist -> throw an error, a database configuration should
        // have already been set up:
        if( !dbConf.success )
        {
            return Promise.reject( new Error( `[MIGRATOR][CHECK_INSTALL] There was an issue reading the database configuration. ` ) );
        }

        // Otherwise instantiate a database driver to query the database with:
        this.dbo = new Driver( dbConf.configuration ).get();

        // Finally, run a query to determine if the migrations table exists::
        let checkMigrationsInstalledResult = await this.dbo
        .query( `SELECT table_name FROM information_schema.tables WHERE table_schema = '${dbConf.configuration.database}' AND table_name = '${this.table}';` );

        DEBUG( checkMigrationsInstalledResult );
        DEBUG( checkMigrationsInstalledResult.rows );

        // Check if there was some sort of error with our query:
        if( !checkMigrationsInstalledResult || !checkMigrationsInstalledResult.rows || _.empty( checkMigrationsInstalledResult.rows ) )
        {
            return Promise.reject( new Error( `[MIGRATOR][CHECK_INSTALL] There was an issue checking for the migrations table. If the migrations table does not exist, delete '${this.file}' in '${this.path}' and generate a new migration ('nkm add migration <MigrationName>') to force install the migrations system.`) );
        }

        // Alert caller that the migration system is installed:
        return Promise.resolve( true );
    }


    /**
     * Installs the migration system
     *
     * @param { string } content The migration configuration file content
     *
     * @return { Promise<T> } A promise that indicates the result of installing the migration system
     *
     * @since 0.1.10
     */
    async install<T extends MigrationPromise>( content: string ): Promise<T>
    {
        // First write the migration configuration to disk:
        let createdMigrationsConfiguration = await this.createFile( this.path, this.file, content );

        // If there was an error, reject the promise:
        if( !createdMigrationsConfiguration.result )
        {
            return Promise.reject( new Error( `[MIGRATOR][INSTALL] There was an issue writing the migration configuration to disk. ` ) );
        }

        // Next, get the databae configuration:
        let dbConf = await this.databaseConfiguration.get();

        // If there was an error, reject the promise::
        if( !dbConf.success )
        {
            return Promise.reject( new Error( `[MIGRATOR][INSTALL] There was an issue reading the database configuration. ` ) );
        }

        // Next, create the migrations table:
        this.dbo = new Driver( dbConf.configuration ).get();

        let createMigrationsTable = await this.dbo
        .query( `create table if not exists ${this.table} ` +
                `( id int(11) not null auto_increment, ` +
                `date varchar(255), ` +
                `name varchar(255), ` +
                `sequel text, ` +
                `applied int(11), ` +
                `primary key (id), ` +
                `unique key unique_date (date) );` );

        // If the query failed:
        if( !createMigrationsTable && !createMigrationsTable.rows )
        {
            // Reject the promise:
            return Promise.reject( new Error( `[MIGRATOR][INSTALL] There was an error installing migrations into the project. ` ) );
        }

        // Return a resolved promise:
        return Promise.resolve( <T>{ result: true, migrations: [] } );
    }


    /**
     * Applies or reverts [a] migration(s)
     *
     * @param options Parameters that shape the migration command
     *
     * @returns { Promise<MigrationPromise> }
     *
     * @since 0.1.0
     */
    async migrate<T extends MigrationPromise>( options ): Promise<T>
    {
        let migrated: MigrationPromise  = null,
            startMigrationTS            = new Date(),
            timeOptions                 = { hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false };


        // First check that the migration system is already installed - if it is NOT
        // then we will need to install it and generate a migration prior to running
        // any migration(s):
        //console.log
        //(
        //    output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
        //    .buffer( new Date().toLocaleTimeString( "en-us", timeOptions ) )
        //    .normalize().buffer( `] ` ).decor( ['bright'] )
        //    .buffer( `Checking the migration system install.` ).normalize().dump()
        //);
        DEBUG( `Check migration system installation` );

        let installed = await this.checkInstall();

        if( !installed )
        {
            return Promise.reject( new Error( `[MIGRATOR][RUN] Unable to verify whether the migration system is installed. If the 'data/migrations' path and/or the 'data/migrations/migrations.json' configuration are missing - generate a migration in order to force installation on your system. ` ) );
        }


        // Next we need to get a list of files within the 'data/migrations' directory:
        //console.log
        //(
        //    output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
        //    .buffer( new Date().toLocaleTimeString( "en-us", timeOptions ) )
        //    .normalize().buffer( `] ` ).decor( ['bright'] )
        //    .buffer( `Getting the list of migrations.` ).normalize().dump()
        //);
        DEBUG( `Get list of available migrations` );

        let hierarchy               = { root: { type: "directory", children: {} } };
            hierarchy.root.children = await this.getDirectoryStructure( this.path );

        // We know the directory structure so lets loop through each month of each year
        // and make a list of all migrations:
        let files   = [],
            map     = {},
            count   = 0;

        // Loop through each year
        for( let year in ( hierarchy.root.children as any ).files )
        {
            // For each year, loop through each month:
            for( let month in ( hierarchy.root.children as any ).files[year].children )
            {
                // For each month, add the migrations to our list:
                for( let migration in ( hierarchy.root.children as any ).files[year].children[month].children )
                {
                    // Create a simple array to store the migration files in, this is the device
                    // we'll use to sort and ensure the proper sequence of migrations:
                    files.push( migration );

                    // Build a map to keep necessary data available to us:
                    map[files[count]] = { timestamp: migration.split( '_' )[0], name: migration.split( '_')[1].split( '.js')[0], path: this.path + '/' + year + '/' + month + '/' + migration };

                    DEBUG( `Migration '${map[files[count]].name}' found at '${map[files[count]].path}'` );

                    count++;
                }
            }
        }

        // Now sort the keys array so we apply migrations in order!:
        files = files.sort( this.alphaNumericSort );


        // Next we need a list of migrations which have already been applied. Fetch
        // the database configuration:
        //console.log
        //(
        //    output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
        //    .buffer( new Date().toLocaleTimeString( "en-us", timeOptions ) )
        //    .normalize().buffer( `] ` ).decor( ['bright'] )
        //    .buffer( `Getting the list of applied migrations.` ).normalize().dump()
        //);
        DEBUG( `Get list of applied migrations` );

        let dbConf = await this.databaseConfiguration.get();

        // If there was an error, reject the promise::
        if( !dbConf.success )
        {
            return Promise.reject( new Error( `[MIGRATOR][MIGRATE] There was an issue reading the database configuration. ` ) );
        }

        // Get a database object::
        this.dbo = new Driver( dbConf.configuration ).get();

        // Query the migrations table for applied migrations:
        let appliedMigrations = await this.dbo
        .query(
            `select * from ${this.table} ` +
            `where applied=1 ` +
            `order by date asc, name asc;`
        );

        // If the query failed:
        if( !appliedMigrations && !appliedMigrations.rows )
        {
            // Reject the promise:
            return Promise.reject( new Error( `[MIGRATOR][MIGRATE] There was an error getting a list of applied migrations from ${this.table} table in the database. ` ) );
        }

        // Now we need to take any rows, and rebuild the migration filename to use as a
        // comparitor so as to ensure we do not rerun the respective migration as we
        // loop through our list of migrations and apply any unapplied migrations:
        let applied = [];

        for( let record in appliedMigrations.rows )
        {
            applied.push( `${appliedMigrations.rows[record].date}_${appliedMigrations.rows[record].name}.js` );
        }


        // Since we sort by the date and name it should already be sorted appropriately, queue
        // up the migrations to process, and process them - keeping in mind whether we are
        // processing forward or stepping back:
        //DEBUG( `${(options.stepBack)?`Step-back ${options.stepBack} migrations` : 'Process migrations forward '}` );
        let processed = 0;
        console.log
        (
            output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
            .buffer( new Date().toLocaleTimeString( "en-us", timeOptions ) )
            .normalize().buffer( `] ` ).decor( ['bright'] )
            .buffer( `${(options.stepBack)?`Stepping back ${options.stepBack} migrations.\n` : 'Processing migrations forward.\n'}` ).normalize().dump()
        );

        let workingSet = [];
        if( !options.stepBack )
        {
            // If we're not stepping back the working set is the files we have in our data/migrations
            // directory, and we'll apply only migrations which haven't already been applied:
            workingSet = files;
        }
        else
        {
            // If we ARE stepping back, the working set will be only the migrations we are going to
            // revert -> so we'll slice the working set such that we only have the number of migrations
            // that we are stepping back, to work with:
            workingSet = applied;

            // Now let's only get the migrations we're going to revert:
            workingSet = workingSet.slice( workingSet.length - options.stepBack, workingSet.length );

            // Finally, let's re-sort them in reverse:
            workingSet = workingSet.reverse();
        }

        if( workingSet.length )
        {
            for( let toProcess in workingSet )
            {
                // Because of the simplicity in the logic behind applying migrations, we need to
                // ensure that if we are applying migrations, that we do not attempt to apply
                // any migrations we iterate over that have already been applied. However, if
                // we're reverting, we actually don't need this check at all, because were ONLY
                // reverting migrations that have for sure been applied -> but we have to create
                // a condition where this code flow is entered:
                if( ( !options.stepBack && applied.indexOf( workingSet[toProcess] ) < 0 ) || ( options.stepBack && applied.indexOf( workingSet[toProcess] ) >= 0 ) )
                {
                    let startProcessTS = new Date();
                    console.log
                    (
                        output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
                        .buffer( startProcessTS.toLocaleTimeString( "en-us", timeOptions ) )
                        .normalize().buffer( `] ` ).decor( ['bright'] )
                        .buffer( `${(options.stepBack)? `Reverting '` : `Applying '`}` ).color( 'cyan' )
                        .buffer( `${map[workingSet[toProcess]].name}` ).color( 'white' ).buffer( `'` ).normalize().dump()
                    );

                    DEBUG( `Checking migration '${map[workingSet[toProcess]].name}` );

                    // As part of the queueing process, check whether the migration is already added to
                    // the migrations table and set as unapplied:
                    let migrationIsQueued = await this.dbo
                    .query(
                        `select * from ${this.table} ` +
                        `where date='${map[workingSet[toProcess]].timestamp}' and name='${map[workingSet[toProcess]].name}' and applied=0;`
                    );

                    if( !migrationIsQueued || !migrationIsQueued.rows )
                    {
                        return Promise.reject( new Error( `[MIGRATION][MIGRATE] There was an issue checking whether a queued migration had been recorded into the ${this.table} table in the database, but had not been applied. ` ) );
                    }

                    // If not, we need to queue it in the database first, then process it. For reverting
                    // migrations, this will conveniently never matter:
                    if( ( migrationIsQueued.rows as any ).length <= 0 && !options.stepBack )
                    {
                        DEBUG( `Queueing migration '${map[workingSet[toProcess]].name}'` );

                        let queueMigration = await this.dbo
                        .query(
                            `insert into ${this.table} ` +
                            `(date, name, applied) ` +
                            `values( '${map[workingSet[toProcess]].timestamp}', '${map[workingSet[toProcess]].name}', 0);`
                        );

                        if( !queueMigration || !queueMigration.rows )
                        {
                            return Promise.reject( new Error( `[MIGRATION][${map[workingSet[toProcess]].name}] There was an issue queueing the migration in the ${this.table} table in the database. ` ) );
                        }
                    }

                    DEBUG( `Migration '${map[workingSet[toProcess]].name}' ${(options.stepBack)? 'to revert was previously applied' : 'queued'}, processing...` );

                    // Require the migration file:
                    let applicableMigration = require( `${Filesystem.getPathToCWD()}/${map[workingSet[toProcess]].path}` );

                    // Instantiate the migration class:
                    let applicableMigrator = new applicableMigration();

                    // Composition rocks:
                    applicableMigrator.dbo = this.dbo;

                    // Run the migration:
                    let migrationProcessed;

                    // If it's not a stepBack request, step up!:
                    if( !options.stepBack )
                    {
                        migrationProcessed = await applicableMigrator.up();
                    }
                    else
                    {
                        migrationProcessed = await applicableMigrator.down();
                    }

                    if( !migrationProcessed || !migrationProcessed.result || !migrationProcessed.rows )
                    {
                        return Promise.reject( new Error( `[MIGRATION][${map[workingSet[toProcess]].name}]: There was an issue processing the migration. `) );
                    }

                    DEBUG( `Migration '${map[workingSet[toProcess]].name}' ${(options.stepBack) ? 'reverted' : 'applied'}...` );

                    // We then need to update the table record such that it reflects
                    // that we have applied the migration:
                    let markApplied = await this.dbo
                    .query(
                        `update ${this.table} set applied=${(options.stepBack) ? '0' : '1'} ` +
                        `where name='${map[workingSet[toProcess]].name}' and date='${map[workingSet[toProcess]].timestamp}';`
                    );

                    if( !markApplied || !markApplied.rows )
                    {
                        return Promise.reject( new Error( `[MIGRATION][${map[workingSet[toProcess]].name}]: There was an issue marking a${(options.stepBack) ? ' reverted migration as unapplied' : 'n applied migration as completed.'}` ) );
                    }

                    let finishProcessTS = new Date();
                    console.log
                    (
                        output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
                        .buffer( finishProcessTS.toLocaleTimeString( "en-us", timeOptions ) )
                        .normalize().buffer( `] ` ).decor( ['bright'] )
                        .buffer( `${(options.stepBack)? `Finished reverting '` : `Finished applying '`}` ).color( 'cyan' )
                        .buffer( `${map[workingSet[toProcess]].name}` ).color( 'white' ).buffer( `' after ` )
                        .color( 'magenta' ).buffer( `${finishProcessTS.getTime() - startProcessTS.getTime()} ms\n` ).normalize().dump()
                    );

                    processed++;
                }
            }

            let finishedMigrationTS = new Date();
            console.log
            (
                output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
                .buffer( finishedMigrationTS.toLocaleTimeString( "en-us", timeOptions ) )
                .normalize().buffer( `] ` ).decor( ['bright'] )
                .buffer
                (
                    `${(processed > 0) ? `${(options.stepBack)? `Successfully reverted ${options.stepBack} migrations.` : `Successfully applied ${processed} migrations.`}` :
                                            `${(options.stepBack)? `Nothing to revert,` : `Migrations are up to date.`}`}`

                ).color( 'cyan' )
                .color( 'white' ).buffer( ` after ` )
                .color( 'magenta' ).buffer( `${finishedMigrationTS.getTime() - startMigrationTS.getTime()} ms` ).normalize().dump()
            );
        }
        else
        {
            let finishedMigrationTS = new Date();
            console.log
            (
                output.normalize().buffer( `[` ).color( 'gray' ).decor( ['bright'] )
                .buffer( finishedMigrationTS.toLocaleTimeString( "en-us", timeOptions ) )
                .normalize().buffer( `] ` ).decor( ['bright'] )
                .buffer( `${(options.stepBack)? `No applied migrations exist.` : `No migrations exist.`}` ).color( 'cyan' )
                .color( 'white' ).buffer( ` after ` )
                .color( 'magenta' ).buffer( `${finishedMigrationTS.getTime() - startMigrationTS.getTime()} ms` ).normalize().dump()
            );
        }

        migrated = { result: true, migrations: ( hierarchy.root.children as any ).files };

        return Promise.resolve( <T>{ ...migrated } );
    }


    /**
     * [DEPRECATED] Runs a [series of] migration(s), if any exist which have not already been applied
     *
     * @param void
     *
     * @returns { Promise<T> } A promise indicating the result of running migrations
     *
     * @since 0.1.11
     */
    async run<T extends MigrationPromise>( options ): Promise<T>
    {
        // First and foremost let's grab

        // First check that the migration system is already installed - if it is NOT
        // then we will install it prior to running any migration(s):
        let installed = await this.checkInstall();

        if( !installed )
        {
            return Promise.reject( new Error( `[MIGRATOR][RUN] Unable to verify whether the migration system is installed. If the 'data/migrations' path and/or the 'data/migrations/migrations.json' configuration are missing - generate a migration in order to force installation on your system. ` ) );
        }

        return Promise.resolve( <T>{ result: true } );
    }


    /**
     * [DEPRECATED] Adds a migration to the filesystem
     *
     * @param void
     *
     * @returns  { Promise<MigrationPromise> }
     *
     * @since 0.1.0
     */
    async add<T extends MigrationPromise>(): Promise<T>
    {
        return Promise.resolve( <T>{ result: true } );
    }

}