@yonderbox/graphql-contentful-mongodb-abstract-schema/cli.js

Imports Contentful space to MongoDB and generates an abstract schema

#!/usr/bin/env node
const pkg = require( './package.json' )

const yves = require( 'yves' )
yves.debugger( 'Y', { stream: null, sortKeys: true, hideFunctions: true, singleLineMax: 0, obfuscates: [ /key/i, /token/i ] } )

const debug = yves.debugger( `${pkg.name.replace( /^@/, '' ).replace( /[/-]+/g, ':' )}` )

const changeCase = require( 'change-case' )
const path = require( 'path' )

const glob = require( 'glob' )
const options = {}

const args = require( 'args' )
const spawn = require( 'child_process' ).spawn

const _ = require( 'lodash' )

process.on( 'unhandledRejection', ( reason, promise ) => {
  debug( 'Unhandled Rejection, reason: %s', reason.toString() )
} )


args
  .option( 'schema-name', 'Path to schema file', 'abstract' )
  .option( 'data-dir', 'The directory which holds the contentful backups', 'data' )
  .option( 'spaces-dir', 'The directory which holds the space schemas (abstract & GraphQL)', 'spaces' )
  .option( 'contentful-file', 'Specific Contentful file to use as source' )
  .option( 'schema-path', 'Path to schema file' )
  .option( 'skip-backup', 'Skip fetching Contentful backup', false )
  .option( 'skip-entries', 'Skip importing Contentful entries', false )
  .option( 'skip-assets', 'Skip importing Contentful assets', false )
  .option( 'skip-locales', 'Skip importing Contentful locales', false )
  .option( 'skip-drop-collections', 'Skip drop MongoDB collections', false )
  .option( 'skip-create-collections', 'Skip create MongoDB collections', false )
  .option( 'skip-create-indexes', 'Skip create MongoDB indexes', false )
  .option( 'skip-schema', 'Skip create schema file', false )
  .option( 'verbose', 'Verbose output' )
  .option( 'management-token', 'Contentful management API token' )
  .option( 'space-id', 'ID of Space with source data' )
  .option( 'environment-id', 'ID of Environment with source data', 'master' )
  .option( 'mongodb-url', 'ID of Environment with source data' )
  .option( 'config-filename', 'Config filename can optionally point to a JSON file, containing camelCased variables like the options here' )


let flags = args.parse( process.argv )

let config

if ( flags.configFilename ) {
  try {
    const filename = path.resolve( flags.configFilename )
    config = require( filename )
    config.name = path.basename( filename, '.js' )
    const setup = {}
    if ( config ) {
      if ( config.url ) setup.mongodbUrl = config.url
      if ( config.root ) setup.schemaName =  config.root
      if ( config.schema ) setup.schemaPath = config.schema
      if ( config.writeGraphQLfile ) setup.writeGraphQLfile = config.writeGraphQLfile
    }
    if ( Object.keys( setup ).length ) {
      debug( 'setup %y', setup )
      debug( 'flags %y', flags )
      flags = Object.assign( {}, setup, flags )
    }
  } catch( e ) {
    debug( 'error %y', e )
  }
}

if ( !flags.schemaPath ) flags.schemaPath = `${flags.schemaName}.schema.json`
debug( 'flags %y', flags )

const JSONStream = require( 'JSONStream' )
const fs = require( 'fs-extra' )
const es = require( 'event-stream' )

const Adapter = require( '@yonderbox/graphql-adapter' )

const adapterOptions = Object.assign( {}, config, {
  adapter: 'mongodb',
  url: flags.mongodbUrl,
  options: {
    // keepAlive: true,
    connectTimeoutMS: 30000,
    directConnection: true,
  },
  db: changeCase.snakeCase( flags.schemaName ),
  schema: flags.schemaPath,
  root: changeCase.pascalCase( flags.schemaName ),
  limit: 0,
  writeGraphQLfile: flags.writeGraphQLfile,
} )

debug( 'adapterOptions %y', adapterOptions )

const default_contentful_language = 'nl-NL'
const default_mongodb_language = 'none'

let language = null
const langauges = {}
const objectHash = require( 'object-hash' )

// # contentTypes
// # editorInterfaces
// # entries
// # assets
// # locales

function fetchBackup( callback ) {
  if ( !flags.skipBackup && flags.managementToken && flags.spaceId ) {
    const dirPath = `${flags.dataDir}/${changeCase.snakeCase( flags.schemaName )}`
    fs.ensureDirSync( dirPath )

    const options = [
      'space',
      'export',
      '--management-Token',
      flags.managementToken,
      '--space-id',
      flags.spaceId,
      '--environment-id',
      flags.environmentId,
      '--export-dir',
      dirPath,
      '--skip-webhooks',
      '--skip-roles',
      //      '--include-drafts',
      //      '--include-archived',
    ]
    /*    debug('contentful: %y',options)*/
    const cmd = spawn( 'contentful', options )

    cmd.stdout.on( 'data', function ( data ) {
      if ( flags.verbose ) {
        console.log( 'stdout: ' + data.toString() )
      } else {
        process.stdout.write( data.toString() )
      }
    } )

    cmd.stderr.on( 'data', function ( data ) {
      if ( flags.verbose ) {
        console.log( 'stderr: ' + data.toString() )
      }
    } )

    cmd.on( 'exit', function ( code ) {
      console.log( 'child process exited with code ' + code.toString() )

      callback && callback()
    } )
  } else {
    callback && callback()
  }
}


function getStream( rootName, contentfulFile ) {
  if ( contentfulFile ) {
    const stream = fs.createReadStream( contentfulFile, { encoding: 'utf8' } )
    const parser = JSONStream.parse( rootName/*['contentTypes', {recurse: true}, ['sys','fields']]*/ )
    return stream.pipe( parser )
  }
}

const schema = {
  collections: {
    assets: {
      type: 'collection',
      name: 'assets',
      schema: {
        _id: { type: 'string', required: true },
        title: { type: 'string', required: true },
        description: { type: 'string', required: true },
        file: {
          type: 'object',
          required: true,
          schema: {
            url: { type: 'string', required: true },
            details: {
              type: 'object',
              required: false,
              schema: {
                size: { type: 'integer', required: false },
                image: {
                  type: 'object',
                  required: false,
                  schema: {
                    width: { type: 'integer', required: false },
                    height: { type: 'integer', required: false },
                  },
                },
                filename: { type: 'string', required: false },
                contentType: { type: 'string', required: false },
              },
            },
            title: { type: 'string', required: false },
          },
        },
      },
    },
  },
  indexes: {}
}

const report = {}

function fetchSchema( adapter, contentfulFile ) {

  function ContentfulType( type ) {
    let result = 'string'
    switch ( type ) {
    case 'Symbol': // fall-through
    case 'Link': // fall-through
    case 'Text':
      result = 'string'
      break
    case 'Integer':
      result = 'integer'
      break
    case 'Number':
      result = 'float'
      break
    case 'Date':
      result = 'date'
      break
    case 'Boolean':
      result = 'boolean'
      break
    case 'Array':
      result = 'array'
      break
    case 'Location': // fall-through
    case 'Object': // fall-through
    case 'YonderBoxGraphQLTypeObject':
      result = 'object'
      break
    }
    return result
  }


  return getStream( 'contentTypes', contentfulFile ).pipe( es.mapSync( function ( data ) {
    for ( let cType of data ) {
      const collectionName = adapter.pluralize( changeCase.snakeCase( cType.name ) )
      schema.collections[ collectionName ] = { type: 'collection', name: collectionName }

      // if (cType.displayField) schema.collections[collectionName].displayField = cType.displayField
      if ( cType.description ) schema.collections[ collectionName ].description = cType.description

      schema.collections[ collectionName ].schema = {}
      schema.collections[ collectionName ].schema[ '_id' ] = { type: 'string', required: true }

      schema.collections[ collectionName ].schema[ 'meta' ] = { type: 'object', required: false, kind: 'meta', schema: {
        createdAt: { type: 'date', required: false },
        updatedAt: { type: 'date', required: false },
        createdBy: { type: 'string', required: false },
        updatedBy: { type: 'string', required: false },
        version: { type: 'integer', required: false },
        firstPublishedAt: { type: 'date', required: false },
        publishedAt: { type: 'date', required: false },
        publishedBy: { type: 'string', required: false },
        publishedVersion: { type: 'integer', required: false },
        publishedCounter: { type: 'integer', required: false },
      } }

      for ( let field of cType.fields ) {
        if ( !field.disabled && !field.ommited ) {
          schema.collections[ collectionName ].schema[ field.id ] = {}
          const type = ContentfulType( field.type )
          schema.collections[ collectionName ].schema[ field.id ].type = type
          if ( field.validations && field.validations.length ) {
            // debug('validation for %s.%s = %y',collectionName,field.id, field.validations)
            for ( let v of field.validations ) {
              if ( v.unique ) {
                const oldIndex = schema.indexes[ `${collectionName}_${field.id}` ]
                const newIndex = { collection: collectionName, spec: { [ field.id ]: 1 }, options: { unique: true } }
                if ( oldIndex && objectHash( oldIndex ) != objectHash( newIndex ) ) {
                  debug( 'Conflicting index specs for %s.%s  old: %y new: %y', collectionName, field.id, oldIndex, newIndex )
                }
                schema.indexes[ `${collectionName}_${field.id}` ] = newIndex
              }
            }
          }

          if ( field.type == 'Array' ) {
            schema.collections[ collectionName ].schema[ field.id ].member = { type: ContentfulType( field.items.type ) }
            if ( field.items.type == 'Link' && ( field.items.linkType == 'Entry' || field.items.linkType == 'Asset' ) ) {
              let references = []
              if ( field.items.linkType == 'Asset' ) references.push( 'assets' )
              if ( field.items.validations ) {
                for ( let v of field.items.validations ) {
                  if ( v.linkContentType ) references = [ ...references, ...v.linkContentType.map( c => changeCase.snakeCase( adapter.pluralize( c ) ) ) ]
                  if ( v.in ) {
                    schema.collections[ collectionName ].schema[ field.id ].member.options = v.in
                  }
                }
              }
              if ( references && references.length ) {
                references.forEach( reference => {
                  const oldIndex = schema.indexes[ `${collectionName}_${field.id}` ]
                  const newIndex = { collection: collectionName, spec: { [ field.id ]: 1 }, options: {} }
                  if ( oldIndex && objectHash( oldIndex ) != objectHash( newIndex ) ) {
                    debug( 'Conflicting index specs for %s.%s  old: %y new: %y', collectionName, field.id, oldIndex, newIndex )
                  }
                  schema.indexes[ `${collectionName}_${field.id}` ] = newIndex
                } )
                if ( references.length == 1 ) {
                  schema.collections[ collectionName ].schema[ field.id ].member.reference = references[ 0 ]
                  // schema.collections[collectionName].schema[field.id].member.reverse = `${schema.collections[collectionName].options.plural}_reverse`
                } else {
                  schema.collections[ collectionName ].schema[ field.id ].member.references = references
                }
              }
            }
          } else if ( field.type == 'Link' && ( field.linkType == 'Entry' || field.linkType == 'Asset' ) ) {
            let references = []
            if ( field.linkType == 'Asset' ) references.push( 'assets' )
            if ( field.validations ) {
              for ( let v of field.validations ) {
                if ( v.linkContentType ) references = [ ...references, ...v.linkContentType.map( c => adapter.pluralize( c ) ) ]
              }
            }
            if ( references && references.length ) {
              references.forEach( reference => {
                const oldIndex = schema.indexes[ `${collectionName}_${field.id}` ]
                const newIndex = { collection: collectionName, spec: { [ field.id ]: 1 }, options: {} }
                if ( oldIndex && objectHash( oldIndex ) != objectHash( newIndex ) ) {
                  debug( 'Conflicting index specs for %s.%s  old: %y new: %y', collectionName, field.id, oldIndex, newIndex )
                }
                schema.indexes[ `${collectionName}_${field.id}` ] = newIndex
              } )
              if ( references.length == 1 ) {
                schema.collections[ collectionName ].schema[ field.id ].reference = references[ 0 ]
              } else {
                schema.collections[ collectionName ].schema[ field.id ].references = references
              }
            }
          }


          schema.collections[ collectionName ].schema[ field.id ].required = field.required

          if ( schema.collections[ collectionName ].schema[ field.id ].type == 'string' &&
            !schema.collections[ collectionName ].schema[ field.id ].reference &&
            !schema.collections[ collectionName ].schema[ field.id ].references ) {
            if ( !schema.indexes[ `${collectionName}` ] ) schema.indexes[ `${collectionName}` ] = { collection: collectionName, spec: {}, options: { default_language: default_mongodb_language } }
            schema.indexes[ `${collectionName}` ].spec[ field.id ] = 'text'
          }
        } else {
          debug( 'Disabled or Ommited: %s.%s', cType.name, field.id )
        }

        if ( field && field.validations ) {
          for ( let v of field.validations ) {
            if ( v.in ) {
              schema.collections[ collectionName ].schema[ field.id ].options = v.in
            }
          }
        }

      }
    }
  } ) )
}


function unlanguage( obj ) {
  if ( typeof( obj ) == 'object' ) {
    let keys = Object.keys( obj )
    if ( keys && /*language &&*/ keys.indexOf( language || default_contentful_language ) >= 0 ) {
      return obj[ language || default_contentful_language ]
    }
    for ( let key of keys ) {
      obj[ key ] = unlanguage( obj[ key ] )
    }
  }
  return obj
}

function unsys( obj ) {
  if ( typeof( obj ) == 'object' ) {
    let keys = Object.keys( obj )
    if ( keys && keys.length == 1 && keys[ 0 ] == 'sys' && obj.sys.type == 'Link' && ( obj.sys.linkType == 'Entry' || obj.sys.linkType == 'Asset' ) && obj.sys.id ) {
      return obj.sys.id
    }
    for ( let key of keys ) {
      obj[ key ] = unsys( obj[ key ] )
    }
  }
  return obj
}

function fetchLocales( contentfulFile ) {
  if ( flags.skipLocales ) return Promise.resolve()
  return new Promise( ( resolve ) => {
    return getStream( 'locales', contentfulFile ).pipe( es.mapSync( ( data ) => {
      // debug('locales: %y',data)
      for ( let lang of data ) {
        debug( 'Language %y = %y', lang.code, lang.name )
        if ( !language || lang.default ) language = lang.code
        langauges[ lang.code ] = { code: lang.code, name: lang.name }
      }
    } ) ).on( 'end', () => {
      resolve()
    } )
  } )
}

function fetchEntries( adapter, contentfulFile ) {
  if ( flags.skipEntries ) return Promise.resolve()
  return new Promise( ( resolve ) => {
    getStream( 'entries', contentfulFile ).pipe( es.mapSync( ( data ) => {
      for ( let entry of data ) {
        const collection = adapter.pluralize( changeCase.snakeCase( entry.sys.contentType.sys.id ) )
        let set = unsys( unlanguage( entry.fields ) )

        if ( adapterOptions.refactor ) {
          for ( let refactor of adapterOptions.refactor ) {
            if ( !refactor.collection || refactor.collection == collection ) {
              if ( refactor.field && refactor.field.old && refactor.field.new ) {
                const newObj = {}
                for ( let okey in set ) {
                  newObj[ ( okey == refactor.field.old ) ? refactor.field.new : okey ] = set[ okey ]
                }
                set = newObj
              }
            }
          }
        }

        set.meta = {}
        if ( _.get( entry, 'sys.createdAt' ) ) set.meta.createdAt = new Date( entry.sys.createdAt )
        if ( _.get( entry, 'sys.updatedAt' ) ) set.meta.updatedAt = new Date( entry.sys.updatedAt )
        if ( _.get( entry, 'sys.createdBy.sys.id' ) ) set.meta.createdBy = entry.sys.createdBy.sys.id
        if ( _.get( entry, 'sys.updatedBy.sys.id' ) ) set.meta.updatedBy = entry.sys.updatedBy.sys.id
        if ( _.get( entry, 'sys.version' ) ) set.meta.version = entry.sys.version
        if ( _.get( entry, 'sys.firstPublishedAt' ) ) set.meta.firstPublishedAt = new Date( entry.sys.firstPublishedAt )
        if ( _.get( entry, 'sys.publishedAt' ) ) set.meta.publishedAt = new Date( entry.sys.publishedAt )
        if ( _.get( entry, 'sys.publishedBy.sys.id' ) ) set.meta.publishedBy = entry.sys.publishedBy.sys.id
        if ( _.get( entry, 'sys.publishedVersion' ) ) set.meta.publishedVersion = entry.sys.publishedVersion
        if ( _.get( entry, 'sys.publishedCounter' ) ) set.meta.publishedCounter = entry.sys.publishedCounter

        const _id = entry.sys.id
        if ( !report[ collection ] ) report[ collection ] = 0
        adapter.update( collection, { _id }, { $set: set }, { upsert: true } ).then( () => {
          debug( 'Entry %y: %s', collection, _id )
          report[ collection ]++
        } ).catch( e => {
          debug( 'catch %y', e )
        } )
      }
    } ) ).on( 'end', () => {
      resolve()
    } )
  } )
}


function fetchAssets( adapter, contentfulFile ) {
  if ( flags.skipAssets ) return Promise.resolve()
  return new Promise( ( resolve ) => {
    return getStream( 'assets', contentfulFile ).pipe( es.mapSync( ( data ) => {
      for ( let asset of data ) {
        const collection = 'assets'
        const set = unsys( unlanguage( asset.fields ) )
        const _id = asset.sys.id

        if ( !report[ collection ] ) report[ collection ] = 0
        adapter.update( collection, { _id }, { $set: set }, { upsert: true } ).then( () => {
          debug( 'Entry %y: %s', collection, _id )
          report[ collection ]++
        } ).catch( e => {
          debug( 'catch %y', e )
        } )
      }
    } ) ).on( 'end', () => {
      resolve()
    } )
  } )
}

function createIndexes( adapter ) {
  if ( flags.skipCreateIndexes ) return Promise.resolve()
  const promises = []
  for ( let i in schema.indexes ) {
    const { collection, spec, options } = schema.indexes[ i ]
    promises.push( adapter.createIndex( collection, spec, options ).then( () => debug( 'Created Index on %s -> %y', collection, spec, options ) ) )
  }
  return Promise.all( promises )
}

function main() {

  fetchBackup( async () => {
    const contentfulFiles = flags.dataDir && glob.sync( path.join( flags.dataDir, changeCase.snakeCase( flags.schemaName ), 'contentful' ) + '*.json', options ).filter( f => f.indexOf( 'error-log' ) < 0 )
    const contentfulFile = flags.contentfulFile || ( contentfulFiles && contentfulFiles.length ? contentfulFiles[ contentfulFiles.length - 1 ] : null )

    debug( 'Contentful %y', contentfulFile )

    if ( contentfulFile ) {
      const adapter =  await Adapter.direct( { ...adapterOptions, name: changeCase.snakeCase( flags.schemaName ) } )

      await adapter.update( 'meta', { _id: '0' }, { $set: { filename: path.basename( contentfulFile ) } }, { upsert: true } )
      const meta = await adapter.query( 'meta', { _id: '0' } )

      debug( 'meta %y', meta )

      const fetchSchemaPipe = fetchSchema( adapter, contentfulFile )
      fetchSchemaPipe.on( 'end', async () => {
        const orderedCollections = {}

        if ( adapterOptions.refactor ) {
          for ( let refactor of adapterOptions.refactor ) {
            for ( let collection in schema.collections ) {
              if ( !refactor.collection || refactor.collection == collection ) {
                if ( refactor.field && refactor.field.old && refactor.field.new ) {
                  const newObj = {}
                  for ( let okey in schema.collections[ collection ].schema ) {
                    newObj[ ( okey == refactor.field.old ) ? refactor.field.new : okey ] = schema.collections[ collection ].schema[ okey ]
                  }
                  schema.collections[ collection ].schema = newObj
                }
              }
            }
          }
        }

        /* apply the extend setting in options */
        if ( adapterOptions.extend ) {
          for ( let ckey in adapterOptions.extend ) {
            for ( let fkey in adapterOptions.extend[ ckey ].schema ) {
              if ( schema.collections[ ckey ] && schema.collections[ ckey ].schema[ fkey ] ) {
                for ( let ekey in adapterOptions.extend[ ckey ].schema[ fkey ] ) {
                  if ( !Object.prototype.hasOwnProperty.call( schema.collections[ ckey ].schema[ fkey ], ekey ) ) {
                    schema.collections[ ckey ].schema[ fkey ][ ekey ] = adapterOptions.extend[ ckey ].schema[ fkey ][ ekey ]
                  }
                }
                if ( schema.collections[ ckey ].schema[ fkey ].breakdown && schema.collections[ ckey ].schema[ fkey ].options ) {
                  schema.collections[ ckey ].breakdown = fkey
                }
              }
            }
          }
        }

        Object.keys( schema.collections ).sort().forEach( function( key ) {
          orderedCollections[ key ] = schema.collections[ key ]
        } )
        schema.collections = orderedCollections

        if ( !flags.skipSchema ) {
          if ( flags.spacesDir ) fs.ensureDirSync( flags.spacesDir )
          //${flags.spacesDir?(`${flags.spacesDir}/`):''}
          fs.writeFileSync( `${adapterOptions.schema}`, JSON.stringify( schema, null, 2 ) )
          debug( 'Schema %y has been written.', path.resolve( adapterOptions.schema ) )
        }

        if ( !flags.skipDropCollections ) {
          for ( let collection of Object.keys( schema.collections ) ) {
            const dropResult = await adapter.dropCollection( collection ).catch( ( e ) => {/* IGNORE ERRORS */ } )
            if ( dropResult ) {
              debug( 'Existing collection %y has been dropped', collection )
            }
          }
        }

        if ( !flags.skipCreateCollections ) {
          for ( let collection of Object.keys( schema.collections ) ) {
            report[ collection ] = 0
            const createResult = await adapter.createCollection( collection ).catch( ( e ) => {/* IGNORE ERRORS */ } )
            if ( createResult ) {
              debug( 'New collection %y has been created', collection )
            }
          }
        }

        fetchLocales( contentfulFile ).then( () => {
          fetchEntries( adapter, contentfulFile ).then( () => {
            fetchAssets( adapter, contentfulFile ).then( () => {
              createIndexes( adapter ).then( () => {
                adapter.stop( () => {
                  debug( 'report %y', report )
                  debug( 'done' )
                } )
              } )
            } )
          } )
        } )
      } )
    } else {
      debug( 'File not found: %s', )
    }
  } )
}

main()