yonderbox-contentful-mongodb-abstract-schema/cli.js

Imports Contentful space to MongoDB and generates an abstract schema

#!/usr/bin/env node
const pkg = require('./package.json')

const yves = require('yves')
yves.debugger('Y', { stream: null, sortKeys: true, hideFunctions: true, singleLineMax: 0, obfuscates: [/key/i, /token/i] })

const debug = yves.debugger(`${pkg.name.replace(/-/g,':')}`)

const changeCase = require('change-case')
const path = require('path')

const inflection = require('inflection' )

const glob = require('glob')
const options = {}

const args =require('args')
const spawn = require('child_process').spawn

const _ =require('lodash')

args
  .option('schema-name', 'Path to schema file','abstract')
  .option('data-dir', 'The directory which holds the contentful backups','data')
  .option('spaces-dir', 'The directory which holds the space schemas (abstract & GraphQL)','spaces')
  .option('contentful-file', 'Specific Contentful file to use as source')
  .option('schema-path', 'Path to schema file')
  .option('skip-backup','Skip fetching Contentful backup',false)
  .option('skip-entries','Skip importing Contentful entries',false)
  .option('skip-assets','Skip importing Contentful assets',false)
  .option('skip-locales','Skip importing Contentful locales',false)
  .option('skip-drop-collections','Skip drop MongoDB collections',false)
  .option('skip-create-collections','Skip create MongoDB collections',false)
  .option('skip-create-indexes','Skip create MongoDB indexes',false)
  .option('verbose', 'Verbose output')
  .option('management-token','Contentful management API token')
  .option('space-id', 'ID of Space with source data')
  .option('environment-id', 'ID of Environment with source data','master')
  .option('mongodb-url', 'ID of Environment with source data')
  .option('config-filename', 'Config filename can optionally point to a JSON file, containing camelCased variables like the options here')


let flags = args.parse(process.argv)

if (flags.configFilename) {
  try {
    const config = require(path.resolve(flags.configFilename))
    const setup = {}
    if (config) {
      if (config.url) setup.mongodbUrl = config.url
      if (config.root) setup.schemaName =  config.root
      if (config.schema) setup.schemaPath = config.schema
      if (config.writeGraphQLfile) setup.writeGraphQLfile = config.writeGraphQLfile
    }
    if (Object.keys(setup).length) {
      debug('setup %y',setup)
      debug('flags %y',flags)
      flags = Object.assign({},setup,flags)
    }
  } catch(e) {
    debug('error %y',e)
  }
}

if (!flags.schemaPath) flags.schemaPath=`${flags.schemaName}.schema.json`
debug('flags %y',flags)

const JSONStream = require('JSONStream')
const fs = require('fs-extra')
const es = require('event-stream')

const GraphqlMongodbAdapter = require('yonderbox-graphql-mongodb-adapter')


const adapterOptions = {
  url: flags.mongodbUrl,
  options: {
    keepAlive: 1,
    connectTimeoutMS: 30000,
  },
  db: changeCase.snakeCase(flags.schemaName),
  schema: flags.schemaPath,
  root: changeCase.pascalCase(flags.schemaName),
  limit:0,
  writeGraphQLfile: flags.writeGraphQLfile,
}

const default_contentful_language = 'nl-NL'
const default_mongodb_language = 'none'

let language = null
const langauges = {}
const objectHash = require('object-hash')

// # contentTypes
// # editorInterfaces
// # entries
// # assets
// # locales

function fetchBackup(callback) {
  if (!flags.skipBackup && flags.managementToken && flags.spaceId) {
    const dirPath = `${flags.dataDir}/${changeCase.snakeCase(flags.schemaName)}`
    fs.ensureDirSync(dirPath)
    const cmd = spawn('contentful', [
      'space',
      'export',
      '--management-Token',
      flags.managementToken,
      '--space-id',
      flags.spaceId,
      '--environment-id',
      flags.environmentId,
      '--export-dir',
      dirPath,
      '--skip-webhooks',
      '--skip-roles',
      //      '--include-drafts',
      //      '--include-archived',
    ])
    cmd.stdout.on('data', function (data) {
      if (flags.verbose) {
        console.log('stdout: ' + data.toString())
      } else {
        process.stdout.write(data.toString())
      }
    })

    cmd.stderr.on('data', function (data) {
      if (flags.verbose) {
        console.log('stderr: ' + data.toString())
      }
    })

    cmd.on('exit', function (code) {
      console.log('child process exited with code ' + code.toString())

      callback && callback()
    })
  } else {
    callback && callback()
  }
}


function getStream(rootName,contentfulFile) {
  if (contentfulFile) {
    const stream = fs.createReadStream(contentfulFile, {encoding: 'utf8'})
    const parser = JSONStream.parse(rootName/*['contentTypes', {recurse: true}, ['sys','fields']]*/)
    return stream.pipe(parser)
  }
}

const schema = {
  collections: {
    assets: {
      type: 'collection',
      name: 'assets',
      schema: {
        _id: { type: 'string', required: true },
        title: { type: 'string', required: true },
        description: { type: 'string', required: true },
        file: {
          type: 'object',
          required: true,
          schema: {
            url: { type: 'string', required: true },
            details: {
              type: 'object',
              required: false,
              schema: {
                size: { type: 'integer', required: false },
                image: {
                  type: 'object',
                  required: false,
                  schema: {
                    width: { type: 'integer', required: false },
                    height: { type: 'integer', required: false },
                  },
                },
                filename: { type: 'string', required: false },
                contentType: { type: 'string', required: false },
              },
            },
            title: { type: 'string', required: false },
          },
        },
      },
    },
  },
  indexes: {}
}

const report = {}

function fetchSchema(mongodb,contentfulFile) {

  function ContentfulType(type) {
    let result='string'
    switch (type) {
    case 'Symbol': // fall-through
    case 'Link': // fall-through
    case 'Text':
      result='string'
      break
    case 'Integer':
      result='integer'
      break
    case 'Number':
      result='float'
      break
    case 'Date':
      result='date'
      break
    case 'Boolean':
      result='boolean'
      break
    case 'Array':
      result='array'
      break
    case 'Location': // fall-through
    case 'Object':
      result='object'
      break
    }
    return result
  }

  return getStream('contentTypes',contentfulFile).pipe(es.mapSync(function (data) {
    for (let cType of data) {
      const collectionName = inflection.pluralize(changeCase.snakeCase(cType.name))
      schema.collections[collectionName] = {type:'collection', name: collectionName}

      // if (cType.displayField) schema.collections[collectionName].displayField = cType.displayField
      if (cType.description) schema.collections[collectionName].description = cType.description

      schema.collections[collectionName].schema={}
      schema.collections[collectionName].schema['_id'] = {type:'string',required:true}

      schema.collections[collectionName].schema['meta'] = {type:'object',required:false,schema:{
        createdAt:{type:'date',required:false},
        updatedAt:{type:'date',required:false},
        createdBy:{type:'string',required:false},
        updatedBy:{type:'string',required:false},
        version:{type:'integer',required:false},
        firstPublishedAt:{type:'date',required:false},
        publishedAt:{type:'date',required:false},
        publishedBy:{type:'string',required:false},
        publishedVersion:{type:'integer',required:false},
        publishedCounter:{type:'integer',required:false},
      }}

      for (let field of cType.fields) {
        if (!field.disabled && !field.ommited) {
          schema.collections[collectionName].schema[field.id] = {}
          const type = ContentfulType(field.type)
          schema.collections[collectionName].schema[field.id].type = type
          if (field.validations && field.validations.length) {
            // debug('validation for %s.%s = %y',collectionName,field.id, field.validations)
            for (let v of field.validations) {
              if (v.unique) {
                const oldIndex = schema.indexes[`${collectionName}_${field.id}`]
                const newIndex = {collection:collectionName,spec:{[field.id]:1}, options: { unique:true } }
                if (oldIndex && objectHash(oldIndex)!=objectHash(newIndex)) {
                  debug('Conflicting index specs for %s.%s  old: %y new: %y',collectionName,field.id,oldIndex,newIndex)
                }
                schema.indexes[`${collectionName}_${field.id}`]=newIndex
              }
            }
          }

          if (field.type == 'Array') {
            schema.collections[collectionName].schema[field.id].member = {type:ContentfulType(field.items.type)}
            if (field.items.type == 'Link' && (field.items.linkType == 'Entry' || field.items.linkType == 'Asset')) {
              let references=[]
              if (field.items.linkType=='Asset') references.push('assets')
              if (field.items.validations) {
                for (let v of field.items.validations) {
                  if (v.linkContentType) references = [...references, ...v.linkContentType.map(c => changeCase.snakeCase(inflection.pluralize(c)))]
                }
              }
              if (references && references.length) {
                references.forEach(reference => {
                  const oldIndex = schema.indexes[`${collectionName}_${field.id}`]
                  const newIndex = {collection:collectionName, spec:{[field.id]:1}, options:{}}
                  if (oldIndex && objectHash(oldIndex)!=objectHash(newIndex)) {
                    debug('Conflicting index specs for %s.%s  old: %y new: %y',collectionName,field.id,oldIndex,newIndex)
                  }
                  schema.indexes[`${collectionName}_${field.id}`]=newIndex
                })
                if (references.length == 1) {
                  schema.collections[collectionName].schema[field.id].member.reference = references[0]
                  // schema.collections[collectionName].schema[field.id].member.reverse = `${schema.collections[collectionName].options.plural}_reverse`
                } else {
                  schema.collections[collectionName].schema[field.id].member.references = references
                }
              }
            }
          } else if (field.type == 'Link' && (field.linkType=='Entry' || field.linkType=='Asset')) {
            let references=[]
            if (field.linkType=='Asset') references.push('assets')
            if (field.validations) {
              for (let v of field.validations) {
                if (v.linkContentType) references = [...references, ...v.linkContentType.map(c => inflection.pluralize(c))]
              }
            }
            if (references && references.length) {
              references.forEach(reference => {
                const oldIndex = schema.indexes[`${collectionName}_${field.id}`]
                const newIndex = {collection:collectionName, spec:{[field.id]:1}, options:{}}
                if (oldIndex && objectHash(oldIndex)!=objectHash(newIndex)) {
                  debug('Conflicting index specs for %s.%s  old: %y new: %y',collectionName,field.id,oldIndex,newIndex)
                }
                schema.indexes[`${collectionName}_${field.id}`]=newIndex
              })
              if (references.length == 1) {
                schema.collections[collectionName].schema[field.id].reference = references[0]
              } else {
                schema.collections[collectionName].schema[field.id].references = references
              }
            }
          }


          schema.collections[collectionName].schema[field.id].required = field.required

          if (schema.collections[collectionName].schema[field.id].type == 'string' &&
            !schema.collections[collectionName].schema[field.id].reference &&
            !schema.collections[collectionName].schema[field.id].references) {
            if (!schema.indexes[`${collectionName}`]) schema.indexes[`${collectionName}`]={collection:collectionName,spec:{},options:{ default_language: default_mongodb_language }}
            schema.indexes[`${collectionName}`].spec[field.id]='text'
          }
        } else {
          debug('Disabled or Ommited: %s.%s',cType.name,field.id)
        }
      }
    }
  }))
}


function unlanguage(obj) {
  if (typeof(obj) == 'object') {
    let keys = Object.keys(obj)
    if (keys && /*language &&*/ keys.indexOf(language || default_contentful_language)>=0) {
      return obj[language || default_contentful_language]
    }
    for (let key of keys) {
      obj[key] = unlanguage(obj[key])
    }
  }
  return obj
}

function unsys(obj) {
  if (typeof(obj) == 'object') {
    let keys = Object.keys(obj)
    if (keys && keys.length==1 && keys[0]=='sys' && obj.sys.type=='Link' && (obj.sys.linkType=='Entry' || obj.sys.linkType=='Asset' ) && obj.sys.id) {
      return obj.sys.id
    }
    for (let key of keys) {
      obj[key] = unsys(obj[key])
    }
  }
  return obj
}

function fetchLocales(contentfulFile) {
  if (flags.skipLocales) return Promise.resolve()
  return new Promise( (resolve) => {
    return getStream('locales',contentfulFile).pipe(es.mapSync((data) => {
      // debug('locales: %y',data)
      for (let lang of data) {
        debug('Language %y = %y',lang.code,lang.name)
        if (!language || lang.default) language = lang.code
        langauges[lang.code]={code:lang.code,name:lang.name}
      }
    })).on('end', () => {
      resolve()
    })
  })
}

function fetchEntries(mongodb,contentfulFile) {
  if (flags.skipEntries) return Promise.resolve()
  return new Promise( (resolve) => {
    getStream('entries',contentfulFile).pipe(es.mapSync((data) => {
      for (let entry of data) {
        const collection = inflection.pluralize(changeCase.snakeCase(entry.sys.contentType.sys.id))
        const set = unsys(unlanguage(entry.fields))
        set.meta = {}
        if (_.get(entry,'sys.createdAt')) set.meta.createdAt = new Date(entry.sys.createdAt)
        if (_.get(entry,'sys.updatedAt')) set.meta.updatedAt = new Date(entry.sys.updatedAt)
        if (_.get(entry,'sys.createdBy.sys.id')) set.meta.createdBy = entry.sys.createdBy.sys.id
        if (_.get(entry,'sys.updatedBy.sys.id')) set.meta.updatedBy = entry.sys.updatedBy.sys.id
        if (_.get(entry,'sys.version')) set.meta.version = entry.sys.version
        if (_.get(entry,'sys.firstPublishedAt')) set.meta.firstPublishedAt = new Date(entry.sys.firstPublishedAt)
        if (_.get(entry,'sys.publishedAt')) set.meta.publishedAt = new Date(entry.sys.publishedAt)
        if (_.get(entry,'sys.publishedBy.sys.id')) set.meta.publishedBy = entry.sys.publishedBy.sys.id
        if (_.get(entry,'sys.publishedVersion')) set.meta.publishedVersion = entry.sys.publishedVersion
        if (_.get(entry,'sys.publishedCounter')) set.meta.publishedCounter = entry.sys.publishedCounter

        const _id = entry.sys.id
        if (!report[collection]) report[collection]=0
        mongodb.update(collection,{_id},{$set:set},{ upsert: true }).then( () => {
          debug('Entry %y: %s',collection,_id)
          report[collection]++
        }).catch( e => {
          debug('catch %y',e)
        })
      }
    })).on('end', () => {
      resolve()
    })
  })
}


function fetchAssets(mongodb,contentfulFile) {
  if (flags.skipAssets) return Promise.resolve()
  return new Promise( (resolve) => {
    return getStream('assets',contentfulFile).pipe(es.mapSync((data) => {
      for (let asset of data) {
        const collection = 'assets'
        const set = unsys(unlanguage(asset.fields))
        const _id = asset.sys.id

        if (!report[collection]) report[collection]=0
        mongodb.update(collection,{_id},{$set:set},{ upsert: true }).then( () => {
          debug('Entry %y: %s',collection,_id)
          report[collection]++
        }).catch( e => {
          debug('catch %y',e)
        })
      }
    })).on('end', () => {
      resolve()
    })
  })
}

function createIndexes(mongodb) {
  if (flags.skipCreateIndexes) return Promise.resolve()
  const promises = []
  for (let i in schema.indexes) {
    const {collection,spec, options} = schema.indexes[i]
    promises.push(mongodb.createIndex(collection,spec,options).then( () => debug('Created Index on %s -> %y',collection,spec, options) ))
  }
  return Promise.all(promises)
}

function main() {

  fetchBackup(async () => {
    const contentfulFiles = flags.dataDir && glob.sync(path.join(flags.dataDir,changeCase.snakeCase(flags.schemaName),'contentful')+'*.json', options).filter(f => f.indexOf('error-log') < 0)
    const contentfulFile = flags.contentfulFile || (contentfulFiles && contentfulFiles.length ? contentfulFiles[contentfulFiles.length-1] : null)

    debug('Contentful %y',contentfulFile)

    if (contentfulFile) {
      const mongodb =  await GraphqlMongodbAdapter.direct(adapterOptions,changeCase.snakeCase(flags.schemaName))

      await mongodb.update('meta', {_id:'0'},{$set:{filename:path.basename(contentfulFile)}},{ upsert: true })
      const meta = await mongodb.query('meta',{_id:'0'})

      debug('meta %y',meta)

      const fetchSchemaPipe = fetchSchema(mongodb,contentfulFile)
      fetchSchemaPipe.on('end', async () => {
        const orderedCollections = {}
        Object.keys(schema.collections).sort().forEach(function(key) {
          orderedCollections[key] = schema.collections[key]
        })
        schema.collections = orderedCollections

        if (flags.spacesDir) fs.ensureDirSync(flags.spacesDir)
        //${flags.spacesDir?(`${flags.spacesDir}/`):''}
        fs.writeFileSync(`${adapterOptions.schema}`, JSON.stringify(schema, null, 2))
        debug('Schema %y has been written.',path.resolve(adapterOptions.schema))

        if (!flags.skipDropCollections) {
          for (let collection of Object.keys(schema.collections)) {
            const dropResult = await mongodb.dropCollection(collection).catch((e) => {/* IGNORE ERRORS */ })
            if (dropResult) {
              debug('Existing collection %y has been dropped',collection)
            }
          }
        }

        if (!flags.skipCreateCollections) {
          for (let collection of Object.keys(schema.collections)) {
            report[collection] = 0
            const createResult = await mongodb.createCollection(collection).catch((e) => {/* IGNORE ERRORS */ })
            if (createResult) {
              debug('New collection %y has been created',collection)
            }
          }
        }

        fetchLocales(contentfulFile).then( () => {
          fetchEntries(mongodb,contentfulFile).then( () => {
            fetchAssets(mongodb,contentfulFile).then( () => {
              createIndexes(mongodb).then( () => {
                mongodb.stop(() => {
                  debug('report %y',report)
                  debug('done')
                })
              })
            })
          })
        })
      })
    } else {
      debug('File not found: %s',)
    }
  })
}

main()