infopack/lib/pipeline.ts

Information package generator

import * as fs from 'fs'
import * as path from 'path'
import { Promise } from 'bluebird'
import * as Handlebars from 'handlebars'
import { readFile } from 'fs/promises'
import _ from 'lodash'
import { argv } from 'process'

function truncateString(str:string, maxLength = 70):string {
  if (str.length > maxLength) {
      return str.slice(0, maxLength - 3) + '...';
  }
  return str;
}

Handlebars.registerHelper('ifEquals', function (arg1, arg2, options) {
  // @ts-ignore
  return (arg2.split('|').indexOf(arg1) > -1) ? options.fn(this) : options.inverse(this)
})

/**
 * The settings object can be used to pass settings down to the run function.
 * This method is particularly useful when working with generators.
 */
export interface PipelineStepSettings { }

export class PipelineStep {
  private settings: PipelineStepSettings
  // eslint-disable-next-line no-use-before-define
  public run: (executor: Executor) => Promise<any>

  constructor (runFn: (executor: Executor) => Promise<any>, settings?: PipelineStepSettings) {
    this.settings = settings || {}
    this.run = runFn
  }
}

export interface PipelineOptions {

  title?: string

  /**
   * If set this string will be suffixed to version with a dash
   */
  versionSuffix?: string

  namespace?: string

  /**
   * The base path for the pipeline. It should be a relative path from infopack folder
   * @default "./"
   */
  basePath?: string

  /**
   * Name of folder where the input file are stored
   * @default "input"
   */
  inputFolderName?: string
  /**
   * Name of folder where the output file are stored
   * @default "output"
   */
  outputFolderName?: string
  /**
   * Name of folder where the input file are stored
   * @default "cache"
   */
  cacheFolderName?: string

  indexHtmlPath?: string

  sidecarHtmlPath?: string
}

/**
 * Main class that produces output from the input via a pipeline
 */
export class Pipeline {
    /**
     * Human readable name
     */
    private title: string
    /**
     * If set this string will be suffixed to version with a dash
     */
    private versionSuffix: string | undefined
    /**
     *  Namespace for the package
     */
    private namespace: string

    /**
     * The steps which will be executed in the run method
     */
    private steps: PipelineStep[]
    /**
     * Absolute path to base path of the infopack
     */
    private basePath: string
    /**
     * Absolute path to the input folder
     */
    private inputPath: string
    /**
     * Absolute path to the output folder
     */
    private outputPath: string
    /**
     * Absolute path to the cache folder
     */
    private cachePath: string

    private indexHtmlPath: string
    private sidecarHtmlPath: string

    public indexHtml: string = ''
    public sidecarHtml: string = ''

    constructor (inputSteps: PipelineStep[], options: PipelineOptions = {}) {
      // cmd prioritized
      const tmpVersionSuffix = argv[2] || options.versionSuffix
      this.steps = inputSteps
      this.namespace = options.namespace || ''
      this.title = options.title || ''
      this.versionSuffix = argv[2] || undefined
      this.versionSuffix = tmpVersionSuffix || undefined
      this.basePath = path.resolve(options.basePath || '')
      this.inputPath = path.join(this.basePath, options.inputFolderName || 'input')
      this.outputPath = path.join(this.basePath, options.outputFolderName || 'output')
      this.cachePath = path.join(this.basePath, options.cacheFolderName || 'cache')
      this.indexHtmlPath = options.indexHtmlPath || path.join(__dirname, '..', 'templates', 'index.template.html')
      this.sidecarHtmlPath = options.sidecarHtmlPath || path.join(__dirname, '..', 'templates', 'sidecar.template.html')
    }

    public getSteps () {
      return this.steps
    }

    public getBasePath (relPath?: string) {
      return path.join(this.basePath, relPath || '')
    }

    public getInputPath (relPath?: string) {
      return path.join(this.inputPath, relPath || '')
    }

    public getOutputPath (): string {
      return this.outputPath
    }

    public getCachePath (): string {
      return this.cachePath
    }

    public getNamespace (): string {
      return this.namespace
    }

    public getTitle (): string {
      return this.title
    }

    public getVersionSuffix (): string | undefined {
      return this.versionSuffix
    }

    /**
     * Import index template
     * @param filePath Optional absolute path to template
     */
    public importIndexTemplate (filePath: string): Promise<string> {
      return readFile(filePath)
        .then(buff => buff.toString())
        .then(html => {
          this.indexHtml = html
          return html
        })
    }

    /**
     * Import sidecar template
     * @param filePath Optional absolute path to template
     */
    public importSidecarTemplate (filePath: string): Promise<string> {
      return readFile(filePath)
        .then(buff => buff.toString())
        .then(html => {
          this.sidecarHtml = html
          return html
        })
    }

    /**
     * Method to start the pipeline operation
     */
    public run = () => {
      console.log('Pipeline run started...')
      console.log('Base path: ' + this.basePath)
      console.log('Input folder: ' + this.inputPath)
      console.log('Output folder: ' + this.outputPath)
      console.log('Cache folder: ' + this.cachePath)
      Promise
        .resolve()
        .then(() => this.importIndexTemplate(this.indexHtmlPath))
        .then(() => this.importSidecarTemplate(this.sidecarHtmlPath))
        .then(() => {
          const executor = new Executor(this)
          return executor.execute()
        })
    }

    public addStep = (step: PipelineStep) => {
      this.steps.push(step)
    }
}

export interface InfopackContentInput {
  /**
   * Path relative to output folder
   */
  path: string
  data: Buffer
  title: string
  description: string
  labels?: Object
  origin?: [string]
}

/**
 * InfopackContent structures the files to be written list
 */
export interface InfopackContent {
  $schema: string
  title: string
  description: string
  /**
   * Path relative to output folder
   */
  path: string
  dirname: string
  filename: string
  extname: string
  data: Buffer
  labels?: Object
  origin?: [string]
}

export interface ExecutorMeta {
  $schema: string
  namespace?: string
  name?: string
  title?: string
  description?: string
  version?: string
  /**
   * Timestamp mainly used in template generator
   */
  packagedAt?: string
}
export class Executor {
  pipeline: Pipeline
  finished: boolean = false
  public currentStep: number = 0
  writeQueue: InfopackContent[] = []
  private meta: ExecutorMeta = { $schema: 'https://schemas.infopack.io/infopack-index.2.schema.json' }
  private files: any[] = []
  private test: any = {}
  constructor (pipeline: Pipeline) {
    this.pipeline = pipeline
  }

  /**
   * This method will add infopackContent to the files queue.
   * @param infopackContent
   */
  public toOutput (infopackContentInput: InfopackContentInput) {
    const infopackContent: InfopackContent = {
      $schema: 'https://schemas.infopack.io/infopack-meta.2.schema.json',
      path: infopackContentInput.path,
      dirname: path.dirname(infopackContentInput.path),
      filename: path.basename(infopackContentInput.path),
      extname: path.extname(infopackContentInput.path),
      data: infopackContentInput.data,
      title: infopackContentInput.title,
      description: infopackContentInput.description
    }

    if (infopackContentInput.labels) {
      infopackContent.labels = infopackContentInput.labels
    }
    if (infopackContentInput.origin) {
      infopackContent.origin = infopackContentInput.origin
    }
    this.writeQueue.push(infopackContent)
  }

  private rmdir (path: string) {
    return new Promise((resolve, reject) => {
      if (!fs.existsSync(path)) resolve()
      console.log('Cleaning ' + path)
      fs.rm(path, { recursive: true, force: true }, (err) => {
        if (err) reject(err)
        resolve()
      })
    })
  }

  private mkdir (path: string) {
    return new Promise((resolve, reject) => {
      fs.mkdir(path, { recursive: true }, (err) => {
        if (err) reject(err)
        return resolve()
      })
    })
  }

  /**
   * Write buffered files to disk
   * @param target Specifies output target. Provide cache to write to cache
   * @returns Promise<any>
   */
  private writeOutput (target?: string) {
    const outputPath = (target === 'cache') ? path.join(this.getCachePath(), this.currentStep + '') : this.getOutputPath()
    const indexPath = outputPath + '/index.html'
    console.log('  Writing output to: ' + outputPath)
    console.log('  Index file path: ' + indexPath)
    return Promise
      .resolve()
      .then(() => Promise.mapSeries(this.writeQueue, (content) => {
        const absFilePath = path.join(outputPath, content.path)
        const absDirPath = path.join(outputPath, content.path.replace(content.filename, ''))

        console.log('  Writing to folder: ' + absDirPath)
        console.log('  Writing file "' + content.title + '" to path ' + absFilePath)

        return this
          .mkdir(path.dirname(absFilePath))
          .then(made => {
            // if (made) console.log('Created folder: ' + made)
            const { data, ...fileMeta } = content
            // write file to disk
            fs.writeFileSync(absFilePath, data)
            // create sidecar json file
            const { $schema, ...metaWithout$schema } = this.meta
            const sidecarData: any = Object.assign({ meta: metaWithout$schema }, fileMeta)
            fs.writeFileSync(absFilePath + '.meta.json', JSON.stringify(sidecarData, null, 2))
            // create sidecar html file
            const template = Handlebars.compile(this.pipeline.sidecarHtml)
            /**
             * Handle https://html-validate.org/rules/long-title.html
             */
            let cappedTitle = truncateString(sidecarData.title + " - " + sidecarData.meta.title)
            fs.writeFileSync(absFilePath + '.meta.html', template({ ...sidecarData, cappedTitle }))

            // push file to meta file index
            if (target !== 'cache') {
              const temp: any = Object.assign({
                metaHtmlPath: `${fileMeta.path}.meta.html`,
                metaJsonPath: `${fileMeta.path}.meta.json`
              }, fileMeta)
              this.files.push(temp)
            }

            // console.log('  write finished')
            return true
          })
      }))
      .then(() => fs.writeFileSync(outputPath + '/infopack.json', JSON.stringify({ $schema: 'https://schemas.infopack.io/infopack-infopack.1.schema.json', framework_version: '2.0.0', implementation: 'node' }, null, 2)))
  }

  public getBasePath (relPath?: string) {
    return this.pipeline.getBasePath(relPath)
  }

  public getInputPath (relPath?: string) {
    return this.pipeline.getInputPath(relPath)
  }

  public getOutputPath (): string {
    return this.pipeline.getOutputPath()
  }

  public getCachePath (): string {
    return this.pipeline.getCachePath()
  }

  public createMeta (): void {
    const packageInfo = JSON.parse(fs.readFileSync(this.getBasePath('package.json')).toString())
    this.meta.namespace = this.pipeline.getNamespace()
    this.meta.title = this.pipeline.getTitle()
    this.meta.name = packageInfo.name
    this.meta.description = packageInfo.description
    this.meta.version = this.pipeline.getVersionSuffix() ? `${packageInfo.version}-${this.pipeline.getVersionSuffix()}` : packageInfo.version
    this.meta.packagedAt = new Date().toISOString().slice(0, 10)
  }

  public execute = () => {
    console.log('Executing')
    return Promise
      .resolve()
      .then(() => this.rmdir(this.getOutputPath()))
      .then(() => this.rmdir(this.getCachePath()))
      .then(() => this.mkdir(this.getOutputPath()))
      .then(() => this.createMeta())
      .then(() => Promise.mapSeries(this.pipeline.getSteps(), (step: PipelineStep, i) => {
        console.log('== running step ' + i + ' ==')
        return step.run(this)
          .then(() => this.writeOutput('cache'))
          .then(() => this.currentStep++)
      }))
      .then(() => this.writeOutput())
      .then(() => {
        console.log('=== Steps complete ===')
        console.log('  Writing index')
        this.files = _.sortBy(this.files, 'path')
        const tempFiles = _.map(this.files, file => {
          const { $schema, ...fileWithout$schema } = file
          return fileWithout$schema
        })
        const templateData: any = Object.assign({ files: tempFiles }, this.meta)
        // write .json file
        fs.writeFileSync(this.getOutputPath() + '/index.json', JSON.stringify(templateData, null, 2))
        // write .html file
        const template = Handlebars.compile(this.pipeline.indexHtml)
        const output = template(templateData)
        fs.writeFileSync(this.getOutputPath() + '/index.html', output)
      })
  }
}