@kumologica/builder/src/deploy/kumohub.js

Kumologica build and deploy module

const fs = require('fs-extra');
const path = require('path');
const AdmZip = require('adm-zip');
const util = require('util');
const exec = util.promisify(require('child_process').exec);
const codegen = require('../build/codegen');
const request = require('request');
const requestAsync = util.promisify(request);
 
/**
 * Class managing login and deployment to kumohub.
 */
class KumohubDeployer {
  constructor(terminal) {
    this.term = terminal;
    this.deploying = false;
  }

  log(text, calog = true, source = 'builder') {
    //console.log(text);
    var t = text;

    if (calog) {
      t = `${source}: ${text}`;
    }
  
    if (this.term) {
      this.term.emit('terminal-output', t);
    } else {
      console.log(text);
    }
  }
  sanitizeLambdaName(name) {
    return name
      .replace('.json', '')
      .trim()
      .replace(/^@/, '')
      .replace(/[^a-zA-Z0-9-]/g, '-')
      .substr(0, 140);
  }

  prepare(params) {
    const deployDir = path.join(params["project-directory"], '/deploy');
    const flowName = params["flow-file-name"].replace('.json', '');

    // setup deployment directory
    fs.ensureDirSync(deployDir);
    fs.emptyDirSync(deployDir);
    fs.ensureDirSync(deployDir + '/node_modules');

    // copy project files
    codegen.generateProjectCode(
      deployDir,
      params["flow-file-name"],
      true
    );

    this.processPackageJson(params, deployDir);

    const zipFileName = `${flowName}/lambda${Date.now()}.zip`;
    const functionName = this.sanitizeLambdaName(params["service-name"] || flowName);
    
    const lambdaName = `${params.account}-${params.workspace}-${params.stage}-${functionName}`;
    const stackName = lambdaName;
    const roleName = lambdaName;
            
    return {
      deployDir,
      flowName,
      zipFileName,
      functionName,
      stackName,
      roleName,
      lambdaName 
    };
  }

  processPackageJson(params, deployDir) {
    
    const pckg = this.loadJsonFile(path.join(params["project-directory"], 'package.json'));

    if (pckg.dependencies["@kumologica/runtime"] || pckg.dependencies["aws-sdk"]) {
      
      delete pckg.dependencies["@kumologica/runtime"];
      delete pckg.dependencies["aws-sdk"];

      this.createFile(deployDir, 'package.json', JSON.stringify(pckg));
    
    } else {
      fs.copySync(
        path.join(params["project-directory"], 'package.json'),
        path.join(deployDir, 'package.json')
      );
    }

    if (pckg.files) {
  
      pckg.files.forEach(f => {
        
        if (!["node_modules/**/*", "lambda.js" ].includes(f)) {
          fs.copySync(path.join(params["project-directory"], f), path.join(deployDir, f));
        }
      });
    }
  }

  /*
   * Main function orchestrating build and deployment of lambda to aws.
   */
  async deploy(params) {

    params["project-directory"] = params["project-directory"] || process.cwd();
    //console.log(`kumohub deploy params: ${JSON.stringify(params)}`);
    
    const deploymentStart = Date.now();
    this.deploying = true;

    this.log(`Deployment to ${params.account}.kumohub.io started:`);
    if (!params["flow-file-name"]) {
      params["flow-file-name"] = codegen.findFlowFile(params["project-directory"]);
    }
    this.log(`  subscription:      ${params.account}`, false);
    this.log(`  username:          ${params.username}`, false);
    this.log(`  workspace:         ${params.workspace}`, false);
    this.log(`  stage:             ${params.stage}`, false);
    this.log(`  policy:            ${params.policy}`, false);
    this.log(`  flow file name:    ${params["flow-file-name"]}`, false);
    this.log(`  working directory: ${params["project-directory"]}`, false);
    this.log(`  environment:       ${params["environment"]}`, false);
    this.log(`  description:       ${params["description"]}`, false);
    this.log(`  triggers:          ${params["triggers"]}`, false);
    this.log('', false);

    const settings = this.prepare(params);

    const nodes = this.loadJsonFile(
      path.join(params["project-directory"], params["flow-file-name"])
    );
    
    let deploymentResponse;
    try {
      await this.buildLambda(settings);

      this.log('Dispatching deployment to kumohub.io');

      await this.login(params.username, params.password, params.account);
      
      deploymentResponse = await this.executeDeployment(path.join(settings.deployDir, settings.zipFileName), params, settings);
      
      this.log(`  Deployment has been dispatched, waiting for deployment status`);

    } catch (error) {
      this.log(`Deployment dispatch failed.`);
      this.log(`  ${error}`);
      this.deploying = false;
      return;
            
    } finally {
      this.deploying = false;
    }
/*
Invoke Error  {"errorType":"ValidationError",
"errorMessage":"Stack:arn:aws:cloudformation:ap-southeast-2:174450237637:stack/demo-nonprod-test-cron-example-flow/60de20c0-6abf-11eb-a2ac-068e268722ac 
is in CREATE_IN_PROGRESS state and can not be updated.",
"code":"ValidationError","message":"Stack:arn:aws:cloudformation:ap-southeast-2:174450237637:stack/demo-nonprod-test-cron-example-flow/60de20c0-6abf-11eb-a2ac-068e268722ac 
is in CREATE_IN_PROGRESS state and can not be updated.",
"time":"2021-02-09T10:14:25.545Z","requestId":"8b526d94-3562-4201-a922-573080096932",
"statusCode":400,"retryable":false,"retryDelay":60.33048478021745,
"stack":["ValidationError: Stack:arn:aws:cloudformation:ap-southeast-2:174450237637:stack/demo-nonprod-test-cron-example-flow/60de20c0-6abf-11eb-a2ac-068e268722ac is in CREATE_IN_PROGRESS state and can not be updated.","    at Request.extractError (/var/runtime/node_modules/aws-sdk/lib/protocol/query.js:50:29)","    at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:106:20)","    at Request.emit (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:78:10)","    at Request.emit (/var/runtime/node_modules/aws-sdk/lib/request.js:688:14)","    at Request.transition (/var/runtime/node_modules/aws-sdk/lib/request.js:22:10)","    at AcceptorStateMachine.runTo (/var/runtime/node_modules/aws-sdk/lib/state_machine.js:14:12)","    at /var/runtime/node_modules/aws-sdk/lib/state_machine.js:26:10","    at Request.<anonymous> (/var/runtime/node_modules/aws-sdk/lib/request.js:38:9)","    at Request.<anonymous> (/var/runtime/node_modules/aws-sdk/lib/request.js:690:12)","    at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:116:18)"]}

*/
    setTimeout(
      this.pollDeploymentLogs.bind(this), 4000,
        params.account, 
        params.workspace, 
        params.stage, 
        settings.functionName, 
        deploymentStart,
        deploymentResponse);

    let response = {};
    
    return response;
  }

  printSignature(workspace, stage, s) {

    this.log('', true, 'Kumohub');
    if (s && s.api && s.api.length > 0) {
      this.log('Available api endpoints:', true, 'Kumohub');
      s.api.forEach(a => this.log(`  ${('    ' + a.verb.toUpperCase()).slice(-6)}  ${a.url}`, true, 'Kumohub'), this);
      this.log('', true, 'Kumohub');
    }

    if (s && s.cron && s.cron.length > 0) {
      this.log('Provisioned cron jobs:', true, 'Kumohub');
      s.cron.forEach(a => this.log(`  '${a.expression}'`, true, 'Kumohub'), this); 
      this.log('', true, 'Kumohub');
    }

    if (s && s.osStream && s.osStream.length > 0) {
      this.log('Attached to object store streams:', true, 'Kumohub');
      s.osStream.forEach(a => this.log(`  ${('    ' + a.operation.toUpperCase()).slice(-6)}  ${workspace}/${stage}/${a.objectStore}`, true, 'Kumohub'), this);
      this.log('', true, 'Kumohub');
    }
  }

  async pollDeploymentLogs(subscription, workspace, stage, service, startTime, signature) {

    const logs = await this.deploymentLogs(subscription, workspace, stage, service, startTime);
    let st = startTime;
    
    if (logs && logs.length > 0) {
      logs.forEach(l => this.log(l.message, true, 'Kumohub'), this);
      st = Date.now();
    }
    
    let serviceDetails = await this.getService(subscription, workspace, stage, service);

    if (serviceDetails && serviceDetails.status == 'Deploying') {
      setTimeout(
        this.pollDeploymentLogs.bind(this), 4000, subscription, workspace, stage, service, st, signature);

    } else if (serviceDetails && serviceDetails.status == 'Running') {
      
      this.deploying = false;
      this.printSignature(workspace, stage, signature);
      this.log(`Service started.`, true,);
    
    } else {
      this.deploying = false;
      this.log(`Service start failed, status: ${serviceDetails.status}.`);
    }
  }

  isDeploying() {
    return this.deploying;
  }

  /*
   * Builds lambda and zips all sources.
   */
  async buildLambda(settings) {
    this.log('Building lambda...');
    console.log(`cwd: ${process.cwd()}`);
    console.log(`settings.deployDir: ${settings.deployDir}`);
    
    const npmCmd = `npm install --production --prefix "${settings.deployDir}"`;
    const response = await exec(npmCmd, { cwd: settings.deployDir });

    if (response) {
      this.log('', false);
      const lines = response.stdout.split('\n');
      for (var i = 0; i < lines.length; i++) {
        if (lines[i]) {
          this.log(lines[i], false);
        }
      }
      this.log('', false);
    }

    this.log('Excluding optional dependencies...');
    // remove this rimraf, its much better to sanitize package.json 
    // Remove aws-sdk library as it is provided by aws nodejs runtime.
    // Reducing the resulting lambda zip file by more than 70%
    //rimraf.sync(path.join(settings.deployDir, 'node_modules', '.bin', 'kumologica-scripts'));
    
    this.log('Zipping lambda...');
    const zip = new AdmZip();
    zip.addLocalFolder(settings.deployDir);
    zip.writeZip(path.join(settings.deployDir, settings.zipFileName));
  }


  loadJsonFile(flowFileName) {
    return JSON.parse(fs.readFileSync(flowFileName));
  }

  createFile(baseDir, fileName, content) {
    fs.outputFileSync(path.join(baseDir, fileName), content, 'utf-8');
  }

  getLoginUrl(account) {
    return `https://admin.${account}.kumohub.io/prod/security/login`;
  }
  getDeploymentLogsUrl(account) {
    return `https://admin.${account}.kumohub.io/prod/logging/logs`;
  }
  getRefreshUrl(account) {
    return `https://admin.${account}.kumohub.io/prod/security/tokenrefresh`;
  }
  getDeployUrl(account) {
    return `https://admin.${account}.kumohub.io/prod/deployer-service/deploy`;
  }
  getServiceUrl(tenantAccount, account, stage, service) {
    return `https://admin.${tenantAccount}.kumohub.io/prod/services-service/${account}/${stage}/${service}`;
  }

  async executeDeployment(zipFile, params, settings) {
   
    // note, environment variable has to be stringified, form data do not
    // support objects
    let data = {
      file: fs.createReadStream(zipFile),
      account: params.workspace,
      stage: params.stage,
      policy: params.policy,
      serviceName: settings.functionName
    };
    if (params.timeout) {
       data.timeout = params.timeout;
    }
    if (params.memory) {
      data.memory = params.memory;
    }
    if (params.description) {
      data.description = params.description;
    }
    if (params.environment && params.environment.length > 0) {
      data.environment = params.environment;
    }
    if (params.triggers) {
      data.triggers = params.triggers;
    }

    try {
      const { statusCode, body } = 
        await requestAsync({
          url: this.getDeployUrl(params.account),
          headers: {'Authorization': `Bearer ${this.tokens.AuthenticationResult.IdToken}`},
          method: 'POST',
          json: true,
          formData: data
      });
    
      if (statusCode === 200) {
        console.log(`body ${JSON.stringify(body)}`);
        return body;
      
      } else {
        throw new Error(`Received an error from Kumohub: ${statusCode} ${JSON.stringify(body)}`);
      }
    } catch (err) {
      console.log(err);
      throw new Error(`executeDeployment: Unexpected error: ${err}`);
    }
  }

  async login(username, password, subscription) {

    try {
      const url = this.getLoginUrl(subscription);

      const { statusCode, body } = await requestAsync({
        uri: url,
        method: 'POST',
        json: true,
        body: {
          username: username,
          password: password
      }
      });

      if (statusCode === 200) {
        this.log(`Successfully logged in to ${subscription}.kumohub.io`);
        
        this.tokens = body;
 
        return body;
      
      } else {
        throw new Error(`Received an error from Kumohub: ${statusCode} ${JSON.stringify(body)}`);
      }
    } catch (err) {
      if (err && err.errno && err.errno == "ENOTFOUND" && err.hostname) {
        throw new Error(`Unable to connect to ${err.hostname}, please check your network or your profile's subscription name is correct.`);
      }
      throw new Error(`Unexpected error: ${JSON.stringify(err)}`);
    }
  }
  
  async deploymentLogs(subscription, account, stage, service, startTime) {
    try {
      const url = `${this.getDeploymentLogsUrl(subscription)}?stage=${stage}&account=${account}&serviceName=${service}&startTime=${startTime}&logType=deployment`;

      const { statusCode, body } = await requestAsync({
        uri: url,
        method: 'GET',
        headers: {'Authorization': `Bearer ${this.tokens.AuthenticationResult.IdToken}`},
        json: true
      });

      if (statusCode === 200) {
        return body;
      } else {
        throw new Error(`Received an error from Kumohub: ${statusCode} ${JSON.stringify(body)}`);
      }
    } catch (err) {
      if (err && err.errno && err.errno == "ENOTFOUND" && err.hostname) {
        throw new Error(`Unable to connect to ${err.hostname}, please check your network or profile subscription name is correct.`);
      }
      throw new Error(`Unexpected error: ${err}`);
    }
  }

  async getService(subscription, account, stage, service) {
    try {
      const url = this.getServiceUrl(subscription, account, stage, service);

      const { statusCode, body } = await requestAsync({
        uri: url,
        method: 'GET',
        headers: {'Authorization': `Bearer ${this.tokens.AuthenticationResult.IdToken}`},
        json: true
      });

      if (statusCode === 200) {
        return body;
      } else {
        throw new Error(`Received an error from Kumohub: ${statusCode} ${JSON.stringify(body)}`);
      }
    } catch (err) {
      if (err && err.errno && err.errno == "ENOTFOUND" && err.hostname) {
        throw new Error(`Unable to connect to ${err.hostname}, please check your network or profile subscription name is correct.`);
      }
      throw new Error(`Unexpected error: ${JSON.stringify(err)}`);
    }
  }
}

module.exports = KumohubDeployer;