hubot-aws-insight/src/aws-insight.js

A hubot scripts that shows you what's running on aws. Needs Python 3

// Description
//   A hubot scripts that shows our aws resources
//
// Configuration:
//
// Commands
//   hubot aws costly
//   hubot aws summary [include default] [exclude tf] [on account <account>]
//   hubot aws grep [<search>] [on account <account>]
//   hubot add aws account 123 nilo
//   hubot forget aws account nilo
//   hubot ls aws accounts
//
// Notes:
//
// Author:
//   Farid Nouri Neshat <FaridN_SOAD@Yahoo.com>

'use strict';

const {spawn} = require('child_process');
const JSONStream = require('json-stream');
const PBatch = require('p-batch');
const S3Lambda = require('s3-lambda');
const sts = new (require('aws-sdk/clients/sts'));
const util = require('util');
const fs = require('fs');
const ColorHash = require('color-hash');

const colorHash = new ColorHash();
const lambda = new S3Lambda({});
const writeFile = util.promisify(fs.writeFile);
const skewFilePath = __dirname + '/../tmp/skew';
const awsFilePath = __dirname + '/../tmp/aws';
const namespace = 'aws-insight.accounts';

module.exports = function (robot) {
  const createListCp = (arn, costly) => {
    saveAccounts();
    const args = [__dirname + '/../aws-list.py', '--arn', arn];
    if (costly) {
      args.push('--costly');
    }
    return spawn('python3', args, {
      env: {
        ...process.env,
        PYTHONPATH: __dirname + '/../pip',
        SKEW_CONFIG: skewFilePath,
        AWS_CONFIG_FILE: awsFilePath,
      },
    });
  };

  const getAccounts = () => robot.brain.get(namespace) || {};

  const setAccount = (name, account) => {
    const accounts = getAccounts();
    accounts[name] = account;
    saveAccounts(accounts);
  };

  const saveAccounts = (accounts) => {
    if (accounts) {
      robot.brain.set(namespace, accounts);
    } else {
      accounts = getAccounts();
    }

    writeSkewFile(accounts);
    writeAwsFile(accounts);

    function writeSkewFile(accounts) {
      const skewFile = `---
  accounts:${Object.entries(accounts).map(([ profile, no ]) => `
    "${no}":
      profile: ${profile}`).join('')}`;
      return writeFile(skewFilePath, skewFile);
    }

    function writeAwsFile(accounts) {
      const awsFile = Object.entries(accounts).map(([ profile, no ]) => {
        return (profile === 'default') ? '[default]' : `[profile ${profile}]
role_arn = arn:aws:iam::${no}:role/nilo-v1
source_profile = default`;
      }).join('\n');

      writeFile(awsFilePath, awsFile);
    }
  };

  sts.getCallerIdentity({}).promise().then(({ Account }) => {
    setAccount('default', Account);
  });

  const run = ({ filterData, res, formatResult, account = '*', costly }) => {
    const cp = createListCp(`arn:aws:*:*:${account}:*`, costly);
    let firstMessagePromise =
      robot.adapter.client.web.chat.postMessage(res.envelope.room,
        'Searching...');
    const text = "I'm still searching, but here's what I found so far:";
    const groups = {};
    let ts;
    const batch = new PBatch(() => {
      const attachments = [];
      for (let service in groups) {
        let result = `For service ${service}:\n`;
        for (let region in groups[service]) {
          result += `In region ${region || 'global'}:\n`;
          for (let type in groups[service][region]) {
            result += formatResult(groups[service][region][type], type) + '\n';
          }
          result += '\n';
        }
        attachments.push({
          color: colorHash.hex(service),
          text: result
        });
      }
      robot.adapter.client.web.chat.update(ts, res.envelope.room, text,
        {attachments});

      // Needed for PBatch.
      return [];
    });

    const update = () => batch.add().catch(console.error);
    cp.stdout.pipe(JSONStream()).on('data', async resource => {
      let [,,service, region, account, type, id] = resource.arn.split(/[:\/]/);
      if (!id) {
        id = type;
      }
      if (filterData &&
        !filterData({ ...resource, service, region, account, type, id })) {
        return;
      }

      if (!groups[service]) {
        groups[service] = {};
      }

      if (!groups[service][region]) {
        groups[service][region] = {};
      }

      groups[service][region][type] = (groups[service][region][type] || [])
        .concat(id);

      if (firstMessagePromise) {
        const result = await firstMessagePromise;
        ts = result.ts;
        firstMessagePromise = null;
      }
      update();
    }).on('end', () => {
      res.send('I think that was all!');
    });

    const onError = msg => {
      msg = msg.toString().trim();
      console.error(msg);
      if (msg) {
        res.send('Something bad happened: ' + msg);
      }
    };

    cp.stderr.on('data', onError);
    cp.on('error', err => onError(err.message || err));
  };

  const getAccount = query => {
    let name;
    const rest = query.replace(/(on|for)? *account *(\S*)/,
      ($0, $1, $2) => {
        name = $2;
        return ' ';
      }).trim();
    return {
      account: getAccounts()[name] || name,
      rest,
    }
  };

  robot.respond(/aws grep (.*?) */i, res => {
    const { account, rest: search } = getAccount(res.match[1]);
    run({
      res,
      account,
      formatResult: (ids, type) =>
        `There's the following ${type}(s): ${ids.join(',')}`,
      filterData: ({data}) => data.includes(search),
    });
  });

  robot.respond(/aws costly *(.*) */i, res => {
    const { account } = getAccount(res.match[1]);
    run({
      account,
      res,
      formatResult: summaryFormat,
      costly: true,
    });
  });

  const summaryFormat = (ids, type) => `There's ${ids.length} ${type}`;

  robot.respond(/aws summary *(.*) */i, async res => {
    const { account, rest } = getAccount(res.match[1]);
    const keywords = rest.match(/(include|exclude) +(default|tf)/g);
    const args = {
      default: false, // exclude default (by default)!
      tf: true, // include tf (by default)
    };

    if (keywords) {
      keywords.forEach(match => {
        const [verb, subject] = match.split(/ +/);
        args[subject] = verb === 'include';
      });
    }

    const toExcludeIds = new Set();

    if (!args.tf) {
      await lambda
        .context({
          bucket: process.env.TERRAFORM_S3_BUCKET,
        })
        .forEach(object => {
          try {
            object = JSON.parse(object);
          } catch (e) {
            console.error("Can't parse the S3 object into JSON.");
            return;
          }
          for (const module of object.modules) {
            for (const key in module.resources) {
              const {primary} = module.resources[key];
              toExcludeIds.add(primary.attributes.arn || primary.id);
            }
          }
        });
    }

    run({
      res,
      account,
      formatResult: (ids, type) =>
        `There's ${ids.length} ${type}`,
      filterData: ({arn, data, id, type}) => {
        let parsed;
        const getData = () => parsed = parsed || JSON.parse(data);
        if (!args.tf) {
          if (toExcludeIds.has(arn) || toExcludeIds.has(id)) {
            return false;
          }
          const data = getData();
          if (data.Attachments && data.Attachments
              .some(({ InstanceId }) => toExcludeIds.has(InstanceId))) {
            return false;
          }
        }

        if (!args.default) {
          const data = getData();
          return !(data.IsDefault || // For vpc and network acl
            data.DefaultForAz || // For subnet
            data.GroupName === 'default' || // for ec2 security group
            data.DBSecurityGroupName === 'default' || // For rds security groups
            type === 'policy' && data.Arn.split(':')[4] === 'aws' || // AWS managed policies

            // For route tables
            (data.Associations && data.Associations.some(({Main}) => Main)));
        }
        return true;
      }
    });
  });

  robot.respond(/add aws account *(\S*) *(\S*)/i, res => {
    res.reply('Added the account.');
    const [,a,b] = res.match;
    let accNo, profile;
    if (parseInt(a)) {
      accNo = a;
      profile = b;
    } else {
      accNo = b;
      profile = a;
    }

    setAccount(profile, accNo);
  });

  robot.respond(/(list|ls|show) aws accounts?/i, res => {
    res.reply(`Here's accounts: ${util.inspect(getAccounts())}`);
  });

  robot.respond(/forget aws account *(\S*)/i, res => {
    const [,profile] = res.match;
    const accounts = getAccounts();
    delete accounts[profile];
    saveAccounts(accounts);
    res.reply(`Yes, master!`);
  });
};