artillery/lib/cmds/run.ts

Cloud-scale load testing. https://www.artillery.io

import { createRequire } from 'node:module';
import { promisify as p } from 'node:util';
import { Args, Command, Flags } from '@oclif/core';
import _csv from 'csv-parse';
import createDebug from 'debug';
import { CommonRunFlags } from '../cli/common-flags.ts';

const debug = createDebug('commands:run');
const require = createRequire(import.meta.url);

import crypto from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { ssms as __ssms } from '../core/index.ts';
import dotenv from 'dotenv';
import _ from 'lodash';
import moment from 'moment';
import createConsoleReporter from '../console-reporter.ts';
import createLauncher from '../launch-platform.ts';

const { SSMS } = __ssms;

import { Plugin as CloudPlugin } from '../platform/cloud/cloud.ts';
import * as telemetry from '../telemetry.ts';
import generateId from '../util/generate-id.ts';
import parseTagString from '../util/parse-tag-string.ts';
import prepareTestExecutionPlan from '../util/prepare-test-execution-plan.ts';
import type { PeriodMetrics } from '../core/ssms.ts';
import type { MergedScript } from '../util.ts';

class RunCommand extends Command {
  // Untyped JS class - properties assigned dynamically
  [key: string]: any;
  static runCommandImplementation: any;

  static aliases = ['run'];
  // Enable multiple args:
  static strict = false;

  async run() {
    const { flags, argv, args } = await this.parse(RunCommand);

    if (flags.platform === 'aws:ecs') {
      // Delegate to existing implementation
      const RunFargateCommand = (await import('./run-fargate.ts')).default;
      return await RunFargateCommand.run(argv as string[]);
    }

    await RunCommand.runCommandImplementation(flags, argv, args);
  }

  // async catch(err) {
  //   throw err;
  // }
}

// Line no. 2 onwards is the description in help output
RunCommand.description = `run a test script locally or on AWS Lambda
Run a test script

Examples:

  To run a test script in my-test.yml to completion from the local machine:

    $ artillery run my-test.yml

  To run a test script but override target dynamically:

    $ artillery run -t https://app2.acmecorp.internal my-test.yml
`;

// TODO: Link to an Examples section in the docs

RunCommand.flags = {
  ...CommonRunFlags,
  // TODO: Deprecation notices for commands below:
  payload: Flags.string({
    char: 'p',
    description: 'Specify a CSV file for dynamic data'
  }),
  solo: Flags.boolean({
    char: 's',
    description: 'Create only one virtual user'
  }),
  platform: Flags.string({
    description: 'Runtime platform',
    default: 'local',
    options: ['local', 'aws:lambda', 'az:aci']
  }),
  'platform-opt': Flags.string({
    description:
      'Set a platform-specific option, e.g. --platform-opt region=eu-west-1 for AWS Lambda',
    multiple: true
  }),
  count: Flags.string({
    // locally defaults to number of CPUs with mode = distribute
    default: '1'
  })
};

RunCommand.args = {
  script: Args.string({
    name: 'script',
    required: true
  })
};

let cloud: InstanceType<typeof CloudPlugin> | undefined;
RunCommand.runCommandImplementation = async (
  flags: Record<string, any>,
  argv: string[],
  args: Record<string, any>
) => {
  // Collect all input files for reading/parsing - via args, --config, or -i
  const inputFiles = argv.concat(flags.input || [], flags.config || []);

  const tagResult = parseTagString(flags.tags);
  if (tagResult.errors.length > 0) {
    console.log(
      'WARNING: could not parse some tags:',
      tagResult.errors.join(', ')
    );
  }

  if (tagResult.tags.length > 10) {
    console.log('A maximum of 10 tags is supported');
    process.exit(1);
  }

  // TODO: Move into PlatformLocal
  if (flags.dotenv) {
    const dotEnvPath = path.resolve(process.cwd(), flags.dotenv);
    try {
      fs.statSync(dotEnvPath);
    } catch (_err) {
      console.log(`WARNING: could not read dotenv file: ${flags.dotenv}`);
    }
    dotenv.config({ path: dotEnvPath });
  }

  if (flags.output) {
    if (!checkDirExists(flags.output)) {
      console.error('Path does not exist:', flags.output);
      process.exit(1);
    }
  }

  const testRunId = process.env.ARTILLERY_TEST_RUN_ID || generateId('t');
  console.log('Test run id:', testRunId);
  global.artillery.testRunId = testRunId;
  cloud = new CloudPlugin(null, null, { flags });
  global.artillery.cloudEnabled = cloud.enabled;

  if (cloud.enabled) {
    try {
      await cloud.init();
    } catch (caughtErr) {
      const err = caughtErr as Error;
      if (err.name === 'CloudAPIKeyMissing') {
        console.error(
          'Error: API key is required to record test results to Artillery Cloud'
        );
        console.error(
          'See https://docs.art/get-started-cloud for more information'
        );

        await gracefulShutdown({ exitCode: 7 });
      } else if (err.name === 'APIKeyUnauthorized') {
        console.error(
          'Error: API key is not recognized or is not authorized to record tests'
        );

        await gracefulShutdown({ exitCode: 7 });
      } else if (err.name === 'PingFailed') {
        console.error(
          'Error: unable to reach Artillery Cloud API. This could be due to firewall restrictions on your network'
        );
        console.log('https://docs.art/cloud/err-ping');
        await gracefulShutdown({ exitCode: 7 });
      } else {
        console.error(
          'Error: something went wrong connecting to Artillery Cloud'
        );
        console.error('Check https://status.artillery.io for status updates');
        console.error(err);
      }
    }
  }

  let script: MergedScript | undefined;
  try {
    script = await prepareTestExecutionPlan(inputFiles, flags, args);
  } catch (err) {
    console.error('Error:', (err as Error).message);
    await gracefulShutdown({ exitCode: 1 });
  }
  // gracefulShutdown() exits the process on failure, so the script
  // is always set past this point:
  const runScript = script as MergedScript;

  var runnerOpts: any = {
    environment: flags.environment,
    // This is used in the worker to resolve
    // the path to the processor module
    scriptPath: args.script,
    // TODO: This should be an array of files, like inputFiles above
    absoluteScriptPath: path.resolve(process.cwd(), args.script),
    plugins: [],
    scenarioName: flags['scenario-name']
  };

  // Set "name" tag if not set explicitly
  if (tagResult.tags.filter((t) => t.name === 'name').length === 0) {
    tagResult.tags.push({
      name: 'name',
      value: path.basename(runnerOpts.scriptPath)
    });
  }
  // Override the "name" tag with the value of --name if set
  if (flags.name) {
    for (const t of tagResult.tags) {
      if (t.name === 'name') {
        t.value = flags.name;
      }
    }
  }

  if (flags.config) {
    runnerOpts.absoluteConfigPath = path.resolve(process.cwd(), flags.config);
  }

  if (process.env.WORKERS) {
    runnerOpts.count = parseInt(process.env.WORKERS, 10) || 1;
  }
  if (flags.solo) {
    runnerOpts.count = 1;
  }

  const platformConfig: Record<string, string> = {};
  if (flags['platform-opt']) {
    for (const opt of flags['platform-opt']) {
      const [k, v] = opt.split('=');
      platformConfig[k] = v;
    }
  }

  const launcherOpts = {
    platform: flags.platform,
    platformConfig,
    mode: flags.platform === 'local' ? 'distribute' : 'multiply',
    count: parseInt(flags.count || 1, 10),
    cliArgs: flags,
    testRunId
  };

  var launcher = await createLauncher(
    runScript,
    runScript.config.payload,
    runnerOpts,
    launcherOpts
  );

  if (!launcher) {
    console.log('Failed to create launcher');
    await gracefulShutdown({ exitCode: 1 });
  }
  // As above: gracefulShutdown() exits when the launcher is missing.
  const activeLauncher = launcher as NonNullable<typeof launcher>;

  const intermediates: Array<Record<string, any>> = [];

  const metricsToSuppress = getPluginMetricsToSuppress(runScript);
  // TODO: Wire up workerLog or something like that
  const consoleReporter = createConsoleReporter(activeLauncher.events, {
    quiet: flags.quiet || false,
    metricsToSuppress
  });

  var reporters = [consoleReporter];
  if (process.env.CUSTOM_REPORTERS) {
    const customReporterNames = process.env.CUSTOM_REPORTERS.split(',');
    for (const name of customReporterNames) {
      // Resolve with CJS semantics, load with import() - handles both
      // CJS and ESM reporters, including ESM with top-level await
      const reporterPath = require.resolve(name);
      const ns = await import(pathToFileURL(reporterPath).href);
      const createReporter = ns.default ?? ns;
      const reporter = createReporter(activeLauncher.events, flags);
      reporters.push(reporter);
    }
  }

  activeLauncher.events.on('phaseStarted', (_phase: unknown) => {});

  activeLauncher.events.on('stats', (stats: Record<string, any>) => {
    if (artillery.runtimeOptions.legacyReporting) {
      const report = SSMS.legacyReport(stats as PeriodMetrics).report();
      intermediates.push(report);
    } else {
      intermediates.push(stats);
    }
  });

  activeLauncher.events.on('done', async (stats: Record<string, any>) => {
    let report: Record<string, any>;
    if (artillery.runtimeOptions.legacyReporting) {
      report = SSMS.legacyReport(stats as PeriodMetrics).report();
      report.phases = _.get(runScript, 'config.phases', []);
    } else {
      report = stats;
    }

    if (flags.output) {
      const logfile = getLogFilename(flags.output) as string;
      if (!flags.quiet) {
        console.log('Log file: %s', logfile);
      }

      for (const ix of intermediates) {
        delete ix.histograms;
        ix.histograms = ix.summaries;
      }
      delete report.histograms;
      report.histograms = report.summaries;

      fs.writeFileSync(
        logfile,
        JSON.stringify(
          {
            aggregate: report,
            intermediate: intermediates
          },
          null,
          2
        ),
        { flag: 'w' }
      );
    }

    // This is used in the beforeExit event handler in gracefulShutdown
    finalReport = report;
    await gracefulShutdown();
  });

  global.artillery.ext({
    ext: 'beforeExit',
    method: async (event) => {
      try {
        const duration = Math.round(
          (event.report?.lastMetricAt - event.report?.firstMetricAt) / 1000
        );
        await sendTelemetry(runScript, flags, { duration });
      } catch (_err) {}
    }
  });

  global.artillery.testInfo = {
    flags,
    testRunId,
    tags: tagResult.tags,
    metadata: {
      testId: testRunId,
      startedAt: Date.now(),
      count: runnerOpts.count || Number(flags.count),
      tags: tagResult.tags,
      launchType: flags.platform,
      artilleryVersion: {
        core: global.artillery.version
      },
      // Properties from the runnable script object:
      // (see runScript alias above)
      testConfig: {
        target: runScript.config.target,
        phases: runScript.config.phases,
        plugins: runScript.config.plugins,
        environment: runScript._environment,
        scriptPath: runScript._scriptPath,
        configPath: runScript._configPath
      }
    }
  };

  global.artillery.globalEvents.emit('test:init', global.artillery.testInfo);

  activeLauncher.run();

  var finalReport = {};
  var shuttingDown = false;
  process.on('SIGINT', async () => {
    gracefulShutdown({ earlyStop: true, exitCode: 130 });
  });
  process.on('SIGTERM', async () => {
    gracefulShutdown({ earlyStop: true, exitCode: 143 });
  });

  async function gracefulShutdown(opts: any = { exitCode: 0 }) {
    debug('shutting down 🦑');
    if (shuttingDown) {
      return;
    }

    debug('Graceful shutdown initiated');

    shuttingDown = true;
    global.artillery.globalEvents.emit('shutdown:start', opts);

    // Run beforeExit first, and then onShutdown

    const ps = [];
    for (const e of global.artillery.extensionEvents) {
      const testInfo = { endTime: Date.now() };
      if (e.ext === 'beforeExit') {
        ps.push(
          e.method({
            ...opts,
            report: finalReport,
            flags,
            runnerOpts,
            testInfo
          })
        );
      }
    }
    await Promise.allSettled(ps);

    const ps2 = [];
    for (const e of global.artillery.extensionEvents) {
      if (e.ext === 'onShutdown') {
        ps2.push(e.method(opts));
      }
    }
    await Promise.allSettled(ps2);

    if (launcher) {
      await launcher.shutdown();
    }

    await (async () => {
      if (reporters) {
        for (const r of reporters) {
          if (r.cleanup) {
            try {
              await p(r.cleanup.bind(r))();
            } catch (cleanupErr) {
              debug(cleanupErr);
            }
          }
        }
      }

      if (
        global.artillery.hasTypescriptProcessor &&
        !process.env.ARTILLERY_TS_KEEP_BUNDLE
      ) {
        try {
          fs.unlinkSync(global.artillery.hasTypescriptProcessor);
        } catch (err) {
          console.log(
            `WARNING: Failed to remove typescript bundled file: ${global.artillery.hasTypescriptProcessor}`
          );
          console.log(err);
        }
        try {
          fs.rmdirSync(path.dirname(global.artillery.hasTypescriptProcessor));
        } catch (_err) {}
      }
      debug('Cleanup finished');
      process.exit(artillery.suggestedExitCode || opts.exitCode);
    })();
  }

  global.artillery.shutdown = gracefulShutdown;
};

async function sendTelemetry(
  script: Record<string, any>,
  flags: Record<string, any>,
  extraProps: Record<string, any>
) {
  if (process.env.WORKER_ID) {
    debug('Telemetry: Running in cloud worker, skipping test run event');
    return;
  }

  function hash(str: string) {
    return crypto.createHash('sha1').update(str).digest('base64');
  }

  const properties: Record<string, any> = {};

  if (script.config?.__createdByQuickCommand) {
    properties.quick = true;
  }
  if (cloud?.enabled && cloud.user) {
    properties.cloud = cloud.user;
  }

  properties.solo = flags.solo;
  try {
    // One-way hash of target endpoint:
    if (script.config?.target) {
      const targetHash = hash(script.config.target);
      properties.targetHash = targetHash;
    }

    if (flags.target) {
      const targetHash = hash(flags.target);
      properties.targetHash = targetHash;
    }

    properties.platform = flags.platform;

    properties.count = flags.count;

    if (properties.targetHash) {
      properties.distinctId = properties.targetHash;
    }

    let macaddr;
    const nonInternalIpv6Interfaces: Array<{ mac: string }> = [];
    for (const [_iface, descrs] of Object.entries(os.networkInterfaces())) {
      for (const o of descrs ?? []) {
        if (o.internal === true) {
          continue;
        }

        //prefer ipv4 interface when available
        if (o.family !== 'IPv4') {
          nonInternalIpv6Interfaces.push(o);
          continue;
        }

        macaddr = o.mac;
        break;
      }
    }

    //default to first ipv6 interface if no ipv4 interface is available
    if (!macaddr && nonInternalIpv6Interfaces.length > 0) {
      macaddr = nonInternalIpv6Interfaces[0].mac;
    }

    if (macaddr) {
      properties.macHash = hash(macaddr);
    }
    properties.hostnameHash = hash(os.hostname());
    properties.usernameHash = hash(os.userInfo().username);

    if (script.config?.engines) {
      properties.loadsEngines = true;
    }

    properties.enginesUsed = [];
    const OSS_ENGINES = [
      'http',
      'socketio',
      'ws',

      'playwright',
      'kinesis',
      'socketio-v3',
      'rediscluster',
      'kafka',
      'tcp',
      'grpc',
      'meteor',
      'graphql-ws',
      'ldap',
      'lambda'
    ];

    for (const scenario of script.scenarios || []) {
      if (OSS_ENGINES.indexOf(scenario.engine || 'http') > -1) {
        if (properties.enginesUsed.indexOf(scenario.engine || 'http') === -1) {
          properties.enginesUsed.push(scenario.engine || 'http');
        }
      }
    }

    // Official plugins:
    if (script.config.plugins) {
      properties.plugins = true;
      properties.officialPlugins = [];
      const OFFICIAL_PLUGINS = [
        'apdex',
        'expect',
        'publish-metrics',
        'metrics-by-endpoint',
        'hls',
        'fuzzer',
        'ensure',
        'memory-inspector',
        'fake-data',
        'slack'
      ];
      for (const p of OFFICIAL_PLUGINS) {
        if (script.config.plugins[p]) {
          properties.officialPlugins.push(p);
        }
      }
    }

    // publish-metrics reporters
    if (script.config.plugins['publish-metrics']) {
      const OFFICIAL_REPORTERS = [
        'datadog',
        'open-telemetry',
        'newrelic',
        'splunk',
        'dynatrace',
        'cloudwatch',
        'honeycomb',
        'mixpanel',
        'prometheus'
      ];

      properties.officialMonitoringReporters = script.config.plugins[
        'publish-metrics'
      ]
        .map((reporter: Record<string, any>) => {
          if (OFFICIAL_REPORTERS.includes(reporter.type)) {
            return reporter.type;
          }
          return undefined;
        })
        .filter((type: string | undefined) => type !== undefined);
    }

    // before/after hooks
    if (script.before) {
      properties.beforeHook = true;
    }

    if (script.after) {
      properties.afterHook = true;
    }

    Object.assign(properties, extraProps);
  } catch (err) {
    debug(err);
  } finally {
    telemetry.capture('test run', properties);
  }
}

function checkDirExists(output: string | undefined): boolean | undefined {
  if (!output) {
    return;
  }
  // If destination is a file check only path to its directory
  const exists = path.extname(output)
    ? fs.existsSync(path.dirname(output))
    : fs.existsSync(output);

  return exists;
}

function getLogFilename(output: string, nameFormat?: string) {
  let logfile: string | undefined;

  // is the destination a directory that exists?
  let isDir = false;
  if (output) {
    try {
      isDir = fs.statSync(output).isDirectory();
    } catch (_err) {
      // ENOENT do nothing, handled in checkDirExists before test run
    }
  }

  const defaultFormat = '[artillery_report_]YMMDD_HHmmSS[.json]';
  if (!isDir && output) {
    // -o is set with a filename (existing or not)
    logfile = output;
  } else if (!isDir && !output) {
    // no -o set
  } else {
    // -o is set with a directory
    logfile = path.join(output, moment().format(nameFormat || defaultFormat));
  }

  return logfile;
}

function getPluginMetricsToSuppress(script: Record<string, any>): string[] {
  if (!script.config.plugins) {
    return [];
  }
  const metrics: string[] = [];
  for (const [plugin, options] of Object.entries<any>(script.config.plugins)) {
    if (options.suppressOutput) {
      metrics.push(`plugins.${plugin}`);
    }
  }
  return metrics;
}

export default RunCommand;