grunt-contrib-qunit/tasks/qunit.js

Run QUnit unit tests in a headless Chrome instance

/*
 * grunt-contrib-qunit
 * https://gruntjs.com/
 *
 * Copyright (c) 2016 "Cowboy" Ben Alman, contributors
 * Licensed under the MIT license.
 */

'use strict';

// Nodejs libs.
var fs = require('fs');
var path = require('path');
var querystring = require('querystring');
// NPM libs.
var EventEmitter = require('eventemitter2');
var puppeteer = require('puppeteer');

var Promise = global.Promise;

// Shared functions

// Allow an error message to retain its color when split across multiple lines.
function formatMessage(message) {
  var str = String(message);
  if (typeof message === 'object' && /^\[object .*\]$/.test(str)) {
    // try to use the JSON as a better string representation
    try {
      str = JSON.stringify(message, null, 2);
    } catch (_) {
    }
  }
  return String(str).split('\n')
    .map(function(s) {
      return s.magenta;
    })
    .join('\n');
}


function createRunEnd() {
  return {
    status: 'passed',
    testCounts: {
      passed: 0,
      failed: 0,
      skipped: 0,
      todo: 0,
      total: 0
    },
    runtime: 0
  };
}

function combineRunEnd(combined, runEnd) {
  if (runEnd.status === 'failed') {
    combined.status = runEnd.status;
  }
  combined.testCounts.passed += runEnd.testCounts.passed;
  combined.testCounts.failed += runEnd.testCounts.failed;
  combined.testCounts.skipped += runEnd.testCounts.skipped;
  combined.testCounts.todo += runEnd.testCounts.todo;
  combined.testCounts.total += runEnd.testCounts.total;
  combined.runtime += runEnd.runtime;
}

function generateMessage(combined) {
  return [
    combined.testCounts.total,
    ' tests completed in ',
    combined.runtime,
    'ms, with ',
    combined.testCounts.failed,
    ' failed, ' +
    combined.testCounts.skipped,
    ' skipped, and ',
    combined.testCounts.todo,
    ' todo.'
  ].join('');
}

// Copied from QUnit source code
function generateHash(module) {
  var hex;
  var i = 0;
  var hash = 0;
  var str = module + '\x1C' + undefined;
  var len = str.length;

  for (; i < len; i++) {
    hash = ((hash << 5) - hash) + str.charCodeAt(i);
    hash |= 0;
  }

  // Convert the possibly negative integer hash code into an 8 character
  // hex string, which isn't strictly necessary but increases user understanding
  // that the id is a SHA-like hash
  hex = (0x100000000 + hash).toString(16);
  if (hex.length < 8) {
    hex = '0000000' + hex;
  }

  return hex.slice(-8);
}

module.exports = function(grunt) {

  var eventBus = new EventEmitter({wildcard: true, maxListeners: 0});

  // Keep track of the last-started module and test. Additionally, keep track
  // of status for individual test files and the entire test suite.
  var options;
  var combinedRunEnd;
  var failureBuffer = [];
  var browser;
  var page;

  // Get an asset file, local to the root of the project.
  var asset = path.join.bind(null, __dirname, '..');

  // If options.force then log an error, otherwise exit with a warning
  function warnUnlessForced(message) {
    if (options && options.force) {
      grunt.log.error(message);
    } else {
      grunt.warn(message);
    }
  }

  function formatFailedAssertion(error) {
    var failure = '' +
      'Message: ' + formatMessage(error.message) + '\n' +
      'Actual: ' + formatMessage(error.actual) + '\n' +
      'Expected: ' + formatMessage(error.expected);
    if (error.stack) {
      failure += '\n' + error.stack.replace(/^\s+(at) /g, '  $1 ');
    }
    return failure;
  }

  function waitForNextRunEnd(url) {
    return new Promise(function(resolve, reject) {
      eventBus.once('qunit.on.runEnd', function() { resolve(); });
      eventBus.once('fail.*', function() { reject(url); });
    });
  }

  // QUnit events.
  eventBus.on('qunit.on.testStart', function(testStart) {
    var name = testStart.fullName.join(' > ');
    grunt.verbose.write(name + '...');
  });

  eventBus.on('qunit.on.testEnd', function(testEnd) {
    var testPassed = (testEnd.status !== 'failed');
    if (testPassed) {
      // plainly "passed", or "skipped", or expected-failing "todo".
      //
      // Either complete the verbose testStart line, or continue dot progress.
      grunt.verbose.ok().or.write('.');
      return;
    }
    if (options && options.summaryOnly) {
      return;
    }

    var failure;
    if (testEnd.status === 'todo') {
      failure = 'Expected at least one failing assertion in todo test';
    } else {
      failure = testEnd.errors.map(formatFailedAssertion).join('\n');
    }

    if (grunt.option('verbose')) {
      grunt.log.error();
      grunt.log.error(failure);
    } else {
      var name = testEnd.fullName.join(' > ');
      failureBuffer.push(name + '\n' + failure);
      grunt.log.write('F'.red);
    }
  });

  eventBus.on('qunit.on.runEnd', function(runEnd) {
    if (!grunt.option('verbose')) {
      // End the non-verbose dot progress line
      if (runEnd.status === 'failed') {
        grunt.log.writeln();
      } else {
        grunt.log.ok();
      }
    }
    if (failureBuffer.length) {
      grunt.log.error(failureBuffer.join('\n'));
      failureBuffer.length = 0;
    }

    combineRunEnd(combinedRunEnd, runEnd);
  });

  // Re-broadcast qunit events on grunt.event.
  eventBus.on('qunit.**', function() {
    var args = [this.event].concat(grunt.util.toArray(arguments));
    grunt.event.emit.apply(grunt.event, args);
  });

  // Built-in error handlers.
  eventBus.on('fail.load', function(url) {
    grunt.verbose.write('...');
    grunt.event.emit('qunit.fail.load', url);
    grunt.log.error('Chrome unable to load \'' + url + '\' URI.');

    combinedRunEnd.status = 'failed';
  });

  eventBus.on('fail.timeout', function() {
    grunt.log.writeln();
    grunt.event.emit('qunit.fail.timeout');
    grunt.log.error('Chrome timed out, possibly due to:\n' +
        '- QUnit is not loaded correctly.\n- A missing QUnit start() call.\n' +
        '- Or, a misconfiguration of this task.');

    combinedRunEnd.status = 'failed';
  });

  eventBus.on('qunit.on.error', function(err) {
    // It is the responsibility of QUnit to ensure a run is marked as failure
    // if there are (unexpected) messages received from window.onerror.
    //
    // Prior to QUnit 2.17, details of global failures were printed by
    // creating a fake test with "testEnd" event. Now, it is our responsiblity
    // to print via `QUnit.on('error')`.
    //
    // NOTE: Avoid relying solely on Puppeteer's "pageerror" event, as that only
    // catches the subset of execution errors that make their way to window.onerror.
    // It misses out on unhandled rejections from the browser, the "No tests were run"
    // internal QUnit error, and anything else reported to QUnit.onUncaughtException
    // by QUnit plugins.
    // https://qunitjs.com/api/extension/QUnit.onUncaughtException/
    grunt.log.writeln();
    grunt.log.error(err);
    grunt.event.emit('qunit.error.onError', err);
  });

  eventBus.on('error.onError', function(msg) {
    // This is important in addition to `QUnit.on('error')` to catch uncaught
    // errors that happen before the bridge is in effect (which in practice
    // will happen at DOMContentLoaded, after qunit.js and test files have done
    // their initial execution, but before QUnit.begin event and any actual tests).
    grunt.log.writeln();
    grunt.log.error(msg.stack || msg);
    grunt.event.emit('qunit.error.onError', msg);
  });

  grunt.registerMultiTask('qunit', 'Run QUnit tests in Headless Chrome.', function() {
    // Chrome sandbox is incompatible with Docker and most CI environments
    var defaultChromiumArgs = (
      process.env.CHROMIUM_FLAGS || (process.env.CI ? '--no-sandbox' : '')
    ).split(' ');

    // Merge task-specific and/or target-specific options with these defaults.
    options = this.options({
      // Default Chrome timeout.
      timeout: 10000,
      // QUnit-Chrome bridge file to be injected.
      inject: asset('chrome/bridge.js'),
      // Explicit non-file URLs to test.
      urls: [],
      force: false,
      // Connect Chrome console output to Grunt output
      console: true,
      // Do not use an HTTP base by default
      httpBase: false,
      summaryOnly: false
    });
    var puppeteerLaunchOptions = Object.assign(
      {
        headless: true,
        args: defaultChromiumArgs
      },
      options.puppeteer
    );

    // This task is asynchronous.
    var done = this.async();

    // Read the content of the specified bridge files
    var bridgeFiles = Array.isArray(options.inject) ? options.inject : [options.inject];
    var bridgContents = [
      '__grunt_contrib_qunit_timeout__ = ' + JSON.stringify(options.timeout) + ';'
    ];

    for (var i = 0; i < bridgeFiles.length; i++) {
      try {
        bridgContents.push(fs.readFileSync(bridgeFiles[i], 'utf8'));
      } catch (err) {
        grunt.fail.fatal('Could not load the specified Chrome/QUnit bridge file: ' + bridgeFiles[i]);
      }
    }

    var unformedUrls;
    if (options.httpBase) {
      // If URLs are explicitly referenced, use them still
      unformedUrls = options.urls;
      // Then create URLs for the src files
      this.filesSrc.forEach(function(testFile) {
        unformedUrls.push(options.httpBase + '/' + testFile);
      });
    } else {
      // Combine any specified URLs with src files.
      unformedUrls = options.urls.concat(this.filesSrc);
    }

    // The final tasks to run before terminating the task
    function finishTask(success) {
      // Close the puppeteer browser
      if (browser) {
        browser.close();
      }
      // Finish the task
      done(success);
    }

    // For console output and eventBus, keep original short strings as configured by
    // and familiar to the user, such as "test/example.html", which are not valid URLs.
    // For page.goto(), we need a valid URL.
    //
    // The appended query strings should be reflected in both, which we used to do
    // with the now-deprecated url.parse()/url.format() that allow and preserve
    // unset protocols and relative paths, unlike its WHATWG URL replacement.
    function appendToUnformedUrl(urlish, queryParam, value) {
      let [beforeFrag, ...fragmentBits] = urlish.split('#');
      let [beforeSearch, ...searchBits] = beforeFrag.split('?');
      const search = querystring.stringify({
        ...querystring.parse(searchBits.join('?')),
        [queryParam]: value
      });
      return beforeSearch + '?' + search +
        (fragmentBits.length ? '#' + fragmentBits.join('#') : '');
    }

    var urls = new Map();
    for (let shortUrl of unformedUrls) {
      // Append a noglobal query string param to all urls
      if (options.noGlobals) {
        shortUrl = appendToUnformedUrl(shortUrl, 'noglobals', 'true');
      }

      // Append moduleId to all urls
      if (grunt.option('modules')) {
        const hashes = grunt.option('modules').split(',').map(function(module) {
          return generateHash(module.trim());
        });
        shortUrl = appendToUnformedUrl(shortUrl, 'moduleId', hashes);
      }

      // Append seed to all urls
      if (grunt.option('seed')) {
        shortUrl = appendToUnformedUrl(shortUrl, 'seed', grunt.option('seed'));
      }

      // Expand "test/example.html" to file URL with absolute file path
      const url = (shortUrl.startsWith('http://') || shortUrl.startsWith('https://')) ?
        shortUrl :
        'file://' + path.resolve(process.cwd(), shortUrl);

      urls.set(shortUrl, url);
    }

    // Reset combined data.
    combinedRunEnd = createRunEnd();

    // Instantiate headless browser
    puppeteer.launch(puppeteerLaunchOptions)
      .then(function(b) {
        browser = b;
        return b.newPage();
      })
      .then(function(p) {
        page = p;
        // emit events published in bridge.js.
        // This function exposure survives url navigations.
        return page.exposeFunction('__grunt_contrib_qunit__', function() {
          eventBus.emit.apply(eventBus, [].slice.call(arguments));
        });
      })
      .then(async function() {
        // Pass through the console logs if instructed
        if (options.console) {
          page.on('console', function(msg) {
            // The `msg` is a puppeteer.ConsoleMessage, which represents the console call
            // including multple arguments passed to it.
            //
            // msg.text() formats the arguments into a naive newline-joined string, and
            // includes error objects as a useless "JSHandle@error".
            //
            // msg.args() returns a JSHandle object for each argument, but all its
            // evaluation features happen asynchronously via the browser, and in this
            // event handler we can't await those easily as the grunt output will have
            // moved on to other tests. If we want to print these, we'd have to refactor
            // this so that the for-loop over "urls" below is aware of async code here.
            // For now, we just let the useless "JSHandle" through and rely on developers
            // to stringify any useful information ahead of time, e.g. `console.warn(String(err))`.
            //
            // Ref https://pptr.dev/#?product=Puppeteer&version=v9.0.0&show=api-class-consolemessage
            var colors = {
              'error': 'red',
              'warning': 'yellow'
            };
            var txt = msg.text();
            var color = colors[msg.type()];
            grunt.log.writeln(color ? txt[color] : txt);
            // grunt.log.writeln(`${msg.location().url}:${msg.location().lineNumber}`.gray); // debug
          });
        }

        // Surface uncaught exceptions
        // Ref https://pptr.dev/#?product=Puppeteer&version=v9.0.0&show=api-event-pageerror
        page.on('pageerror', function(err) {
          eventBus.emit('error.onError', err);
        });

        // Whenever a page is loaded with a new document, before scripts execute, inject the bridge file.
        // Tell the client that when DOMContentLoaded fires, it needs to tell this
        // script to inject the bridge. This should ensure that the bridge gets
        // injected before any other DOMContentLoaded or window.load event handler.
        page.evaluateOnNewDocument(
          'if (window.QUnit) {\n' +
            bridgContents.join(';') +
            '\n} else {\n' +
            'document.addEventListener("DOMContentLoaded", function() {\n' +
            bridgContents.join(';') +
            '\n});\n}\n'
        );

        for (const [shortUrl, url] of urls) {
          // Reset current module.
          grunt.event.emit('qunit.spawn', shortUrl);
          grunt.verbose.subhead('Testing ' + shortUrl + ' ').or.write('Testing ' + shortUrl + ' ');

          await Promise.all([
            // Setup "once" listener for qunit.done / fail events
            waitForNextRunEnd(shortUrl),

            // Navigate to the url to be tested
            page.goto(url, { timeout: options.timeout })
          ]);
        }
      })
      .then(function() {
        // All tests have been run.
        var message = generateMessage(combinedRunEnd);
        var success = (combinedRunEnd.status === 'passed');

        // Log results.
        if (!success) {
          warnUnlessForced(message);
        } else {
          grunt.verbose.writeln();
          grunt.log.ok(message);
        }

        if (!success && options && options.force) {
          success = true;
        }

        // All done!
        finishTask(success);
      })
      .catch(function(err) {
        // If anything goes wrong, terminate the grunt task
        grunt.log.error('There was an error with headless chrome');
        grunt.fail.fatal(err);
        finishTask(false);
      });
  });

};