@revoloo/cypress6
Version:
Cypress.io end to end testing tool
1,469 lines (1,260 loc) • 65.7 kB
JavaScript
require('../spec_helper')
const R = require('ramda')
const _ = require('lodash')
const path = require('path')
const EE = require('events')
const http = require('http')
const Promise = require('bluebird')
const electron = require('electron')
const commitInfo = require('@cypress/commit-info')
const Fixtures = require('../support/helpers/fixtures')
const snapshot = require('snap-shot-it')
const stripAnsi = require('strip-ansi')
const debug = require('debug')('test')
const pkg = require('@packages/root')
const launcher = require('@packages/launcher')
const extension = require('@packages/extension')
const argsUtil = require(`${root}lib/util/args`)
const { fs } = require(`${root}lib/util/fs`)
const ciProvider = require(`${root}lib/util/ci_provider`)
const settings = require(`${root}lib/util/settings`)
const Events = require(`${root}lib/gui/events`)
const Windows = require(`${root}lib/gui/windows`)
const interactiveMode = require(`${root}lib/modes/interactive-e2e`)
const runMode = require(`${root}lib/modes/run`)
const api = require(`${root}lib/api`)
const cwd = require(`${root}lib/cwd`)
const user = require(`${root}lib/user`)
const config = require(`${root}lib/config`)
const cache = require(`${root}lib/cache`)
const errors = require(`${root}lib/errors`)
const plugins = require(`${root}lib/plugins`)
const cypress = require(`${root}lib/cypress`)
const { ProjectBase } = require(`${root}lib/project-base`)
const { ProjectE2E } = require(`${root}lib/project-e2e`)
const { ServerE2E } = require(`${root}lib/server-e2e`)
const Reporter = require(`${root}lib/reporter`)
const Watchers = require(`${root}lib/watchers`)
const browsers = require(`${root}lib/browsers`)
const videoCapture = require(`${root}lib/video_capture`)
const browserUtils = require(`${root}lib/browsers/utils`)
const chromeBrowser = require(`${root}lib/browsers/chrome`)
const openProject = require(`${root}lib/open_project`)
const env = require(`${root}lib/util/env`)
const v = require(`${root}lib/util/validation`)
const system = require(`${root}lib/util/system`)
const appData = require(`${root}lib/util/app_data`)
const electronApp = require('../../lib/util/electron-app')
const savedState = require(`${root}lib/saved_state`)
const TYPICAL_BROWSERS = [
{
name: 'chrome',
family: 'chromium',
channel: 'stable',
displayName: 'Chrome',
version: '60.0.3112.101',
path: '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
majorVersion: '60',
}, {
name: 'chromium',
family: 'chromium',
channel: 'stable',
displayName: 'Chromium',
version: '49.0.2609.0',
path: '/Users/bmann/Downloads/chrome-mac/Chromium.app/Contents/MacOS/Chromium',
majorVersion: '49',
}, {
name: 'chrome',
family: 'chromium',
channel: 'canary',
displayName: 'Canary',
version: '62.0.3197.0',
path: '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
majorVersion: '62',
},
]
const ELECTRON_BROWSER = {
name: 'electron',
family: 'chromium',
displayName: 'Electron',
path: '',
version: '99.101.1234',
majorVersion: 99,
}
const previousCwd = process.cwd()
const snapshotConsoleLogs = function (name) {
const args = _
.chain(console.log.args)
.map((innerArgs) => {
return innerArgs.join(' ')
}).join('\n')
.value()
// our cwd() is currently the project
// so must switch back to original
process.chdir(previousCwd)
return snapshot(name, stripAnsi(args))
}
describe('lib/cypress', () => {
require('mocha-banner').register()
beforeEach(function () {
this.timeout(8000)
cache.__removeSync()
Fixtures.scaffold()
this.todosPath = Fixtures.projectPath('todos')
this.pristinePath = Fixtures.projectPath('pristine')
this.noScaffolding = Fixtures.projectPath('no-scaffolding')
this.recordPath = Fixtures.projectPath('record')
this.pluginConfig = Fixtures.projectPath('plugin-config')
this.pluginBrowser = Fixtures.projectPath('plugin-browser')
this.idsPath = Fixtures.projectPath('ids')
// force cypress to call directly into main without
// spawning a separate process
sinon.stub(videoCapture, 'start').resolves({})
sinon.stub(plugins, 'init').resolves(undefined)
sinon.stub(electronApp, 'isRunning').returns(true)
sinon.stub(extension, 'setHostAndPath').resolves()
sinon.stub(launcher, 'detect').resolves(TYPICAL_BROWSERS)
sinon.stub(process, 'exit')
sinon.stub(ServerE2E.prototype, 'reset')
sinon.stub(errors, 'warning')
.callThrough()
.withArgs('INVOKED_BINARY_OUTSIDE_NPM_MODULE')
.returns(null)
sinon.spy(errors, 'log')
sinon.spy(errors, 'logException')
sinon.spy(console, 'log')
// to make sure our Electron browser mock object passes validation during tests
sinon.stub(process, 'versions').value({
chrome: ELECTRON_BROWSER.version,
electron: '123.45.6789',
})
this.expectExitWith = (code) => {
expect(process.exit).to.be.calledWith(code)
}
// returns error object
this.expectExitWithErr = (type, msg1, msg2) => {
expect(errors.log, 'error was logged').to.be.calledWithMatch({ type })
expect(process.exit, 'process.exit was called').to.be.calledWith(1)
const err = errors.log.getCall(0).args[0]
if (msg1) {
expect(err.message, 'error text').to.include(msg1)
}
if (msg2) {
expect(err.message, 'second error text').to.include(msg2)
}
return err
}
})
afterEach(() => {
Fixtures.remove()
// make sure every project
// we spawn is closed down
try {
return openProject.close()
} catch (e) {
// ...
}
})
context('test browsers', () => {
// sanity checks to make sure the browser objects we pass during tests
// all pass the internal validation function
it('has valid browsers', () => {
expect(v.isValidBrowserList('browsers', TYPICAL_BROWSERS)).to.be.true
})
it('has valid electron browser', () => {
expect(v.isValidBrowserList('browsers', [ELECTRON_BROWSER])).to.be.true
})
it('allows browser major to be a number', () => {
const browser = {
name: 'Edge Beta',
family: 'chromium',
displayName: 'Edge Beta',
version: '80.0.328.2',
path: '/some/path',
majorVersion: 80,
}
expect(v.isValidBrowserList('browsers', [browser])).to.be.true
})
it('validates returned list', () => {
return browserUtils.getBrowsers().then((list) => {
expect(v.isValidBrowserList('browsers', list)).to.be.true
})
})
})
context('error handling', function () {
it('exits if config cannot be parsed', function () {
return cypress.start(['--config', 'xyz'])
.then(() => {
const err = this.expectExitWithErr('COULD_NOT_PARSE_ARGUMENTS')
snapshot('could not parse config error', stripAnsi(err.message))
})
})
it('exits if env cannot be parsed', function () {
return cypress.start(['--env', 'a123'])
.then(() => {
const err = this.expectExitWithErr('COULD_NOT_PARSE_ARGUMENTS')
snapshot('could not parse env error', stripAnsi(err.message))
})
})
it('exits if reporter options cannot be parsed', function () {
return cypress.start(['--reporterOptions', 'nonono'])
.then(() => {
const err = this.expectExitWithErr('COULD_NOT_PARSE_ARGUMENTS')
snapshot('could not parse reporter options error', stripAnsi(err.message))
})
})
})
context('invalid config', function () {
beforeEach(function () {
this.win = {
on: sinon.stub(),
webContents: {
on: sinon.stub(),
},
}
sinon.stub(electron.app, 'on').withArgs('ready').yieldsAsync()
sinon.stub(Windows, 'open').resolves(this.win)
})
it('shows warning if config is not valid', function () {
return cypress.start(['--config=test=false', '--cwd=/foo/bar'])
.then(() => {
expect(errors.warning).to.be.calledWith('INVALID_CONFIG_OPTION')
expect(console.log).to.be.calledWithMatch('`test` is not a valid configuration option')
expect(console.log).to.be.calledWithMatch('https://on.cypress.io/configuration')
})
})
it('shows warning when multiple config are not valid', function () {
return cypress.start(['--config=test=false,foo=bar', '--cwd=/foo/bar'])
.then(() => {
expect(errors.warning).to.be.calledWith('INVALID_CONFIG_OPTION')
expect(console.log).to.be.calledWithMatch('`test` is not a valid configuration option')
expect(console.log).to.be.calledWithMatch('`foo` is not a valid configuration option')
expect(console.log).to.be.calledWithMatch('https://on.cypress.io/configuration')
snapshotConsoleLogs('INVALID_CONFIG_OPTION')
})
})
it('does not show warning if config is valid', function () {
return cypress.start(['--config=trashAssetsBeforeRuns=false'])
.then(() => {
expect(errors.warning).to.not.be.calledWith('INVALID_CONFIG_OPTION')
})
})
})
context('--get-key', () => {
it('writes out key and exits on success', function () {
return Promise.all([
user.set({ name: 'brian', authToken: 'auth-token-123' }),
ProjectBase.id(this.todosPath)
.then((id) => {
this.projectId = id
}),
])
.then(() => {
sinon.stub(api, 'getProjectToken')
.withArgs(this.projectId, 'auth-token-123')
.resolves('new-key-123')
return cypress.start(['--get-key', `--project=${this.todosPath}`])
}).then(() => {
expect(console.log).to.be.calledWith('new-key-123')
this.expectExitWith(0)
})
})
it('logs error and exits when user isn\'t logged in', function () {
return user.set({})
.then(() => {
return cypress.start(['--get-key', `--project=${this.todosPath}`])
}).then(() => {
this.expectExitWithErr('NOT_LOGGED_IN')
})
})
it('logs error and exits when project does not have an id', function () {
return user.set({ authToken: 'auth-token-123' })
.then(() => {
return cypress.start(['--get-key', `--project=${this.pristinePath}`])
}).then(() => {
this.expectExitWithErr('NO_PROJECT_ID', this.pristinePath)
})
})
it('logs error and exits when project could not be found at the path', function () {
return user.set({ authToken: 'auth-token-123' })
.then(() => {
return cypress.start(['--get-key', '--project=path/to/no/project'])
}).then(() => {
this.expectExitWithErr('NO_PROJECT_FOUND_AT_PROJECT_ROOT', 'path/to/no/project')
})
})
it('logs error and exits when project token cannot be fetched', function () {
return Promise.all([
user.set({ authToken: 'auth-token-123' }),
ProjectBase.id(this.todosPath)
.then((id) => {
this.projectId = id
}),
])
.then(() => {
sinon.stub(api, 'getProjectToken')
.withArgs(this.projectId, 'auth-token-123')
.rejects(new Error())
return cypress.start(['--get-key', `--project=${this.todosPath}`])
}).then(() => {
this.expectExitWithErr('CANNOT_FETCH_PROJECT_TOKEN')
})
})
})
context('--new-key', () => {
it('writes out key and exits on success', function () {
return Promise.all([
user.set({ name: 'brian', authToken: 'auth-token-123' }),
ProjectBase.id(this.todosPath)
.then((id) => {
this.projectId = id
}),
])
.then(() => {
sinon.stub(api, 'updateProjectToken')
.withArgs(this.projectId, 'auth-token-123')
.resolves('new-key-123')
return cypress.start(['--new-key', `--project=${this.todosPath}`])
}).then(() => {
expect(console.log).to.be.calledWith('new-key-123')
this.expectExitWith(0)
})
})
it('logs error and exits when user isn\'t logged in', function () {
return user.set({})
.then(() => {
return cypress.start(['--new-key', `--project=${this.todosPath}`])
}).then(() => {
this.expectExitWithErr('NOT_LOGGED_IN')
})
})
it('logs error and exits when project does not have an id', function () {
return user.set({ authToken: 'auth-token-123' })
.then(() => {
return cypress.start(['--new-key', `--project=${this.pristinePath}`])
}).then(() => {
this.expectExitWithErr('NO_PROJECT_ID', this.pristinePath)
})
})
it('logs error and exits when project could not be found at the path', function () {
return user.set({ authToken: 'auth-token-123' })
.then(() => {
return cypress.start(['--new-key', '--project=path/to/no/project'])
}).then(() => {
this.expectExitWithErr('NO_PROJECT_FOUND_AT_PROJECT_ROOT', 'path/to/no/project')
})
})
it('logs error and exits when project token cannot be fetched', function () {
return Promise.all([
user.set({ authToken: 'auth-token-123' }),
ProjectBase.id(this.todosPath)
.then((id) => {
this.projectId = id
}),
])
.then(() => {
sinon.stub(api, 'updateProjectToken')
.withArgs(this.projectId, 'auth-token-123')
.rejects(new Error())
return cypress.start(['--new-key', `--project=${this.todosPath}`])
}).then(() => {
this.expectExitWithErr('CANNOT_CREATE_PROJECT_TOKEN')
})
})
})
context('--run-project', () => {
beforeEach(() => {
sinon.stub(electron.app, 'on').withArgs('ready').yieldsAsync()
sinon.stub(runMode, 'waitForSocketConnection').resolves()
sinon.stub(runMode, 'listenForProjectEnd').resolves({ stats: { failures: 0 } })
sinon.stub(browsers, 'open')
sinon.stub(commitInfo, 'getRemoteOrigin').resolves('remoteOrigin')
})
it('runs project headlessly and exits with exit code 0', function () {
return cypress.start([`--run-project=${this.todosPath}`])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER)
this.expectExitWith(0)
})
})
it('sets --headed false if --headless', function () {
sinon.spy(cypress, 'startInMode')
return cypress.start([`--run-project=${this.todosPath}`, '--headless'])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER)
this.expectExitWith(0)
// check how --headless option sets --headed
expect(cypress.startInMode).to.be.calledOnce
expect(cypress.startInMode).to.be.calledWith('run')
const startInModeOptions = cypress.startInMode.firstCall.args[1]
expect(startInModeOptions).to.include({
headless: true,
headed: false,
})
})
})
it('throws an error if both --headed and --headless are true', function () {
// error is thrown synchronously
expect(() => cypress.start([`--run-project=${this.todosPath}`, '--headless', '--headed']))
.to.throw('Impossible options: both headless and headed are true')
})
describe('strips --', () => {
beforeEach(() => {
sinon.spy(argsUtil, 'toObject')
})
it('strips leading', function () {
return cypress.start(['--', `--run-project=${this.todosPath}`])
.then(() => {
expect(argsUtil.toObject).to.have.been.calledWith([`--run-project=${this.todosPath}`])
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER)
this.expectExitWith(0)
})
})
it('strips in the middle', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--', '--browser=electron'])
.then(() => {
expect(argsUtil.toObject).to.have.been.calledWith([`--run-project=${this.todosPath}`, '--browser=electron'])
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER)
this.expectExitWith(0)
})
})
})
it('runs project headlessly and exits with exit code 10', function () {
sinon.stub(runMode, 'runSpecs').resolves({ totalFailed: 10 })
return cypress.start([`--run-project=${this.todosPath}`])
.then(() => {
this.expectExitWith(10)
})
})
it('does not generate a project id even if missing one', function () {
sinon.stub(api, 'createProject')
return user.set({ authToken: 'auth-token-123' })
.then(() => {
return cypress.start([`--run-project=${this.noScaffolding}`])
}).then(() => {
this.expectExitWith(0)
}).then(() => {
expect(api.createProject).not.to.be.called
return (new ProjectBase(this.noScaffolding)).getProjectId()
.then(() => {
throw new Error('should have caught error but did not')
}).catch((err) => {
expect(err.type).to.eq('NO_PROJECT_ID')
})
})
})
it('does not add project to the global cache', function () {
return cache.getProjectRoots()
.then((projects) => {
// no projects in the cache
expect(projects.length).to.eq(0)
return cypress.start([`--run-project=${this.todosPath}`])
}).then(() => {
return cache.getProjectRoots()
}).then((projects) => {
// still not projects
expect(projects.length).to.eq(0)
})
})
it('runs project by relative spec and exits with status 0', function () {
const relativePath = path.relative(cwd(), this.todosPath)
return cypress.start([
`--run-project=${this.todosPath}`,
`--spec=${relativePath}/tests/test2.coffee`,
])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, {
url: 'http://localhost:8888/__/#/tests/integration/test2.coffee',
})
this.expectExitWith(0)
})
})
it('runs project by specific spec with default configuration', function () {
return cypress.start([`--run-project=${this.idsPath}`, `--spec=${this.idsPath}/cypress/integration/bar.js`, '--config', 'port=2020'])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, { url: 'http://localhost:2020/__/#/tests/integration/bar.js' })
this.expectExitWith(0)
})
})
it('runs project by specific absolute spec and exits with status 0', function () {
return cypress.start([`--run-project=${this.todosPath}`, `--spec=${this.todosPath}/tests/test2.coffee`])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, { url: 'http://localhost:8888/__/#/tests/integration/test2.coffee' })
this.expectExitWith(0)
})
})
it('runs project by limiting spec files via config.testFiles string glob pattern', function () {
return cypress.start([`--run-project=${this.todosPath}`, `--config=testFiles=${this.todosPath}/tests/test2.coffee`])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, { url: 'http://localhost:8888/__/#/tests/integration/test2.coffee' })
this.expectExitWith(0)
})
})
it('runs project by limiting spec files via config.testFiles as a JSON array of string glob patterns', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--config=testFiles=["**/test2.coffee","**/test1.js"]'])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, { url: 'http://localhost:8888/__/#/tests/integration/test2.coffee' })
}).then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, { url: 'http://localhost:8888/__/#/tests/integration/test1.js' })
this.expectExitWith(0)
})
})
it('does not watch settings or plugins in run mode', function () {
const watch = sinon.spy(Watchers.prototype, 'watch')
const watchTree = sinon.spy(Watchers.prototype, 'watchTree')
return cypress.start([`--run-project=${this.pluginConfig}`])
.then(() => {
expect(watchTree).not.to.be.called
expect(watch).not.to.be.called
this.expectExitWith(0)
})
})
it('scaffolds out integration and example specs if they do not exist when not runMode', function () {
return config.get(this.pristinePath)
.then((cfg) => {
return fs.statAsync(cfg.integrationFolder)
.then(() => {
throw new Error('integrationFolder should not exist!')
}).catch(() => {
return cypress.start([`--run-project=${this.pristinePath}`, '--no-run-mode'])
}).then(() => {
return fs.statAsync(cfg.integrationFolder)
}).then(() => {
return Promise.join(
fs.statAsync(path.join(cfg.integrationFolder, 'examples', 'actions.spec.js')),
fs.statAsync(path.join(cfg.integrationFolder, 'examples', 'files.spec.js')),
fs.statAsync(path.join(cfg.integrationFolder, 'examples', 'viewport.spec.js')),
)
})
})
})
it('does not scaffold when headless and exits with error when no existing project', function () {
const ensureDoesNotExist = function (inspection, index) {
if (!inspection.isRejected()) {
throw new Error(`File or folder was scaffolded at index: ${index}`)
}
expect(inspection.reason()).to.have.property('code', 'ENOENT')
}
return Promise.all([
fs.statAsync(path.join(this.pristinePath, 'cypress')).reflect(),
fs.statAsync(path.join(this.pristinePath, 'cypress.json')).reflect(),
])
.each(ensureDoesNotExist)
.then(() => {
return cypress.start([`--run-project=${this.pristinePath}`])
}).then(() => {
return Promise.all([
fs.statAsync(path.join(this.pristinePath, 'cypress')).reflect(),
fs.statAsync(path.join(this.pristinePath, 'cypress.json')).reflect(),
])
}).each(ensureDoesNotExist)
.then(() => {
this.expectExitWithErr('CONFIG_FILE_NOT_FOUND', this.pristinePath)
})
})
it('does not scaffold integration or example specs when runMode', function () {
return settings.write(this.pristinePath, {})
.then(() => {
return cypress.start([`--run-project=${this.pristinePath}`])
}).then(() => {
return fs.statAsync(path.join(this.pristinePath, 'cypress', 'integration'))
}).then(() => {
throw new Error('integration folder should not exist!')
}).catch({ code: 'ENOENT' }, () => {})
})
it('scaffolds out fixtures + files if they do not exist', function () {
return config.get(this.pristinePath)
.then((cfg) => {
return fs.statAsync(cfg.fixturesFolder)
.then(() => {
throw new Error('fixturesFolder should not exist!')
}).catch(() => {
return cypress.start([`--run-project=${this.pristinePath}`, '--no-run-mode'])
}).then(() => {
return fs.statAsync(cfg.fixturesFolder)
}).then(() => {
return fs.statAsync(path.join(cfg.fixturesFolder, 'example.json'))
})
})
})
it('scaffolds out support + files if they do not exist', function () {
const supportFolder = path.join(this.pristinePath, 'cypress/support')
return config.get(this.pristinePath)
.then(() => {
return fs.statAsync(supportFolder)
.then(() => {
throw new Error('supportFolder should not exist!')
}).catch({ code: 'ENOENT' }, () => {
return cypress.start([`--run-project=${this.pristinePath}`, '--no-run-mode'])
}).then(() => {
return fs.statAsync(supportFolder)
}).then(() => {
return fs.statAsync(path.join(supportFolder, 'index.js'))
}).then(() => {
return fs.statAsync(path.join(supportFolder, 'commands.js'))
})
})
})
it('removes fixtures when they exist and fixturesFolder is false', function (done) {
config.get(this.idsPath)
.then((cfg) => {
this.cfg = cfg
return fs.statAsync(this.cfg.fixturesFolder)
}).then(() => {
return settings.read(this.idsPath)
}).then((json) => {
json.fixturesFolder = false
return settings.write(this.idsPath, json)
}).then(() => {
return cypress.start([`--run-project=${this.idsPath}`])
}).then(() => {
return fs.statAsync(this.cfg.fixturesFolder)
.then(() => {
throw new Error('fixturesFolder should not exist!')
}).catch(() => {
return done()
})
})
})
it('runs project headlessly and displays gui', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--headed'])
.then(() => {
expect(browsers.open).to.be.calledWithMatch(ELECTRON_BROWSER, {
proxyServer: 'http://localhost:8888',
show: true,
})
this.expectExitWith(0)
})
})
it('turns on reporting', function () {
sinon.spy(Reporter, 'create')
return cypress.start([`--run-project=${this.todosPath}`])
.then(() => {
expect(Reporter.create).to.be.calledWith('spec')
this.expectExitWith(0)
})
})
it('can change the reporter to nyan', function () {
sinon.spy(Reporter, 'create')
return cypress.start([`--run-project=${this.todosPath}`, '--reporter=nyan'])
.then(() => {
expect(Reporter.create).to.be.calledWith('nyan')
this.expectExitWith(0)
})
})
it('can change the reporter with cypress.json', function () {
sinon.spy(Reporter, 'create')
return config.get(this.idsPath)
.then((cfg) => {
this.cfg = cfg
return settings.read(this.idsPath)
}).then((json) => {
json.reporter = 'dot'
return settings.write(this.idsPath, json)
}).then(() => {
return cypress.start([`--run-project=${this.idsPath}`])
}).then(() => {
expect(Reporter.create).to.be.calledWith('dot')
this.expectExitWith(0)
})
})
it('runs tests even when user isn\'t logged in', function () {
return user.set({})
.then(() => {
return cypress.start([`--run-project=${this.todosPath}`])
}).then(() => {
this.expectExitWith(0)
})
})
it('logs warning when projectId and key but no record option', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--key=asdf'])
.then(() => {
expect(errors.warning).to.be.calledWith('PROJECT_ID_AND_KEY_BUT_MISSING_RECORD_OPTION', 'abc123')
expect(console.log).to.be.calledWithMatch('You also provided your Record Key, but you did not pass the --record flag.')
expect(console.log).to.be.calledWithMatch('cypress run --record')
expect(console.log).to.be.calledWithMatch('https://on.cypress.io/recording-project-runs')
})
})
it('logs warning when removing old browser profiles fails', function () {
const err = new Error('foo')
sinon.stub(browsers, 'removeOldProfiles').rejects(err)
return cypress.start([`--run-project=${this.todosPath}`])
.then(() => {
expect(errors.warning).to.be.calledWith('CANNOT_REMOVE_OLD_BROWSER_PROFILES', err.stack)
expect(console.log).to.be.calledWithMatch('Warning: We failed to remove old browser profiles from previous runs.')
expect(console.log).to.be.calledWithMatch(err.message)
})
})
it('does not log warning when no projectId', function () {
return cypress.start([`--run-project=${this.pristinePath}`, '--key=asdf'])
.then(() => {
expect(errors.warning).not.to.be.calledWith('PROJECT_ID_AND_KEY_BUT_MISSING_RECORD_OPTION', 'abc123')
expect(console.log).not.to.be.calledWithMatch('cypress run --key <record_key>')
})
})
it('does not log warning when projectId but --record false', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--key=asdf', '--record=false'])
.then(() => {
expect(errors.warning).not.to.be.calledWith('PROJECT_ID_AND_KEY_BUT_MISSING_RECORD_OPTION', 'abc123')
expect(console.log).not.to.be.calledWithMatch('cypress run --key <record_key>')
})
})
it('logs error when supportFile doesn\'t exist', function () {
return settings.write(this.idsPath, { supportFile: '/does/not/exist' })
.then(() => {
return cypress.start([`--run-project=${this.idsPath}`])
}).then(() => {
this.expectExitWithErr('SUPPORT_FILE_NOT_FOUND', 'Your `supportFile` is set to `/does/not/exist`,')
})
})
it('logs error when browser cannot be found', function () {
browsers.open.restore()
return cypress.start([`--run-project=${this.idsPath}`, '--browser=foo'])
.then(() => {
this.expectExitWithErr('BROWSER_NOT_FOUND_BY_NAME')
// get all the error args
const argsSet = errors.log.args
const found1 = _.find(argsSet, (args) => {
return _.find(args, (arg) => {
return arg.message && arg.message.includes(
'Browser: \'foo\' was not found on your system or is not supported by Cypress.',
)
})
})
expect(found1, 'foo should not be found').to.be.ok
const found2 = _.find(argsSet, (args) => {
return _.find(args, (arg) => {
return arg.message && arg.message.includes(
'Cypress supports the following browsers:',
)
})
})
expect(found2, 'supported browsers should be listed').to.be.ok
const found3 = _.find(argsSet, (args) => {
return _.find(args, (arg) => {
return arg.message && arg.message.includes(
'Available browsers found on your system are:\n- chrome\n- chromium\n- chrome:canary\n- electron',
)
})
})
expect(found3, 'browser names should be listed').to.be.ok
})
})
describe('no specs found', function () {
it('logs error and exits when spec file was specified and does not exist', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--spec=path/to/spec'])
.then(() => {
// includes the search spec
this.expectExitWithErr('NO_SPECS_FOUND', 'path/to/spec')
this.expectExitWithErr('NO_SPECS_FOUND', 'We searched for any files matching this glob pattern:')
// includes the project path
this.expectExitWithErr('NO_SPECS_FOUND', this.todosPath)
})
})
it('logs error and exits when spec absolute file was specified and does not exist', function () {
return cypress.start([
`--run-project=${this.todosPath}`,
`--spec=${this.todosPath}/tests/path/to/spec`,
])
.then(() => {
// includes path to the spec
this.expectExitWithErr('NO_SPECS_FOUND', 'tests/path/to/spec')
// includes folder name
this.expectExitWithErr('NO_SPECS_FOUND', this.todosPath)
})
})
it('logs error and exits when no specs were found at all', function () {
return cypress.start([
`--run-project=${this.todosPath}`,
'--config=integrationFolder=cypress/specs',
])
.then(() => {
this.expectExitWithErr('NO_SPECS_FOUND', 'We searched for any files inside of this folder:')
this.expectExitWithErr('NO_SPECS_FOUND', 'cypress/specs')
})
})
})
it('logs error and exits when project has cypress.json syntax error', function () {
return fs.writeFileAsync(`${this.todosPath}/cypress.json`, '{\'foo\': \'bar}')
.then(() => {
return cypress.start([`--run-project=${this.todosPath}`])
}).then(() => {
this.expectExitWithErr('ERROR_READING_FILE', this.todosPath)
})
})
it('logs error and exits when project has cypress.env.json syntax error', function () {
return fs.writeFileAsync(`${this.todosPath}/cypress.env.json`, '{\'foo\': \'bar}')
.then(() => {
return cypress.start([`--run-project=${this.todosPath}`])
}).then(() => {
this.expectExitWithErr('ERROR_READING_FILE', this.todosPath)
})
})
it('logs error and exits when project has invalid cypress.json values', function () {
return settings.write(this.todosPath, { baseUrl: 'localhost:9999' })
.then(() => {
return cypress.start([`--run-project=${this.todosPath}`])
}).then(() => {
this.expectExitWithErr('SETTINGS_VALIDATION_ERROR', 'cypress.json')
})
})
it('logs error and exits when project has invalid config values from the CLI', function () {
return cypress.start([
`--run-project=${this.todosPath}`,
'--config=baseUrl=localhost:9999',
])
.then(() => {
this.expectExitWithErr('CONFIG_VALIDATION_ERROR', 'localhost:9999')
this.expectExitWithErr('CONFIG_VALIDATION_ERROR', 'We found an invalid configuration value')
})
})
it('logs error and exits when project has invalid config values from env vars', function () {
process.env.CYPRESS_BASE_URL = 'localhost:9999'
return cypress.start([`--run-project=${this.todosPath}`])
.then(() => {
this.expectExitWithErr('CONFIG_VALIDATION_ERROR', 'localhost:9999')
this.expectExitWithErr('CONFIG_VALIDATION_ERROR', 'We found an invalid configuration value')
})
})
const renamedConfigs = [
{
old: 'blacklistHosts',
new: 'blockHosts',
},
]
renamedConfigs.forEach(function (config) {
it(`logs error and exits when using an old configuration option: ${config.old}`, function () {
return cypress.start([
`--run-project=${this.todosPath}`,
`--config=${config.old}=''`,
])
.then(() => {
this.expectExitWithErr('RENAMED_CONFIG_OPTION', config.old)
this.expectExitWithErr('RENAMED_CONFIG_OPTION', config.new)
})
})
})
// TODO: make sure we have integration tests around this
// for headed projects!
// also make sure we test the rest of the integration functionality
// for headed errors! <-- not unit tests, but integration tests!
it('logs error and exits when project folder has read permissions only and cannot write cypress.json', function () {
// test disabled if running as root - root can write all things at all times
if (process.geteuid() === 0) {
return
}
const permissionsPath = path.resolve('./permissions')
const cypressJson = path.join(permissionsPath, 'cypress.json')
return fs.outputFileAsync(cypressJson, '{}')
.then(() => {
// read only
return fs.chmodAsync(permissionsPath, '555')
}).then(() => {
return cypress.start([`--run-project=${permissionsPath}`])
}).then(() => {
return fs.chmodAsync(permissionsPath, '777')
}).then(() => {
return fs.removeAsync(permissionsPath)
}).then(() => {
this.expectExitWithErr('ERROR_READING_FILE', path.join(permissionsPath, 'cypress.json'))
})
})
it('logs error and exits when reporter does not exist', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--reporter', 'foobarbaz'])
.then(() => {
this.expectExitWithErr('INVALID_REPORTER_NAME', 'foobarbaz')
})
})
describe('state', () => {
beforeEach(function () {
return appData.remove()
.then(() => {
return savedState.formStatePath(this.todosPath)
}).then((statePathStart) => {
this.statePath = appData.projectsPath(statePathStart)
})
})
it('does not save project state', function () {
return cypress.start([`--run-project=${this.todosPath}`, `--spec=${this.todosPath}/tests/test2.coffee`])
.then(() => {
this.expectExitWith(0)
// this should not save the project's state
// because its a noop in 'cypress run' mode
return openProject.getProject().saveState()
}).then(() => {
return fs.statAsync(this.statePath)
.then(() => {
throw new Error(`saved state should not exist but it did here: ${this.statePath}`)
}).catch({ code: 'ENOENT' }, () => {})
})
})
})
describe('morgan', () => {
it('sets morgan to false', function () {
return cypress.start([`--run-project=${this.todosPath}`])
.then(() => {
expect(openProject.getProject().cfg.morgan).to.be.false
this.expectExitWith(0)
})
})
})
describe('config overrides', () => {
it('can override default values', function () {
return cypress.start([`--run-project=${this.todosPath}`, '--config=requestTimeout=1234,videoCompression=false'])
.then(() => {
const { cfg } = openProject.getProject()
expect(cfg.videoCompression).to.be.false
expect(cfg.requestTimeout).to.eq(1234)
expect(cfg.resolved.videoCompression).to.deep.eq({
value: false,
from: 'cli',
})
expect(cfg.resolved.requestTimeout).to.deep.eq({
value: 1234,
from: 'cli',
})
this.expectExitWith(0)
})
})
it('can override values in plugins', function () {
plugins.init.restore()
return cypress.start([
`--run-project=${this.pluginConfig}`, '--config=requestTimeout=1234,videoCompression=false',
'--env=foo=foo,bar=bar',
])
.then(() => {
const { cfg } = openProject.getProject()
expect(cfg.videoCompression).to.eq(20)
expect(cfg.defaultCommandTimeout).to.eq(500)
expect(cfg.env).to.deep.eq({
foo: 'bar',
bar: 'bar',
})
expect(cfg.resolved.videoCompression).to.deep.eq({
value: 20,
from: 'plugin',
})
expect(cfg.resolved.requestTimeout).to.deep.eq({
value: 1234,
from: 'cli',
})
expect(cfg.resolved.env.foo).to.deep.eq({
value: 'bar',
from: 'plugin',
})
expect(cfg.resolved.env.bar).to.deep.eq({
value: 'bar',
from: 'cli',
})
this.expectExitWith(0)
})
})
})
describe('plugins', () => {
beforeEach(() => {
plugins.init.restore()
browsers.open.restore()
const ee = new EE()
ee.kill = () => {
// ughh, would be nice to test logic inside the launcher
// that cleans up after the browser exit
// like calling client.close() if available to let the
// browser free any resources
return ee.emit('exit')
}
ee.destroy = () => {
return ee.emit('closed')
}
ee.isDestroyed = () => {
return false
}
ee.loadURL = () => {}
ee.focusOnWebView = () => {}
ee.webContents = {
debugger: {
on: sinon.stub(),
attach: sinon.stub(),
sendCommand: sinon.stub().resolves(),
},
getOSProcessId: sinon.stub(),
setUserAgent: sinon.stub(),
session: {
clearCache: sinon.stub().resolves(),
setProxy: sinon.stub().resolves(),
setUserAgent: sinon.stub(),
on: sinon.stub(),
},
}
sinon.stub(browserUtils, 'launch').resolves(ee)
sinon.stub(Windows, 'create').returns(ee)
})
context('before:browser:launch', () => {
it('chrome', function () {
// during testing, do not try to connect to the remote interface or
// use the Chrome remote interface client
const criClient = {
ensureMinimumProtocolVersion: sinon.stub().resolves(),
close: sinon.stub().resolves(),
on: sinon.stub(),
send: sinon.stub(),
}
sinon.stub(chromeBrowser, '_connectToChromeRemoteInterface').resolves(criClient)
// the "returns(resolves)" stub is due to curried method
// it accepts URL to visit and then waits for actual CRI client reference
// and only then navigates to that URL
sinon.stub(chromeBrowser, '_navigateUsingCRI').resolves()
sinon.stub(chromeBrowser, '_handleDownloads').resolves()
sinon.stub(chromeBrowser, '_setAutomation').returns()
return cypress.start([
`--run-project=${this.pluginBrowser}`,
'--browser=chrome',
])
.then(() => {
const { args } = browserUtils.launch.firstCall
// when we work with the browsers we set a few extra flags
const chrome = _.find(TYPICAL_BROWSERS, { name: 'chrome' })
const launchedChrome = R.merge(chrome, {
isHeadless: false,
isHeaded: true,
})
expect(args[0], 'found and used Chrome').to.deep.eq(launchedChrome)
const browserArgs = args[2]
expect(browserArgs.slice(0, 4), 'first 4 custom launch arguments to Chrome').to.deep.eq([
'chrome', 'foo', 'bar', 'baz',
])
this.expectExitWith(0)
expect(chromeBrowser._navigateUsingCRI).to.have.been.calledOnce
expect(chromeBrowser._setAutomation).to.have.been.calledOnce
expect(chromeBrowser._connectToChromeRemoteInterface).to.have.been.calledOnce
})
})
it('electron', function () {
const writeVideoFrame = sinon.stub()
videoCapture.start.returns({ writeVideoFrame })
return cypress.start([
`--run-project=${this.pluginBrowser}`,
'--browser=electron',
])
.then(() => {
expect(Windows.create).to.be.calledWithMatch(this.pluginBrowser, {
browser: 'electron',
foo: 'bar',
onNewWindow: sinon.match.func,
onScreencastFrame: sinon.match.func,
})
this.expectExitWith(0)
})
})
})
})
describe('--port', () => {
beforeEach(() => {
return runMode.listenForProjectEnd.resolves({ stats: { failures: 0 } })
})
it('can change the default port to 5544', function () {
const listen = sinon.spy(http.Server.prototype, 'listen')
const open = sinon.spy(ServerE2E.prototype, 'open')
return cypress.start([`--run-project=${this.todosPath}`, '--port=5544'])
.then(() => {
expect(openProject.getProject().cfg.port).to.eq(5544)
expect(listen).to.be.calledWith(5544)
expect(open).to.be.calledWithMatch({ port: 5544 })
this.expectExitWith(0)
})
})
// TODO: handle PORT_IN_USE short integration test
it('logs error and exits when port is in use', function () {
let server = http.createServer()
server = Promise.promisifyAll(server)
return server.listenAsync(5544, '127.0.0.1')
.then(() => {
return cypress.start([`--run-project=${this.todosPath}`, '--port=5544'])
}).then(() => {
this.expectExitWithErr('PORT_IN_USE_LONG', '5544')
})
})
})
describe('--env', () => {
beforeEach(() => {
process.env = _.omit(process.env, 'CYPRESS_DEBUG')
return runMode.listenForProjectEnd.resolves({ stats: { failures: 0 } })
})
it('can set specific environment variables', function () {
return cypress.start([
`--run-project=${this.todosPath}`,
'--video=false',
'--env',
'version=0.12.1,foo=bar,host=http://localhost:8888,baz=quux=dolor',
])
.then(() => {
expect(openProject.getProject().cfg.env).to.deep.eq({
version: '0.12.1',
foo: 'bar',
host: 'http://localhost:8888',
baz: 'quux=dolor',
})
this.expectExitWith(0)
})
})
it('parses environment variables with empty values', function () {
return cypress.start([
`--run-project=${this.todosPath}`,
'--video=false',
'--env=FOO=,BAR=,BAZ=ipsum',
])
.then(() => {
expect(openProject.getProject().cfg.env).to.deep.eq({
FOO: '',
BAR: '',
BAZ: 'ipsum',
})
this.expectExitWith(0)
})
})
})
describe('--config-file', () => {
it('false does not require cypress.json to run', function () {
return fs.statAsync(path.join(this.pristinePath, 'cypress.json'))
.then(() => {
throw new Error('cypress.json should not exist')
}).catch({ code: 'ENOENT' }, () => {
return cypress.start([
`--run-project=${this.pristinePath}`,
'--no-run-mode',
'--config-file',
'false',
]).then(() => {
this.expectExitWith(0)
})
})
})
it('with a custom config file fails when it doesn\'t exist', function () {
this.filename = 'abcdefgh.test.json'
return fs.statAsync(path.join(this.todosPath, this.filename))
.then(() => {
throw new Error(`${this.filename} should not exist`)
}).catch({ code: 'ENOENT' }, () => {
return cypress.start([
`--run-project=${this.todosPath}`,
'--no-run-mode',
'--config-file',
this.filename,
]).then(() => {
this.expectExitWithErr('CONFIG_FILE_NOT_FOUND', this.filename, this.todosPath)
})
})
})
})
})
// most record mode logic is covered in e2e tests.
// we only need to cover the edge cases / warnings
context('--record', () => {
beforeEach(function () {
sinon.stub(api, 'createRun').resolves()
sinon.stub(electron.app, 'on').withArgs('ready').yieldsAsync()
sinon.stub(browsers, 'open')
sinon.stub(runMode, 'waitForSocketConnection').resolves()
sinon.stub(runMode, 'waitForTestsToFinishRunning').resolves({
stats: {
tests: 1,
passes: 2,
failures: 3,
pending: 4,
skipped: 5,
wallClockDuration: 6,
},
tests: [],
hooks: [],
video: 'path/to/video',
shouldUploadVideo: true,
screenshots: [],
config: {},
spec: {},
})
return Promise.all([
// make sure we have no user object
user.set({}),
ProjectBase.id(this.todosPath)
.then((id) => {
this.projectId = id
}),
])
})
it('uses process.env.CYPRESS_PROJECT_ID', function () {
sinon.stub(env, 'get').withArgs('CYPRESS_PROJECT_ID').returns(this.projectId)
return cypress.start([
'--cwd=/foo/bar',
`--run-project=${this.noScaffolding}`,
'--record',
'--key=token-123',
])
.then(() => {
expect(api.createRun).to.be.calledWithMatch({ projectId: this.projectId })
expect(errors.warning).not.to.be.called
this.expectExitWith(3)
})
})
it('uses process.env.CYPRESS_RECORD_KEY', function () {
sinon.stub(env, 'get')
.withArgs('CYPRESS_PROJECT_ID').returns('foo-project-123')
.withArgs('CYPRESS_RECORD_KEY').returns('token')
return cypress.start([
'--cwd=/foo/bar',
`--run-project=${this.noScaffolding}`,
'--record',
])
.then(() => {
expect(api.createRun).to.be.calledWithMatch({
projectId: 'foo-project-123',
recordKey: 'token',
})
expect(errors.warning).not.to.be.called
this.expectExitWith(3)
})
})
it('errors and exits when using --group but ciBuildId could not be generated', function () {
sinon.stub(ciProvider, 'provider').returns(null)
return cypress.start([
`--run-project=${this.recordPath}`,
'--record',
'--key=token-123',
'--group=e2e-tests',
])
.then(() => {
this.expectExitWithErr('INDETERMINATE_CI_BUILD_ID')
return snapshotConsoleLogs('INDETERMINATE_CI_BUILD_ID-group 1')
})
})
it('errors and exits when using --parallel but ciBuildId could not be generated', function () {
sinon.stub(ciProvider, 'provider').returns(null)
return cypress.start([
`--run-project=${this.recordPath}`,
'--record',
'--key=token-123',
'--parallel',
])
.then(() => {
this.expectExitWithErr('INDETERMINATE_CI_BUILD_ID')
return snapshotConsoleLogs('INDETERMINATE_CI_BUILD_ID-parallel 1')
})
})
it('errors and exits when using --parallel and --group but ciBuildId could not be generated', function () {
sinon.stub(ciProvider, 'provider').returns(null)
return cypress.start([
`--run-project=${this.recordPath}`,
'--record',
'--key=token-123',
'--group=e2e-tests-chrome',
'--parallel',
])
.then(() => {
this.expectExitWithErr('INDETERMINATE_CI_BUILD_ID')
return snapshotConsoleLogs('INDETERMINATE_CI_BUILD_ID-parallel-group 1')
})
})
it('errors and exits when using --ci-build-id with no group or parallelization', function () {
return cypress.start([
`--run-project=${this.recordPath}`,
'--record',
'--key=token-123',
'--ci-build