backtrace-morgue
Version:
command line interface to the Backtrace object store
1,819 lines (1,534 loc) • 213 kB
JavaScript
#!/usr/bin/env node
'use strict';
// setup abort controller
require("../lib/abortController");
const axios = require('axios');
const Callstack = require('../lib/callstack.js');
const CoronerClient = require('../lib/coroner.js');
const crdb = require('../lib/crdb.js');
const BPG = require('../lib/bpg.js');
const minimist = require('minimist');
const os = require('os');
const ip = require('ip');
const ipv6 = require('ip6addr');
const bar = require('./bar.js');
const ta = require('time-ago');
const histogram = require('./histogram.js');
const printf = require('printf');
const moment = require('moment');
const moment_tz = require('moment-timezone');
const fs = require('fs');
const mkdirp = require('mkdirp');
const promptLib = require('prompt');
const path = require('path');
const table = require('table').table;
const bt = require('@backtrace/node');
const spawn = require('child_process').spawn;
const url = require('url');
const util = require('util');
const packageJson = require(path.join(__dirname, "..", "package.json"));
const sprintf = require('extsprintf').sprintf;
const chrono = require('chrono-node');
const zlib = require('zlib');
const symbold = require('../lib/symbold.js');
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
const Slack = require('slack-node');
const metricsImporterCli = require('../lib/metricsImporter/cli.js');
const alertsCli = require("../lib/alerts/cli");
const timeCli = require('../lib/cli/time');
const queryCli = require('../lib/cli/query');
const { chalk, err, error_color, errx, success_color, warn } = require('../lib/cli/errors');
const WorkflowsCli = require('../lib/workflows/cli.js');
const WorkflowsClient = require('../lib/workflows/client.js');
const bold = chalk.bold;
const cyan = chalk.cyan;
const grey = chalk.grey;
const yellow = chalk.yellow;
const blue = chalk.blue;
const green = chalk.green;
const red = chalk.red;
const label_color = yellow.bold;
var flamegraph = path.join(__dirname, "..", "assets", "flamegraph.pl");
var callstackError = false;
var range_start = null;
var range_stop = null;
var endpoint;
var endpointToken;
var reverse = 1;
var ARGV;
const configDir = process.env.MORGUE_CONFIG_DIR ||
path.join(os.homedir(), ".morgue");
const configFile = path.join(configDir, "current.json");
const BACKTRACE_ROLES = ['admin', 'member', 'guest']
const backtraceDatabaseDirectory = path.join(configDir, "backtrace");
const client = bt.BacktraceClient.initialize({
url: "https://submit.backtrace.io/backtrace/2cfca2efffd862c7ad7188be8db09d8697bd098a3561cd80a56fe5c4819f5d14/json",
timeout: 1500,
userAttributes: {
version: packageJson.version,
},
database: {
enable: true,
path: backtraceDatabaseDirectory,
autoSend: false,
captureNativeCrashes: true,
createDatabaseDirectory: true
},
metrics: {
enable: false,
},
});
function usage(str) {
if (typeof str === 'string')
err(str + "\n");
console.error("Usage: morgue <command> [options]");
console.error("");
console.error("Options:");
console.error(" -v, --version Print version number and exit");
console.error(" --debug Enable verbose debug printing");
console.error(" -k Disable SSL verification with CA");
console.error(" --timeout ms Set the timeout on API requests in milliseconds");
console.error("");
console.error("Documentation is available at:");
console.error("https://github.com/backtrace-labs/backtrace-morgue#readme");
process.exit(1);
}
function nsToUs(tm) {
return Math.round((tm[0] * 1000000) + (tm[1] / 1000));
}
function oidToString(oid) {
return oid.toString(16);
}
function oidFromString(oid) {
return parseInt(oid, 16);
}
/* Standardized success/failure callbacks. */
function std_success_cb(r) {
console.log(success_color('Success'));
}
function std_json_cb(r) {
console.log(success_color('Success:'));
console.log(JSON.stringify(r, null, 4));
}
function std_failure_cb(e) {
var msg = e.toString();
if (e.response_obj && e.response_obj.bodyData) {
try {
var je = JSON.parse(e.response_obj.bodyData);
if (je && je.error && je.error.message) {
msg = je.error.message;
}
} catch (ex) {
if (e.response_obj.debug) {
console.log('Response:\n', e.response_obj.bodyData);
}
console.log('ex = ', ex);
}
}
errx(msg);
}
function objToPath(oid, resource) {
var str = oid;
if (typeof oid !== 'string')
str = oidToString(oid);
if (resource)
str += ":" + resource;
else
str += ".bin";
return str;
}
function printSamples(requests, samples, start, stop, concurrency) {
var i;
var sum = 0;
var minimum, maximum, tps;
start = nsToUs(start);
stop = nsToUs(stop);
for (i = 0; i < samples.length; i++) {
var value = parseInt(samples[i]);
sum += value;
if (!maximum || value > maximum)
maximum = value;
if (!minimum || value < minimum)
minimum = value;
}
sum = Math.ceil(sum / samples.length);
tps = Math.floor(requests / ((stop - start) / 1000000));
process.stdout.write(grey(sprintf("# %12s %12s %12s %12s %12s %12s %12s\n",
"Concurrency", "Requests", "Time", "Minimum", "Average",
"Maximum", "Throughput")));
process.stdout.write(printf(" %12d %12ld %12f %12ld %12ld %12ld %12ld\n",
concurrency, requests, (stop - start) / 1000000,
minimum, sum, maximum, tps));
return;
}
function sequence(tasks) {
return tasks.reduce((chain, s) => {
if (typeof s === 'function')
return chain.then(s).catch((e) => { return Promise.reject(e) });
return chain.then(() => { return s; }).catch((e) => { return Promise.reject(e) });
}, Promise.resolve());
}
function prompt_for(items) {
return new Promise((resolve, reject) => {
promptLib.get(items, (err, result) => {
if (err)
reject(err);
else
resolve(result);
});
});
}
var commands = {
actions: coronerActions,
attachment: coronerAttachment,
attribute: coronerAttribute,
view: coronerView,
audit: coronerAudit,
project: coronerProject,
projects: coronerProjects,
log: coronerLog,
bpg: coronerBpg,
error: coronerError,
list: coronerList,
callstack: coronerCallstack,
deduplication: coronerDeduplication,
clean: coronerClean,
report: coronerReport,
latency: coronerLatency,
tenant: coronerTenant,
similarity: coronerSimilarity,
flamegraph: coronerFlamegraph,
control: coronerControl,
invite: coronerInvite,
ls: coronerList,
describe: coronerDescribe,
cts: coronerCts,
ci: coronerCI,
token: coronerToken,
session: coronerSession,
access: coronerAccessControl,
limit: coronerLimit,
set: coronerSet,
get: coronerGet,
put: coronerPut,
login: coronerLogin,
logout: coronerLogout,
nuke: coronerNuke,
delete: coronerDelete,
repair: coronerRepair,
reprocess: coronerReprocess,
retention: coronerRetention,
sampling: coronerSampling,
service: coronerService,
symbol: coronerSymbol,
symbold: symboldClient,
scrubber: coronerScrubber,
setup: coronerSetup,
status: coronerStatus,
user: coronerUser,
users: coronerUsers,
merge: mergeFingerprints,
unmerge: unmergeFingerprints,
"metrics-importer": metricsImporterCmd,
stability: coronerStability,
alerts: alertsCmd,
workflows: workflowsCmd,
};
process.stdout.on('error', function(){process.exit(0);});
process.stderr.on('error', function(){process.exit(0);});
main();
function coronerError(argv, config) {
if (argv._.length < 2) {
errx("Missing error string");
}
throw Error(argv._[1]);
}
function coronerProject(argv, config) {
abortIfNotLoggedIn(config);
let subcommand = argv._[1];
var coroner = coronerClientArgv(config, argv);
var bpg = coronerBpgSetup(coroner, argv);
if(!subcommand) {
errx("Invalid project command. Try 'morgue project create <your-project-name>'")
}
if(subcommand !== 'create') {
errx("Invalid project command. Try 'morgue project create <your-project-name>'")
}
let project = argv._[2];
if (!project) {
errx("Missing project name");
}
if(!config || !config.config) {
errx("Invalid config");
}
let validationRe = /^[a-zA-Z0-9-]+$/;
let validProjName = validationRe.test(project);
if(!validProjName) {
errx("Illegal name only use a-z, A-Z, 0-9, or \"-\"");
}
if(!config.config.user || !config.config.user.uid) {
errx("Invalid user");
}
let user = config.config.user.uid;
if(!config.config.universe || !config.config.universe.id) {
errx("Invalid universe")
}
let universe = config.config.universe.id;
const request = bpgSingleRequest({
action: "create",
type: "configuration/project",
object: {
pid: 0,
deleted: 0,
name: project,
owner: user,
universe: universe,
},
});
bpgPost(bpg, request, bpgCbFn('Project', 'create'));
}
async function coronerProjects(argv, config) {
abortIfNotLoggedIn(config);
let subcommand = argv._[1];
var coroner = coronerClientArgv(config, argv);
var bpg = coronerBpgSetup(coroner, argv);
if(subcommand !== 'list') {
errx("Invalid projects command. Try 'morgue project list'")
}
if(!config || !config.config) {
errx("Invalid config");
}
if(!config.config.universe || !config.config.universe.id) {
errx("Invalid universe")
}
const projects = await bpgPostAsync(bpg, bpgSingleRequest({
action: "get",
type: "configuration/project",
}));
const users = await bpgPostAsync(bpg, bpgSingleRequest({
action: "get",
type: "configuration/users",
}));
const userIdMap = {}
users.forEach((u) => userIdMap[u.uid] = u )
console.log('Projects:')
console.log(projects.map((p, i) => `${i+1}: name=${p.name}, owner=${userIdMap[p.owner]?.username}`).join('\n'))
}
/**
* @brief Returns the universe/project pair to use for coroner commands.
*/
function coronerParams(argv, config) {
var p = {};
if (Array.isArray(argv._) === true && argv._.length > 1) {
var split;
split = argv._[1].split('/');
if (split.length === 1) {
var first;
/* Try to automatically derive a path from the one argument. */
for (first in config.config.universes) break;
p.universe = first;
p.project = argv._[1];
} else {
p.universe = split[0];
p.project = split[1];
}
}
if (argv.token) {
p.token = argv.token;
} else if (argv["api-token"]) {
/* argv.token is used for other things as well. */
p.token = argv["api-token"];
}
return p;
}
function saveConfig(coroner, callback) {
makeConfigDir(function(err) {
if (err) return callback(err);
var config = {
config: coroner.config,
endpoint: coroner.endpoint,
};
if (Array.isArray(coroner.config.endpoints.post)) {
var ep = coroner.config.endpoints.post;
var fu = url.parse(coroner.endpoint);
var i = Math.max(0, ep.findIndex(ep => ep.protocol === "https"));
config.submissionEndpoint = ep[i].protocol + '://' +
fu.hostname + ':' + ep[i].port + '/post';
}
var text = JSON.stringify(config, null, 2);
fs.writeFile(configFile, text, callback);
});
}
function loadConfig(callback) {
makeConfigDir(function(err) {
if (err) return callback(err);
fs.readFile(configFile, {encoding: 'utf8'}, function(err, text) {
var json;
if (text && text.length > 0) {
try {
json = JSON.parse(text);
} catch (err) {
return callback(new Error(err.message));
}
} else {
json = {};
}
callback(null, json);
});
});
}
function makeConfigDir(callback) {
mkdirp(configDir, {mode: "0700"}, callback);
}
function abortIfNotLoggedIn(config) {
if (config && config.config && config.config.token) return;
/* If an endpoint is specified, then synthensize aa configuration structure. */
if (endpoint) {
config.config = {};
/* We rely on host-based authentication if no token is specified. */
config.config.token = endpointToken;
if (!config.config.token)
config.config.token = '00000';
config.endpoint = endpoint;
return;
}
errx("Must login first.");
}
async function coronerSetupNext(coroner, bpg, setupCfg) {
var model = bpg.get();
process.stderr.write('\n');
/* Do this one first so superuser isn't set up before this is. */
const cons_l = model.listener.find((l) => {
return l.get('type') === 'http/console';
});
if (cons_l) {
const dns_name = cons_l.get('dns_name');
if (!dns_name || dns_name.length === 0)
return coronerSetupDns(coroner, bpg, cons_l, setupCfg);
}
if (!model.universe || model.universe.length === 0)
return coronerSetupUniverse(coroner, bpg, setupCfg);
if (!model.users || model.users.length === 0)
return coronerSetupUser(coroner, bpg, setupCfg);
process.stderr.write(
'Please use a web browser to complete setup:\n');
process.stderr.write(cyan.bold(coroner.endpoint + '/config/' + model.universe[0].get('name') + '\n'));
return;
}
async function coronerSetupDns(coroner, bpg, cons_l, setupCfg) {
let dns_name = setupCfg?.dns_name;
if (!dns_name) {
console.log(bold('Specify DNS name users will use to reach the server'));
console.log(
'We must specify this so that services accessing the server via SSL\n' +
'can reach it without skipping validation.\n');
const result = await promptLib.get([
{
name: 'dns_name',
description: 'DNS name',
pattern: /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/,
type: 'string',
required: true,
}]);
dns_name = result.dns_name;
}
var model = bpg.get();
bpg.modify(cons_l, { dns_name });
bpg.commit();
return coronerSetupNext(coroner, bpg, setupCfg);
}
async function coronerSetupUser(coroner, bpg, setupCfg) {
let username = setupCfg?.username;
let email = setupCfg?.email;
let password = setupCfg?.password;
// Simplify a bit by requiring all inputs even if only one is missing. Caller
// using the JSON file is expected to provide all inputs ahead of time.
if (!username) {
console.log(bold('Create an administrator'));
console.log(
'We must create an administrator user. This user will be used to configure\n' +
'the server as well as perform system-wide administrative tasks.\n');
const result = await promptLib.get([
{
name: 'username',
description: 'Username',
pattern: /^[a-z0-9\_]+$/,
type: 'string',
required: true
},
{
name: 'email',
description: 'E-mail address',
required: true,
type: 'string',
pattern: /^([a-zA-Z0-9_\-\.]+)@([a-zA-Z0-9_\-\.]+)\.([a-zA-Z]{2,5})$/
},
{
name: 'password',
description: 'Password',
required: true,
hidden: true,
replace: '*',
type: 'string'
},
{
name: 'passwordConfirm',
description: 'Confirm password',
required: true,
hidden: true,
replace: '*',
type: 'string'
}]);
if (result.password !== result.passwordConfirm) {
errx('Passwords do not match.');
}
username = result.username;
email = result.email;
password = result.password;
}
var model = bpg.get();
var user = bpg.new('users');
user.set('uid', 0);
user.set('superuser', 1);
user.set('method', 'password');
user.set('universe', model.universe[0].get('id'));
user.set('username', username);
user.set('email', email);
user.set('password', BPG.blobText(password));
bpg.create(user);
bpg.commit();
return coronerSetupNext(coroner, bpg, setupCfg);
}
async function coronerSetupUniverse(coroner, bpg, setupCfg) {
let universe_name = setupCfg?.universe;
if (!universe_name) {
console.log(bold('Create an organization'));
console.log(
'We must configure the organization that is using the object store.\n' +
'Please provide a one word name for the organization using the object store.\n' +
'For example, if your company name is "Appleseed Systems I/O", you could\n' +
'use the name "appleseed". The name must be lowercase.\n');
const result = await promptLib.get([{
name: 'universe',
description: 'Organization name',
message: 'Must be lowercase and only contains letters.',
type: 'string',
pattern: /^[a-z0-9]+$/,
required: true
}]);
universe_name = result.universe;
}
var universe = bpg.new('universe');
universe.set('id', 0);
universe.set('name', universe_name);
bpg.create(universe);
bpg.commit();
return coronerSetupNext(coroner, bpg, setupCfg);
}
function coronerBpgSetup(coroner, argv) {
var coronerd = {
url: coroner.endpoint,
session: { token: '000000000' }
};
var opts = {};
var bpg = {};
if (coroner.config && coroner.config.token)
coronerd.session.token = coroner.config.token;
if (argv.debug)
opts.debug = true;
bpg = new BPG.BPG(coronerd, opts);
return bpg;
}
function coronerClient(config, insecure, debug, endpoint, timeout) {
return new CoronerClient({
insecure: insecure,
debug: debug,
endpoint: endpoint,
timeout: timeout,
config: config.config,
});
}
function coronerClientArgv(config, argv) {
if (argv.token && argv.endpoint) {
config.config.token = argv.token;
config.endpoint = argv.endpoint;
}
return coronerClient(
config,
!!argv.k,
!!argv.debug,
config.endpoint,
argv.timeout
);
}
function coronerClientArgvSubmit(config, argv) {
return coronerClient(config, !!argv.k, argv.debug,
config.submissionEndpoint, argv.timeout);
}
function coronerSetupStart(coroner, argv) {
var bpg = coronerBpgSetup(coroner, argv);
let setupCfg;
if (argv.setup_json && fs.existsSync(argv.setup_json)) {
setupCfg = JSON.parse(fs.readFileSync(argv.setup_json, 'utf8'));
}
coronerSetupNext(coroner, bpg, setupCfg)
.then(() => console.log(`Setup complete`))
.catch((err) => console.log(`Setup failed: ${err}`));
}
function coronerSetup(argv, config) {
var coroner, pu;
process.env.NODE_TLS_REJECT_UNAUTHORIZED = (!argv.k) ? "1" : "0";
try {
pu = url.parse(argv._[1]);
} catch (error) {
errx('Usage: morgue setup <url>');
}
if (pu.protocol !== 'http:' &&
pu.protocol !== 'https:') {
errx('Usage: morgue setup <url>');
}
coroner = coronerClient(config, true, !!argv.debug, argv._[1], argv.timeout);
process.stderr.write(bold('Determining system state...'));
coroner.get('/api/is_configured', '', function(error, response) {
response = parseInt(response + '');
if (response === 0) {
process.stderr.write(red('unconfigured\n'));
return coronerSetupStart(coroner, argv);
} else if (response === 1) {
process.stderr.write(green('configured\n\n'));
console.log(bold('Please login to continue setup.'));
return coronerLogin(argv, config, coronerSetupStart);
} else {
process.stderr.write(red('\n\nUnexpected response when checking the server\'s status.\n\n'));
process.stderr.write(red('This could be caused by one of the following:\n'));
process.stderr.write(red(' * Either the coronerd or backtrace-nginx services are not running on the server, or are in a failed state.\n'));
process.stderr.write(red(' * A proxy or forwarder is handling the request and is returning a response that the morgue client cannot understand.\n'));
process.stderr.write(red(' * Certificate validation is failing when trying to communicate with the server (try using -k if using self-signed certificates).\n'));
process.stderr.write(red(' * The destination URL is incorrect.\n'));
process.stderr.write(red('\n'));
process.stderr.write(red('To view the response from the server, try running the following command:\n'));
process.stderr.write(red(' curl ' + argv._[1] + '/api/is_configured\n'));
process.exit(1);
}
});
}
function userUsage(error_str) {
if (typeof error_str === 'string')
err(error_str + '\n');
console.log("Usage: morgue user reset [options]");
console.log("Valid options:");
console.log(" --password=P Specify password to use for reset.");
console.log(" --universe=U Specify universe scope.");
console.log(" --user=USER Specify user to reset password for");
process.exit(1);
}
function userReset(argv, config) {
var ctx = {
user: argv.user,
password: argv.password,
role: argv.role,
coroner: coronerClientArgv(config, argv),
};
var prompts = [];
var tasks = [];
ctx.bpg = coronerBpgSetup(ctx.coroner, argv),
ctx.model = ctx.bpg.get();
/* If no universe specified, use the first one. */
ctx.universe = argv.universe;
if (!ctx.universe && config && config.config && config.config.universes)
ctx.universe = Object.keys(config.config.universes)[0];
if (!ctx.universe) {
coronerUsage("No universes.");
}
/* Find the universe with the specified name. */
for (var i = 0; i < ctx.model.universe.length; i++) {
if (ctx.model.universe[i].get("name") === ctx.universe) {
ctx.univ_obj = ctx.model.universe[i];
break;
}
}
if (!ctx.univ_obj) {
userUsage("Must specify known universe.");
}
if (!ctx.user) {
prompts.push({name: "username", message: "User", required: true});
}
if (ctx.role && !BACKTRACE_ROLES.includes(ctx.role)) {
console.error(`Role must be one of: ${BACKTRACE_ROLES.join(', ')}.`);
return
}
if (prompts.length > 0) {
tasks.push(prompt_for(prompts));
tasks.push((result) => {
if (result.username)
ctx.user = result.username;
if (result.password)
ctx.password = result.password;
});
}
tasks.push(() => {
/* Find the user with the specified name. */
for (var i = 0; i < ctx.model.users.length; i++) {
if (ctx.model.users[i].get("username") === ctx.user &&
ctx.model.users[i].get("universe") === ctx.univ_obj.get("id")) {
ctx.user_obj = ctx.model.users[i];
break;
}
}
if (!ctx.user_obj) {
return Promise.reject("Must specify valid user.");
}
let modifyFields = {}
if (!ctx.role && !ctx.password){
return Promise.reject("Must specify a field to modify.");
}
if (ctx.password) {
modifyFields.password = BPG.blobText(ctx.password)
}
if (ctx.role) {
modifyFields.role = ctx.role
}
try {
ctx.bpg.modify(ctx.user_obj, modifyFields);
ctx.bpg.commit();
console.log(success_color("User successfully modified."));
} catch(e) {
return Promise.reject(e);
}
});
sequence(tasks).catch((e) => {
err(e.toString());
process.exit(1);
});
}
function addDomainWhitelist(argv, config) {
const domain = argv.domain
const role = argv.role
const method = argv.method
var coroner = coronerClientArgv(config, argv);
var bpg = coronerBpgSetup(coroner, argv);
var universe = argv.universe;
if (!universe)
universe = Object.keys(config.config.universes)[0];
let model = bpg.get();
let universeId;
/* Find the universe with the specified name. */
for (let i = 0; i < model.universe.length; i++) {
if (model.universe[i].get('name') === universe) {
const un = model.universe[i];
universeId = un.get('id');
}
}
if (!universeId){
return Promise.reject(`Missing config universe.`);
}
if (!domain || !role || !method) {
return Promise.reject(`Missing arguments: domain, role, or method.`);
}
if (!BACKTRACE_ROLES.includes(role)) {
return Promise.reject(`Role must be one of: ${BACKTRACE_ROLES.join(', ')}.`);
}
var signup = bpg.new('signup_whitelist');
signup.set('universe', universeId);
signup.set('domain', domain);
signup.set('role', role);
signup.set('method', method);
bpg.create(signup);
bpg.commit();
console.log(`Created domain whitelist for domain ${domain}.`)
}
async function listTeamlessUsers(argv, config) {
var coroner = coronerClientArgv(config, argv);
var bpg = coronerBpgSetup(coroner, argv);
// Get all users and team_members
const allUsers = await bpgPostAsync(bpg, bpgSingleRequest({
action: "get",
type: "configuration/users",
}));
const users = allUsers.filter((u) => !isBacktraceUser(u))
const teamMembers = await bpgPostAsync(bpg, bpgSingleRequest({
action: "get",
type: "configuration/team_member",
}));
// get set of all user ids
const allTeamMemberIds = teamMembers.map((teamMember) => teamMember.user)
const teamMemberIds = [...new Set(allTeamMemberIds)]
// filter all users by ones that are not included in the teamMembers set
const noTeamUsers = users.filter((user) => !teamMemberIds.includes(user.uid) )
if(!noTeamUsers.length){
console.log('No teamless users.')
return
}
// log results
console.log('Users with no teams:')
console.log(noTeamUsers.map((u, i) => `${i+1}. username=${u.username}, email=${u.email}, role=${u.role}`).join('\n'))
}
function isBacktraceUser(user) {
if(!user) return false
return user.username === 'Backtrace' || user.email.includes('@backtrace.io')
}
function coronerUser(argv, config) {
argv._.shift();
if (argv._.length === 0) {
userUsage();
}
if (argv._[0] !== "reset") {
userUsage("Only the reset subcommand is supported.");
}
argv._.shift();
userReset(argv, config);
}
function coronerUsers(argv, config) {
argv._.shift();
const action = argv._[0]
switch (action) {
case 'add-domain-whitelist':
addDomainWhitelist(argv, config);
break
case 'list-teamless-users':
listTeamlessUsers(argv, config)
break
}
}
function dump_obj(o)
{
console.log(util.inspect(o, {showHidden: false, depth: null}));
}
function _coronerMerge(coroner, universe, project, fingerprints, action) {
const query = {
actions: {
fingerprint: [
{
type: action,
arguments: fingerprints
}
]
}
};
dump_obj(query);
coroner.query(universe, project, query, function(err) {
if (err) {
errx(err.message);
}
console.log(success_color('Success.'));
});
}
async function _workflowsMerge(coroner, universe, project, fingerprints) {
const client = await WorkflowsClient.fromCoroner(coroner);
try {
await client.mergeFingerprints(universe, project, fingerprints)
console.log(success_color('Success.'));
} catch (err) {
errx(err.message);
}
}
async function mergeFingerprints(argv, config) {
abortIfNotLoggedIn(config);
if (argv._.length === 0) {
return userUsage();
}
const { universe, project } = coronerParams(argv, config);
if (!universe || !project) {
return usage("Missing project, universe arguments");
}
const fingerprints = argv._.slice(2);
if (!fingerprints.length) {
return usage("At least one fingerprint must be specified")
}
const coroner = coronerClientArgv(config, argv);
const isWorkflowsAvailable = await WorkflowsClient.isAvailable(coroner)
if (isWorkflowsAvailable) {
if (argv.debug) {
console.log('Merging using the Workflows service')
}
return _workflowsMerge(coroner, universe, project, fingerprints)
} else {
if (argv.debug) {
console.log('Merging using Coroner directly')
}
return _coronerMerge(coroner, universe, project, fingerprints, 'merge');
}
}
function unmergeFingerprints(argv, config) {abortIfNotLoggedIn(config);
if (argv._.length === 0) {
return userUsage();
}
const { universe, project} = coronerParams(argv, config);
if (!universe || !project) {
return usage("Missing project, universe arguments");
}
const fingerprints = argv._.slice(2);
if (!fingerprints.length) {
return usage("At least one fingerprint must be specified")
}
const coroner = coronerClientArgv(config, argv);
return _coronerMerge(coroner, universe, project, fingerprints, 'unmerge');
}
/*
* Usage: ci <project> --run=<attribute> --tag=<branch> <value> --slack=<base> --target=<channel>
*
* coroner ci cts run sbahra-123 --slack=292191/12392139/119212912 --target=#build
*/
function coronerCI(argv, config) {
abortIfNotLoggedIn(config);
var coroner = coronerClientArgv(config, argv);
let message = '';
let slack;
let universe = argv.universe;
if (!universe) {
universe = Object.keys(config.config.universes)[0];
}
let project = argv._[1];
let value = argv._[2];
let attribute = argv.run;
/* Get a summary of issues found by tool for a given run. */
let query = queryCli.argvQuery(argv);
let q_v = query.query;
q_v.filter[0][attribute] = [ [ "equal", value ] ];
q_v.filter[0]['fingerprint;issues;state'] = [ [ "regular-expression", "open|progress" ] ];
q_v.group = ["tool"];
q_v.fold = {};
q_v.fold.fingerprint = [[ "unique" ]];
if (!argv.terminal && argv.slack && argv.target) {
slack = new Slack();
slack.setWebhook('https://hooks.slack.com/services/' + argv.slack);
}
coroner.query(universe, project, q_v, function(err, result) {
if (err) {
errx(err.message);
}
var rp = new crdb.Response(result.response);
rp = rp.unpack();
let total_f = 0;
let total_c = 0;
for (let j in rp) {
total_f += rp[j]['unique(fingerprint)'][0];
total_c += rp[j].count;
}
delete q_v.filter[0][attribute];
q_v.filter[0]['fingerprint;issues;tags'] = [["contains", argv.tag]];
q_v.fold.classifiers = [[ "distribution" ]];
delete q_v.group;
coroner.query(universe, project, q_v, function(err, result) {
if (err) {
errx(err.message);
}
var rp = new crdb.Response(result.response);
rp = rp.unpack();
let fields = [];
fields.push({title:"",short:false,value:""});
let open_count = 0;
if (rp && rp['*'] && rp['*'].count > 0) {
open_count = rp['*'].count;
fields.push({
title: "Failure",
value: open_count + ' regressions introduced in `' +
argv.tag + '` in an open state.\n'
});
let d_v = rp['*']['distribution(classifiers)'][0].vals;
for (var i = 0; i < d_v.length; i++) {
if (d_v[i][0].length > 32)
d_v[i][0] = d_v[i][0].substring(0, 16) + '...';
fields.push({ title: "`" + d_v[i][0] + "`", value: d_v[i][1] + "", "short" : true });
}
} else {
fields.push({
title: "Success",
value: "No open regressions found."
});
}
message += 'Found ' + total_c + ' errors across ' + total_f + ' open issues.\n';
function c_url(a, o, v) {
return config.endpoint + "/p/" + project + "/triage?time=month&filters=((" +
a + "%2C" + o + "%2C" + v + ")%2C(fingerprint%3Bissues%3Bstate%2Cregex%2Copen%7Cprogress))";
}
fields.push({title:"",short:false,value:""});
if (argv.author) {
fields.push({
title: "Author",
short: true,
value: argv.author
});
}
if (argv.build) {
fields.push({
title: "Build",
short: true,
value: argv.build + ""
});
}
if (total_f > 0 || open_count > 0) {
fields.push({
title: "Actions",
short: false,
value:
"<" + c_url("fingerprint%3Bissues%3Btags", "contains", argv.tag) + "|View regressions> | " +
"<" + c_url(argv.run, "equal", value) + "|View all defects>"
});
}
if (! argv.terminal) {
if (slack) {
if (argv.author) {
slack.webhook({
channel: '@' + argv.author,
username: 'Backtrace',
attachments: [
{
color : open_count > 0 ? "#FF0000" : "good",
footer: "Backtrace",
footer_icon: "https://backtrace.io/images/icon.png",
author_name: value,
ts: parseInt(Date.now() / 1000),
fields: fields,
text: message
}
]
}, function (e, r) {
});
}
slack.webhook({
channel: argv.target,
username: 'Backtrace',
attachments: [
{
color : open_count > 0 ? "#FF0000" : "good",
footer: "Backtrace",
footer_icon: "https://backtrace.io/images/icon.png",
author_name: value,
ts: parseInt(Date.now() / 1000),
fields: fields,
text: message
}
]
}, function (e, r) {
});
}
} else {
console.log(JSON.stringify({msg: message, fields: fields}));
}
});
});
}
/*
* Usage: cts <project> <attribute> <target>
*
* Sets marker for uniquely introduced issues.
*/
function coronerCts(argv, config) {
/* First extract a list of all fingerprint values for the given target. */
abortIfNotLoggedIn(config);
var coroner = coronerClientArgv(config, argv);
let universe = argv.universe;
if (!universe)
universe = Object.keys(config.config.universes)[0];
let project = argv._[1];
let attribute = argv._[2];
let value = argv._[3];
let query = queryCli.argvQuery(argv);
let q_v = query.query;
q_v.filter[0][attribute] = [ [ "equal", value ] ];
q_v.filter[0]["fingerprint;issues;tags"] = [ [ "not-contains", value ] ];
q_v.group = ["fingerprint"];
q_v.fold = {};
q_v.fold[attribute] = [[ "count" ]];
if (argv.query) {
console.log(JSON.stringify(q_v, null, 2));
return;
}
let fingerprint = {};
coroner.query(universe, project, q_v, function(err, result) {
if (err) {
errx(err.message);
}
var rp = new crdb.Response(result.response);
rp = rp.unpack();
for (var k in rp) {
fingerprint[k] = true;
}
/* Now we have suspect fingerprints. Eliminate those not unique to the run. */
delete(q_v.filter[0][attribute]);
q_v.fold[attribute] = [ [ "distribution", 8192 ] ];
coroner.query(universe, project, q_v, function(err, result) {
if (err) {
errx(err.message);
}
var rp = new crdb.Response(result.response);
rp = rp.unpack();
for (var k in rp) {
if (!fingerprint[k])
continue;
var dt = rp[k]["distribution(" + attribute + ")"][0];
if (dt.keys > 1)
delete fingerprint[k];
}
/* Construct a query to set tags for each of these issues. */
delete(q_v.group);
delete(q_v.fold);
let n_issues = Object.keys(fingerprint).length;
if (n_issues === 0) {
console.log('No new issues introduced.');
return;
} else {
console.log('Setting tag ' + value + ' on ' + n_issues + ' issues.');
}
let filter_string = '';
let first = true;
for (var k in fingerprint) {
if (first === false)
filter_string += '|';
filter_string += '^' + k + "$";
first = false;
}
q_v.filter[0].fingerprint = [ [ "regular-expression", filter_string ] ];
delete q_v.filter[0].timestamp;
q_v.set = {"tags" : value + ""};
q_v.table = "issues";
q_v.select = ["tags"];
delete q_v.filter[0]["fingerprint;issues;tags"];
q_v.filter[0]["tags"] = [ [ "not-contains", value ] ];
coroner.query(universe, project, q_v, function(error, result) {
if (err) {
errx(err.message);
}
});
return;
});
});
}
/**
* @brief Implements the logout command.
*/
function coronerLogout(argv, config) {
abortIfNotLoggedIn(config);
var coroner = coronerClientArgv(config, argv);
coroner.http_get('/api/logout', { token: argv.token || coroner.config.token },
null, function(error, result) {
if (error)
errx(error + '');
console.log(success_color('Logged out.'));
});
}
function coronerAccessControlUsage() {
console.error('Usage:');
console.error('morgue access <action> [params...]');
console.error('');
console.error('actions:');
console.error(' - team');
console.error(' - project');
console.error('');
console.error('action team:');
console.error(' morgue access team <create|remove|details> <team>');
console.error(' morgue access team add-user <team> <user>');
console.error(' morgue access team remove-user <team> <user>');
console.error(' morgue access team list');
console.error('');
console.error('action project:');
console.error(' morgue access project <project> add-team <team> <role>');
console.error(' morgue access project <project> remove-team <team>');
console.error(' morgue access project <project> add-user <user> <role>');
console.error(' morgue access project <project> remove-user <user>');
console.error(' morgue access project <project> details');
}
function coronerTeamCreate({bpg, argv, universeId, model}) {
const teamName = argv._[3];
let team = bpg.new('team');
team.set('name', teamName);
team.set('universe', universeId);
team.set('id', 0);
bpg.create(team);
bpg.commit();
}
function coronerTeamDelete({bpg, argv, model}) {
const teamName = argv._[3];
let team = model.team.find(function(t) {
return t.get('name') == teamName;
});
if (team === undefined) {
err("Team not found");
return;
}
bpg.delete(team);
bpg.commit();
}
function coronerTeamList({argv, universeId, model}) {
model.team.filter(t => t.get('universe') == universeId).forEach(t => {
console.log(t.get('name'));
});
}
function coronerTeamUserAdd({bpg, argv, universeId, model}) {
const teamName = argv._[3];
const userName = argv._[4];
const team = model.team.find(function(t) {
return t.get('name') == teamName;
});
if (team === undefined) {
err("Team not found");
return;
}
const user = model.users.find(function(u) {
return u.get('username') == userName;
});
if (user === undefined) {
err("User not found");
return;
}
let tm = bpg.new('team_member');
tm.set('team', team.get('id'));
tm.set('user', user.get('uid'));
bpg.create(tm);
bpg.commit();
}
function coronerTeamUserDelete({bpg, argv, universeId, model}) {
const teamName = argv._[3];
const userName = argv._[4];
let team = model.team.find(function(t) {
return t.get('name') == teamName;
});
if (team === undefined) {
err("Team not found");
return;
}
const user = model.users.find(function(u) {
return u.get('username') == userName;
});
if (user === undefined) {
err("User not found");
return;
}
const tm = model.team_member.find(function(tm) {
return tm.get('user') == user.get('uid') && tm.get('team') == team.get('tid');
});
if (tm === undefined) {
err(`User '${userName}' is not a member of team '${teamName}'`);
return;
}
bpg.delete(tm);
bpg.commit();
}
function coronerTeamDetails({argv, model}) {
const teamName = argv._[3];
let team = model.team.find(function(t) {
return t.get('name') == teamName;
});
if (team === undefined) {
err("Team not found");
return;
}
const teamId = team.get('id');
const idToUser = function() {
let ret = {}
const arr = model.users.map(u => [u.get('uid'), u.get('username')]);
arr.forEach((a) => ret[a[0]] = a[1])
return ret
}();
console.log(blue("Team members:"))
model.team_member.filter(tm => tm.get('team') == teamId).forEach(function(tm) {
const name = idToUser[tm.get('user')] || '<unknown_name>';
console.log(` - ${name}`);
});
console.log(blue("\nTeam is a member of projects:"));
model.project_member_team.filter(pm => pm.get('team') == teamId).forEach(pm => {
const projectBpg = model.project.find(p => p.get('pid') == pm.get('project'));
if (projectBpg === undefined)
errx(`Project with id ${pm.get('project')} not found`);
console.log(` - ${projectBpg.get('name')} - ${pm.get('role')}`);
});
}
function coronerProjectAddTeamUser({mode, bpg, argv, model, idSupply}) {
const projectName = argv._[2];
const suppliedName = argv._[4];
const role = argv._[5];
const project = model.project.find(p => p.get('name') == projectName);
if (project === undefined)
errx(`project not found: ${projectName}`.red);
const id = idSupply[mode](suppliedName);
if (role === undefined)
errx('need to supply role'.red);
if (role.match(/(guest|member|admin)/) == false)
errx('unknown role'.red);
let add = bpg.new(`project_member_${mode}`);
add.set('project', project.get('pid'));
add.set(mode, id);
add.set('role', role);
bpg.create(add);
bpg.commit();
}
function coronerProjectRemoveTeamUser({mode, bpg, argv, model, idSupply}) {
const projectName = argv._[2];
const suppliedName = argv._[4];
const id = idSupply[mode](suppliedName);
const project = model.project.find(p => p.get('name') == projectName);
if (project === undefined)
errx(`project not found: ${projectName}`.red);
let bpgObject = model[`project_member_${mode}`].find(pm => pm.get(mode) == id && pm.get('project') == project.get('pid'));
if (bpgObject === undefined)
errx(`${mode} not found for project ${projectName}`);
bpg.delete(bpgObject);
bpg.commit();
}
function coronerProjectAccessDetails({argv, model}) {
const projectName = argv._[2];
const project = model.project.find(p => p.get('name') == projectName);
if (project === undefined)
errx(`project not found: ${projectName}`.red);
const users = model.project_member_user.filter(pm => pm.get('project') == project.get('pid'));
const teams = model.project_member_team.filter(pm => pm.get('project') == project.get('pid'));
if (users.length == 0 && teams.length == 0) {
console.log(`Project ${projectName} has no access control`);
return;
}
console.log('Teams:')
teams.forEach(pm => {
const team = model.team.find(t => t.get('id') == pm.get('team'))
if (team === undefined)
errx(`Team ${pm.get('team')} not found`)
console.log(`${team.get('name')} - ${pm.get('role')}`);
})
console.log("--\n")
console.log('Users:')
users.forEach(pm => {
const user = model.users.find(u => u.get('uid') == pm.get('user'))
if (user === undefined)
errx(`User ${pm.get('user')} not found`)
console.log(`${user.get('username')} - ${pm.get('role')}`);
})
console.log("--\n")
}
/**
* @brief Implements the limit command.
*/
function coronerAccessControl(argv, config) {
var options = null;
abortIfNotLoggedIn(config);
var coroner = coronerClientArgv(config, argv);
let universe = argv.universe;
if (!universe)
universe = Object.keys(config.config.universes)[0];
let bpg = coronerBpgSetup(coroner, argv);
let model = bpg.get();
let universeId;
/* Find the universe with the specified name. */
for (let i = 0; i < model.universe.length; i++) {
if (model.universe[i].get('name') === universe) {
const un = model.universe[i];
universeId = un.get('id');
}
}
if (universeId === undefined) {
errx("Universe not found".red);
}
/* The sub-command. */
const submodule = argv._[1];
if (submodule == 'team') {
const actionHandlers = {
create: coronerTeamCreate,
remove: coronerTeamDelete,
list: coronerTeamList,
details: coronerTeamDetails,
'add-user': coronerTeamUserAdd,
'remove-user': coronerTeamUserDelete,
};
const params = {
bpg: bpg,
argv: argv,
universeId: universeId,
model: model,
};
const action = argv._[2];
const handler = actionHandlers[action];
if (handler !== undefined) {
handler(params);
} else {
coronerAccessControlUsage();
}
} else if (submodule == 'project') {
// access project <project> action [user|team] [role]
const idSupply = {
team: (suppliedName) => {
const t = model.team.find(t => t.get('name') == suppliedName);
if (t === undefined)
errx(`team not found: ${suppliedName}`.red);
return t.get('id');
},
user: (suppliedName) => {
const u = model.users.find(u => u.get('username') == suppliedName);
if (u === undefined)
errx(`user not found: ${suppliedName}`.red);
return u.get('uid');
}
};
const params = {
bpg: bpg,
argv: argv,
model: model,
idSupply: idSupply,
}
const actionHandlers = {
'add-team': (ps) => coronerProjectAddTeamUser(Object.assign({mode: 'team'}, ps)),
'remove-team': (ps) => coronerProjectRemoveTeamUser(Object.assign({mode: 'team'}, ps)),
'add-user': (ps) => coronerProjectAddTeamUser(Object.assign({mode: 'user'}, ps)),
'remove-user': (ps) => coronerProjectRemoveTeamUser(Object.assign({mode: 'user'}, ps)),
'details': coronerProjectAccessDetails,
}
const action = argv._[3];
const handler = actionHandlers[action];
if (handler !== undefined) {
handler(params);
} else {
coronerAccessControlUsage();
}
} else {
coronerAccessControlUsage();
return;
}
}
/**
* @brief Implements the limit command.
*/
function coronerLimit(argv, config) {
var options = null;
var project, universe, pid, un, target;
abortIfNotLoggedIn(config);
var coroner = coronerClientArgv(config, argv);
universe = argv.universe;
if (!universe)
universe = Object.keys(config.config.universes)[0];
/* The sub-command. */
var action = argv._[1];
if (action == 'list') {
var bpg = coronerBpgSetup(coroner, argv);
var model = bpg.get();
/* Find the universe with the specified name. */
if (universe) {
for (var i = 0; i < model.universe.length; i++) {
if (model.universe[i].get('name') === universe) {
un = target = model.universe[i];
break;
}
}
}
coroner.http_get('/api/limits', {universe: universe, token: coroner.config.token}, null, function(error, result) {
if (error)
errx(error + '');
var rp = JSON.parse(result.bodyData);
for (var uni in rp) {
if (un && un.get('name') !== uni)
continue;
var st = printf("%3d %16s limit=%d,counter=%d,rejected=%d",
rp[uni].id, bold(uni),
rp[uni].submissions.limit,
rp[uni].submissions.counter,
rp[uni].submissions.rejected);
console.log(st);
}
return;
});
} else {
var bpg = coronerBpgSetup(coroner, argv);
var model = bpg.get();
/* Find the universe with the specified name. */
for (var i = 0; i < model.universe.length; i++) {
if (model.universe[i].get('name') === universe) {
un = target = model.universe[i];
break;
}
}
if (!un)
errx('universe not found');
if (action === 'reset') {
var limit;
console.log('Resetting limits for [' + un.get('id') + '/' + un.get('name') + ']...');
for (var i = 0; i < model.limits.length; i++) {
if (model.limits[i].get('universe') === un.get('id')) {
limit = model.limits[i];
break;
}
}
if (!limit)
errx('Specified universe has no limits.');
bpg.delete(limit);
bpg.commit();
bpg.create(limit);
bpg.commit();
return;
}
if (action === 'delete') {
var limit;
if (!un)
errx('Usage: morgue limit delete --universe=<universe>');
for (var i = 0; i < model.limits.length; i++) {
if (model.limits[i].get('universe') === un.get('id')) {
limit = model.limits[i];
}
}
if (!limit)
errx('Limit not found.');
console.log(('Deleting limit [' +
yellow(limit.get('universe') + ']...')));
bpg.delete(limit);
bpg.commit();
return;
}
if (action === 'create') {
var definition = {};
if (!un)
errx('Must specify a universe');
if (!argv.submissions)
errx('--submissions must be specified');
var limit = bpg.new('limits');
limit.set('universe', un.get('id'));
definition.submissions = {
'period' : 'month',
'day' : 1,
'limit' : [argv.submissions, argv.submissions]
};
limit.set('definition', JSON.stringify(definition));
limit.set('metadata', '{}');
if (argv.metadata)
limit.set('metadata', argv.metadata);
bpg.create(limit);
try {
bpg.commit();
} catch (e) {
errx(e + '');
}
console.log(success_color('Limit successfully created.'));
return;
}
errx('Unknown subcommand.');
}
}
function tenantURL(config, tn) {
/*
* If there is no current universe, return the URL unchanged.
* If this were just a split on ., it'd probably change the root domain
* in this case.
*/
if (!config.config.universe)
return config.endpoint;
const uname = config.config.universe.name;
let pattern = uname;
let replacement = tn;
const tsep = config.config.tenant_separator;
if (tsep) {
/*
* Since the universe name and separator are known, go ahead and be
* stricter.
*
* For example, this would prevent localhost from becoming otherhost if
* moving from universe local to universe other.
*/
pattern += tsep;
replacement += tsep;
}
return config.endpoint.replace(pattern, replacement);
}
function coronerInvite(argv, config) {
var options = null;
abortIfNotLoggedIn(config);
var coroner = coronerClientArgv(config, argv);
var usageText =
'Usage: morgue invite <create | list | resend>\n' +
' create <username> <email>\n' +
' --role=<"guest" | "member" | "admin">\n' +
' --metadata=<metadata>\n' +