@atomist/org-visualizer
Version:
Organization Visualizer using Atomist project scanning
508 lines • 24.7 kB
JavaScript
"use strict";
/*
* Copyright © 2019 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
Object.defineProperty(exports, "__esModule", { value: true });
const automation_client_1 = require("@atomist/automation-client");
const sdm_pack_fingerprints_1 = require("@atomist/sdm-pack-fingerprints");
const categories_1 = require("../../../customize/categories");
const bands_1 = require("../../../util/bands");
const commonBands_1 = require("../../../util/commonBands");
const ProjectAnalysisResult_1 = require("../../ProjectAnalysisResult");
const pgUtils_1 = require("./pgUtils");
const ProjectAnalysisResultStore_1 = require("./ProjectAnalysisResultStore");
// tslint:disable:max-file-line-count
class PostgresProjectAnalysisResultStore {
constructor(clientFactory) {
this.clientFactory = clientFactory;
}
distinctRepoCount(workspaceId) {
const sql = `SELECT COUNT(*) FROM (SELECT DISTINCT owner, name FROM repo_snapshots
WHERE workspace_id ${workspaceId === "*" ? "<>" : "="} $1) as repos`;
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const result = yield client.query(sql, [workspaceId]);
return +result.rows[0].count;
}));
}
latestTimestamp(workspaceId) {
const sql = `SELECT timestamp FROM repo_snapshots WHERE workspace_id ${workspaceId === "*" ? "<>" : "="} $1
ORDER BY timestamp DESC LIMIT 1`;
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const result = yield client.query(sql, [workspaceId]);
return result.rows[0].timestamp;
}));
}
loadInWorkspace(workspaceId) {
const wsid = workspaceId || "*";
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const sql = `SELECT id, owner, name, url, commit_sha, analysis, timestamp
from repo_snapshots
WHERE workspace_id ${wsid !== "*" ? "=" : "<>"} $1`;
const rows = yield client.query(sql, [wsid]);
return rows.rows.map(row => (Object.assign({}, row, { repoRef: rowToRepoRef(row) })));
}), []);
}
loadById(id) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const result = yield client.query(`SELECT owner, name, url, commit_sha, analysis, timestamp
FROM repo_snapshots
WHERE id = $1`, [id]);
return result.rows.length === 1 ? {
id,
analysis: result.rows[0].analysis,
timestamp: result.rows[0].timestamp,
workspaceId: result.rows[0].workspace_id,
repoRef: rowToRepoRef(result.rows[0]),
} : undefined;
}));
});
}
loadByRepoRef(repo) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const result = yield client.query(`SELECT id, owner, name, url, commit_sha, analysis, timestamp
FROM repo_snapshots
WHERE owner = $1 AND name = $2 AND commit_sha = $3`, [repo.owner, repo.repo, repo.sha]);
return result.rows.length >= 1 ? {
id: result.rows[0].id,
analysis: result.rows[0].analysis,
timestamp: result.rows[0].timestamp,
workspaceId: result.rows[0].workspace_id,
repoRef: rowToRepoRef(result.rows[0]),
} : undefined;
}));
});
}
persist(repos) {
return __awaiter(this, void 0, void 0, function* () {
return this.persistAnalysisResults(ProjectAnalysisResult_1.isProjectAnalysisResult(repos) ? [repos] : repos);
});
}
distinctFingerprintKinds(workspaceId) {
return __awaiter(this, void 0, void 0, function* () {
const sql = `SELECT distinct f.name, feature_name as type
from repo_fingerprints rf, repo_snapshots rs, fingerprints f
WHERE rf.repo_snapshot_id = rs.id AND rf.fingerprint_id = f.id AND rs.workspace_id ${workspaceId === "*" ? "!=" : "="} $1`;
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const result = yield client.query(sql, [workspaceId]);
return result.rows;
}), []);
});
}
fingerprintUsageForType(workspaceId, type) {
return fingerprintUsageForType(this.clientFactory, workspaceId, type);
}
storeIdeal(workspaceId, ideal) {
return __awaiter(this, void 0, void 0, function* () {
if (sdm_pack_fingerprints_1.isConcreteIdeal(ideal)) {
yield pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
// Clear out any existing ideal
yield client.query("DELETE FROM ideal_fingerprints WHERE workspace_id = $1 AND fingerprint_id IN " +
"(SELECT id from fingerprints where feature_name = $2 AND name = $3)", [workspaceId, ideal.ideal.type, ideal.ideal.name]);
const fid = yield this.ensureFingerprintStored(ideal.ideal, client);
yield client.query(`INSERT INTO ideal_fingerprints (workspace_id, fingerprint_id, authority)
values ($1, $2, 'local-user')`, [
workspaceId, fid
]);
}));
}
else {
throw new Error("Elimination ideals not yet supported");
}
});
}
setIdeal(workspaceId, fingerprintId) {
return __awaiter(this, void 0, void 0, function* () {
const ideal = yield this.loadFingerprintById(fingerprintId);
if (!ideal) {
throw new Error(`Fingerprint with id=${fingerprintId} not found and cannot be used as an ideal`);
}
const ci = {
reason: "Local database",
ideal,
};
yield this.storeIdeal(workspaceId, ci);
});
}
loadIdeals(workspaceId) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const rows = yield client.query(`SELECT id, name, feature_name as type, sha, data
FROM ideal_fingerprints, fingerprints
WHERE workspace_id = $1 AND ideal_fingerprints.fingerprint_id = fingerprints.id`, [workspaceId]);
if (!rows.rows) {
return [];
}
return rows.rows.map(idealRowToIdeal);
}), []);
});
}
noteProblem(workspaceId, fingerprintId) {
return __awaiter(this, void 0, void 0, function* () {
const fingerprint = yield this.loadFingerprintById(fingerprintId);
if (!fingerprint) {
throw new Error(`Fingerprint with id=${fingerprintId} not found and cannot be noted as problem`);
}
yield this.storeProblemFingerprint(workspaceId, { fingerprint, severity: "warn", authority: "local-user" });
});
}
storeProblemFingerprint(workspaceId, fp) {
return __awaiter(this, void 0, void 0, function* () {
yield pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
// Clear out any existing ideal
const fid = yield this.ensureFingerprintStored(fp.fingerprint, client);
yield client.query(`INSERT INTO problem_fingerprints (workspace_id, fingerprint_id, severity, authority, date_added)
values ($1, $2, $3, $4, current_timestamp)`, [
workspaceId, fid, fp.severity, fp.authority
]);
}));
});
}
loadProblems(workspaceId) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const rows = yield client.query(`SELECT id, name, feature_name as type, sha, data, authority, severity, description, url
FROM problem_fingerprints, fingerprints
WHERE workspace_id = $1 AND problem_fingerprints.fingerprint_id = fingerprints.id`, [workspaceId]);
if (!rows.rows) {
return [];
}
return rows.rows.map(problemRowToProblem);
}), []);
});
}
loadIdeal(workspaceId, type, name) {
return __awaiter(this, void 0, void 0, function* () {
const rawRow = yield pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const rows = yield client.query(`SELECT id, name, feature_name as type, sha, data
FROM ideal_fingerprints, fingerprints
WHERE workspace_id = $1 AND ideal_fingerprints.fingerprint_id = fingerprints.id
AND feature_name = $2 AND name = $3`, [workspaceId, type, name]);
return rows.rows.length === 1 ? rows.rows[0] : undefined;
}));
if (!rawRow) {
return undefined;
}
return idealRowToIdeal(rawRow);
});
}
loadFingerprintById(id) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const rows = yield client.query(`SELECT id, name, feature_name as type, sha, data FROM fingerprints
WHERE id = $1`, [id]);
return rows.rows.length === 1 ? rows.rows[0] : undefined;
}));
});
}
fingerprintsInWorkspace(workspaceId, type, name, dedup) {
return __awaiter(this, void 0, void 0, function* () {
return fingerprintsInWorkspace(this.clientFactory, workspaceId, type, name, dedup);
});
}
fingerprintsForProject(snapshotId) {
return __awaiter(this, void 0, void 0, function* () {
return fingerprintsForProject(this.clientFactory, snapshotId);
});
}
persistAnalytics(data) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
for (const { kind, workspaceId, cohortAnalysis } of data) {
const sql = `INSERT INTO fingerprint_analytics (feature_name, name, workspace_id, entropy, variants, count)
values ($1, $2, $3, $4, $5, $6)
ON CONFLICT ON CONSTRAINT fingerprint_analytics_pkey DO UPDATE SET entropy = $4, variants = $5, count = $6`;
yield client.query(sql, [kind.type, kind.name, workspaceId,
cohortAnalysis.entropy, cohortAnalysis.variants, cohortAnalysis.count]);
}
return true;
}));
});
}
persistAnalysisResults(analysisResultIterator) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(this.clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
var e_1, _a;
const persistResults = [];
try {
for (var analysisResultIterator_1 = __asyncValues(analysisResultIterator), analysisResultIterator_1_1; analysisResultIterator_1_1 = yield analysisResultIterator_1.next(), !analysisResultIterator_1_1.done;) {
const analysisResult = analysisResultIterator_1_1.value;
if (!analysisResult.analysis) {
throw new Error("Analysis is undefined!");
}
persistResults.push(yield this.persistOne(client, analysisResult));
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (analysisResultIterator_1_1 && !analysisResultIterator_1_1.done && (_a = analysisResultIterator_1.return)) yield _a.call(analysisResultIterator_1);
}
finally { if (e_1) throw e_1.error; }
}
return persistResults.reduce(ProjectAnalysisResultStore_1.combinePersistResults, ProjectAnalysisResultStore_1.emptyPersistResult);
}), ProjectAnalysisResultStore_1.emptyPersistResult);
});
}
persistOne(client, analysisResult) {
return __awaiter(this, void 0, void 0, function* () {
const repoRef = analysisResult.analysis.id;
if (!repoRef) {
return {
attemptedCount: 1,
succeeded: [],
failed: [{
repoUrl: "missing repoRef",
whileTryingTo: "build object to persist",
message: "No RepoRef",
}],
};
}
if (!repoRef.url || !repoRef.sha) {
return {
attemptedCount: 1,
succeeded: [],
failed: [{
repoUrl: "missing repoUrl. Repo is named " + repoRef.repo,
whileTryingTo: "build object to persist",
message: `Incomplete RepoRef ${JSON.stringify(repoRef)}`,
}],
};
}
try {
// Whack any snapshot we already hold for this repository
yield deleteOldSnapshotForRepository(repoRef, client);
// Use this as unique database id
const id = repoRef.url.replace("/", "") + "_" + repoRef.sha;
const shaToUse = !!analysisResult.analysis.gitStatus ?
analysisResult.analysis.gitStatus.sha :
repoRef.sha;
yield client.query(`
INSERT INTO repo_snapshots (id, workspace_id, provider_id, owner, name, url, commit_sha, analysis, query, timestamp)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, current_timestamp)`, [id,
analysisResult.workspaceId,
"github",
repoRef.owner,
repoRef.repo,
repoRef.url,
shaToUse,
analysisResult.analysis,
analysisResult.query,
]);
const fingerprintPersistResults = yield this.persistFingerprints(analysisResult.analysis, id, client);
fingerprintPersistResults.failures.forEach(f => {
automation_client_1.logger.error(`Could not persist fingerprint.
Error: ${f.error.message}
Repo: ${repoRef.url}
Fingerprint: ${JSON.stringify(f.failedFingerprint, undefined, 2)}`);
});
return {
succeeded: [id],
attemptedCount: 1,
failed: [],
};
}
catch (err) {
return {
attemptedCount: 1,
succeeded: [],
failed: [{
repoUrl: repoRef.url,
whileTryingTo: "persist in DB",
message: err.message,
}],
};
}
});
}
// Persist the fingerprints for this analysis
persistFingerprints(pa, id, client) {
return __awaiter(this, void 0, void 0, function* () {
let insertedCount = 0;
const failures = [];
for (const fp of pa.fingerprints) {
const aspectName = fp.type || "unknown";
const fingerprintId = aspectName + "_" + fp.name + "_" + fp.sha;
// console.log("Persist fingerprint " + JSON.stringify(fp) + " for id " + id);
// Create fp record if it doesn't exist
try {
yield this.ensureFingerprintStored(fp, client);
yield client.query(`INSERT INTO repo_fingerprints (repo_snapshot_id, fingerprint_id)
values ($1, $2) ON CONFLICT DO NOTHING
`, [id, fingerprintId]);
insertedCount++;
}
catch (error) {
failures.push({ failedFingerprint: fp, error });
}
}
return {
insertedCount,
failures,
};
});
}
/**
* Persist the given fingerprint if it's not already known
* @param {FP} fp
* @param {Client} client
* @return {Promise<void>}
*/
ensureFingerprintStored(fp, client) {
return __awaiter(this, void 0, void 0, function* () {
const aspectName = fp.type || "unknown";
const fingerprintId = aspectName + "_" + fp.name + "_" + fp.sha;
// console.log("Persist fingerprint " + JSON.stringify(fp) + " for id " + id);
// Create fp record if it doesn't exist
yield client.query(`INSERT INTO fingerprints (id, name, feature_name, sha, data)
values ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING
`, [fingerprintId, fp.name, aspectName, fp.sha, JSON.stringify(fp.data)]);
return fingerprintId;
});
}
}
exports.PostgresProjectAnalysisResultStore = PostgresProjectAnalysisResultStore;
function idealRowToIdeal(rawRow) {
if (!!rawRow.data) {
const result = {
ideal: rawRow,
reason: `Local database row ${rawRow.id}`,
};
return result;
}
throw new Error("Elimination ideals not yet supported");
}
function problemRowToProblem(rawRow) {
return {
fingerprint: {
name: rawRow.name,
type: rawRow.feature_name,
data: rawRow.data,
sha: rawRow.sha,
},
authority: rawRow.authority,
severity: rawRow.severity,
description: rawRow.description,
url: rawRow.url,
};
}
/**
* Raw fingerprints in the workspace
* @return {Promise<FP[]>}
*/
function fingerprintsInWorkspace(clientFactory, workspaceId, type, name, dedup) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const sql = `SELECT ${dedup ? "DISTINCT" : ""} f.name as fingerprintName, f.feature_name, f.sha, f.data
from repo_fingerprints rf, repo_snapshots rs, fingerprints f
WHERE rf.repo_snapshot_id = rs.id AND rf.fingerprint_id = f.id AND rs.workspace_id ${workspaceId === "*" ? "!=" : "="} $1
AND ${type ? "feature_name = $2" : "true"} AND ${name ? "f.name = $3" : "true"}
ORDER BY feature_name, fingerprintName ASC`;
const params = [workspaceId];
if (!!type) {
params.push(type);
}
if (!!name) {
params.push(name);
}
const rows = yield client.query(sql, params);
return rows.rows.map(row => {
return {
name: row.fingerprintname,
type: row.feature_name,
sha: row.sha,
data: row.data,
};
});
}), []);
});
}
function fingerprintsForProject(clientFactory, snapshotId) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const sql = `SELECT f.name as fingerprintName, f.feature_name, f.sha, f.data
from repo_fingerprints rf, repo_snapshots rs, fingerprints f
WHERE rs.id = $1 AND rf.repo_snapshot_id = rs.id AND rf.fingerprint_id = f.id
ORDER BY feature_name, fingerprintName ASC`;
const rows = yield client.query(sql, [snapshotId]);
return rows.rows.map(row => {
return {
name: row.fingerprintname,
type: row.feature_name,
sha: row.sha,
data: row.data,
};
});
}), []);
});
}
function fingerprintUsageForType(clientFactory, workspaceId, type) {
return __awaiter(this, void 0, void 0, function* () {
return pgUtils_1.doWithClient(clientFactory, (client) => __awaiter(this, void 0, void 0, function* () {
const sql = `SELECT name, feature_name as type, variants, count, entropy, compliance
from fingerprint_analytics f
WHERE f.workspace_id ${workspaceId === "*" ? "!=" : "="} $1
AND ${type ? "f.feature_name = $2" : "true"}
ORDER BY entropy DESC`;
const params = [workspaceId];
if (!!type) {
params.push(type);
}
const rows = yield client.query(sql, params);
return rows.rows.map(r => ({
name: r.name,
type: r.type,
variants: +r.variants,
count: +r.count,
entropy: +r.entropy,
compliance: +r.compliance,
entropy_band: bands_1.bandFor(commonBands_1.EntropySizeBands, +r.entropy, { casing: bands_1.BandCasing.Sentence, includeNumber: false }),
// This is really confusing but the Aspect.name is feature_name alias type in the db
categories: categories_1.getCategories({ name: r.type }),
}));
}), []);
});
}
/**
* Delete the data we hold for this repository.
*/
function deleteOldSnapshotForRepository(repoRef, client) {
return __awaiter(this, void 0, void 0, function* () {
yield client.query(`DELETE from repo_fingerprints WHERE repo_snapshot_id IN
(SELECT id from repo_snapshots WHERE url = $1)`, [repoRef.url]);
yield client.query(`DELETE from repo_snapshots WHERE url = $1`, [repoRef.url]);
});
}
// TODO GitHub only
function rowToRepoRef(row) {
return automation_client_1.GitHubRepoRef.from(Object.assign({}, row, { repo: row.name }));
}
//# sourceMappingURL=PostgresProjectAnalysisResultStore.js.map