@atomist/sdm-pack-aspect
Version:
an Atomist SDM Extension Pack for visualizing drift across an organization
314 lines • 14.7 kB
JavaScript
;
/*
* Copyright © 2019 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var g = generator.apply(thisArg, _arguments || []), i, q = [];
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
function fulfill(value) { resume("next", value); }
function reject(value) { resume("throw", value); }
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
};
Object.defineProperty(exports, "__esModule", { value: true });
const automation_client_1 = require("@atomist/automation-client");
const sdm_1 = require("@atomist/sdm");
const Octokit = require("@octokit/rest");
const _ = require("lodash");
const analytics_1 = require("../analytics");
const common_1 = require("../common");
const Spider_1 = require("../Spider");
/**
* Spider GitHub. Ensure that GITHUB_TOKEN environment variable is set.
*/
class GitHubSpider {
constructor(cloner, queryFunction = queryByCriteria) {
this.cloner = cloner;
this.queryFunction = queryFunction;
}
spider(criteria, analyzer, opts) {
var e_1, _a;
return __awaiter(this, void 0, void 0, function* () {
let repoCount = 0;
const keepExisting = [];
const errors = [];
const analyzeAndPersistResults = [];
try {
const it = this.queryFunction(process.env.GITHUB_TOKEN, criteria);
let bucket = [];
function runAllPromisesInBucket() {
return __awaiter(this, void 0, void 0, function* () {
const aResults = yield Promise.all(bucket);
for (const ar of aResults) {
// Avoid hitting the database in parallel to avoid locking
analyzeAndPersistResults.push(yield runPersist(criteria, opts, ar));
}
automation_client_1.logger.debug("Computing analytics over fingerprints...");
yield analytics_1.computeAnalytics(opts.persister, opts.workspaceId);
Spider_1.logTimings(analyzer.timings);
bucket = [];
});
}
try {
for (var it_1 = __asyncValues(it), it_1_1; it_1_1 = yield it_1.next(), !it_1_1.done;) {
const sourceData = it_1_1.value;
++repoCount;
const repo = {
owner: sourceData.owner.login,
repo: sourceData.name,
url: sourceData.url,
};
if (yield common_1.existingRecordShouldBeKept(opts, repo)) {
keepExisting.push(repo.url);
automation_client_1.logger.debug("Found valid record for " + JSON.stringify(repo));
}
else {
automation_client_1.logger.debug("Performing fresh analysis of " + JSON.stringify(repo));
try {
bucket.push(runAnalysis(this.cloner, dropIrrelevantFields(sourceData), criteria, analyzer));
if (bucket.length >= opts.poolSize) {
// Run all promises together. Effectively promise pooling
yield runAllPromisesInBucket();
}
}
catch (err) {
errors.push({
repoUrl: sourceData.url,
whileTryingTo: "clone, analyze, and persist", message: err.message,
});
automation_client_1.logger.error("Failure analyzing repo at %s: %s", sourceData.url, err.message);
}
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (it_1_1 && !it_1_1.done && (_a = it_1.return)) yield _a.call(it_1);
}
finally { if (e_1) throw e_1.error; }
}
yield runAllPromisesInBucket();
}
catch (e) {
automation_client_1.logger.error("Error spidering: %s", e.message);
throw e;
}
const analyzeResults = _.reduce(analyzeAndPersistResults, combineAnalyzeAndPersistResult, emptyAnalyzeAndPersistResult);
return {
repositoriesDetected: repoCount,
projectsDetected: analyzeResults.projectCount,
failed: [...errors,
...analyzeResults.failedToPersist,
...analyzeResults.failedToCloneOrAnalyze],
keptExisting: keepExisting,
persistedAnalyses: analyzeResults.persisted,
};
});
}
}
exports.GitHubSpider = GitHubSpider;
function dropIrrelevantFields(sourceData) {
return {
owner: { login: sourceData.owner.login },
name: sourceData.name,
url: sourceData.url,
html_url: sourceData.html_url,
timestamp: sourceData.timestamp,
query: sourceData.query,
};
}
const emptyAnalyzeAndPersistResult = {
failedToCloneOrAnalyze: [],
failedToPersist: [],
repoCount: 0,
projectCount: 0,
persisted: [],
millisTaken: 0,
};
function combineAnalyzeAndPersistResult(one, two) {
return {
failedToCloneOrAnalyze: one.failedToCloneOrAnalyze.concat(two.failedToCloneOrAnalyze),
failedToPersist: one.failedToPersist.concat(two.failedToPersist),
repoCount: one.repoCount + two.repoCount,
projectCount: one.projectCount + two.projectCount,
persisted: one.persisted.concat(two.persisted),
millisTaken: one.millisTaken + two.millisTaken,
};
}
/**
* Future for doing the work
* @return {Promise<void>}
*/
function runAnalysis(cloner, sourceData, criteria, analyzer) {
return __awaiter(this, void 0, void 0, function* () {
const startTime = new Date().getTime();
let project;
let clonedIn;
try {
project = yield cloner.clone(sourceData);
clonedIn = new Date().getTime() - startTime;
automation_client_1.logger.debug("Successfully cloned %s in %d milliseconds", sourceData.url, clonedIn);
if (!project.id.sha) {
const sha = yield sdm_1.execPromise("git", ["rev-parse", "HEAD"], {
cwd: project.baseDir,
});
project.id.sha = sha.stdout.trim();
automation_client_1.logger.debug(`Set sha to ${project.id.sha}`);
}
}
catch (err) {
return {
failedToCloneOrAnalyze: [{ repoUrl: sourceData.url, whileTryingTo: "clone", message: err.message }],
repoCount: 1,
projectCount: 0,
millisTaken: new Date().getTime() - startTime,
sourceData,
};
}
if (criteria.projectTest && !(yield criteria.projectTest(project))) {
automation_client_1.logger.debug("Skipping analysis of %s as it doesn't pass projectTest", project.id.url);
return {
failedToCloneOrAnalyze: [],
repoCount: 1,
projectCount: 0,
millisTaken: new Date().getTime() - startTime,
sourceData,
};
}
let analyzeResults;
try {
analyzeResults = yield common_1.analyze(project, analyzer, criteria);
const millisTaken = new Date().getTime() - startTime;
automation_client_1.logger.debug("Successfully analyzed %s in %d milliseconds including clone time of %d", sourceData.url, millisTaken, clonedIn);
return {
failedToCloneOrAnalyze: [],
repoCount: 1,
projectCount: 0,
millisTaken,
analyzeResults,
sourceData,
};
}
catch (err) {
automation_client_1.logger.error("Could not analyze " + sourceData.url + ": " + err.message, err);
return {
failedToCloneOrAnalyze: [{ repoUrl: sourceData.url, whileTryingTo: "analyze", message: err.message }],
repoCount: 1,
projectCount: 0,
millisTaken: new Date().getTime() - startTime,
sourceData,
};
}
});
}
function runPersist(criteria, opts, ar) {
return __awaiter(this, void 0, void 0, function* () {
const persistResults = [];
automation_client_1.logger.debug("Persisting...");
if (!ar.analyzeResults) {
return {
failedToCloneOrAnalyze: ar.failedToCloneOrAnalyze,
repoCount: ar.repoCount,
projectCount: ar.projectCount,
failedToPersist: [],
persisted: [],
millisTaken: ar.millisTaken,
};
}
for (const repoInfo of ar.analyzeResults.repoInfos) {
const persistResult = yield common_1.persistRepoInfo(opts, repoInfo, {
sourceData: ar.sourceData,
url: ar.sourceData.html_url,
timestamp: ar.sourceData.timestamp,
query: ar.sourceData.query,
});
persistResults.push(persistResult);
}
return {
failedToCloneOrAnalyze: ar.failedToCloneOrAnalyze,
repoCount: 1,
projectCount: 1,
failedToPersist: _.flatMap(persistResults, r => r.failed),
persisted: _.flatMap(persistResults, p => p.succeeded),
millisTaken: ar.millisTaken,
};
});
}
function queryByCriteria(token, criteria) {
return __asyncGenerator(this, arguments, function* queryByCriteria_1() {
var e_2, _a;
const octokit = new Octokit({
auth: token ? "token " + token : undefined,
baseUrl: "https://api.github.com",
});
let results = [];
let retrieved = 0;
for (const q of criteria.githubQueries) {
automation_client_1.logger.debug("Running query " + q + "...");
const options = octokit.search.repos.endpoint.merge({ q });
try {
for (var _b = __asyncValues(octokit.paginate.iterator(options)), _c; _c = yield __await(_b.next()), !_c.done;) {
const response = _c.value;
retrieved += response.data.length;
const newResults = response.data
.filter((r) => !results.some(existing => existing.full_name === r.full_name));
newResults.forEach((r) => {
r.query = q;
r.timestamp = new Date();
});
for (const newResult of newResults) {
yield yield __await(newResult);
}
automation_client_1.logger.debug(`Looked at ${retrieved} repos of max ${criteria.maxRetrieved}...`);
if (retrieved > criteria.maxRetrieved) {
break;
}
if (results.length > criteria.maxReturned) {
results = results.slice(0, criteria.maxReturned);
break;
}
}
}
catch (e_2_1) { e_2 = { error: e_2_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) yield __await(_a.call(_b));
}
finally { if (e_2) throw e_2.error; }
}
}
});
}
//# sourceMappingURL=GitHubSpider.js.map