@codefresh-io/cf-git-providers
Version:
An NPM module/CLI for interacting with various git providers
711 lines • 27.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const plugin_retry_1 = require("@octokit/plugin-retry");
const rest_1 = require("@octokit/rest");
const lodash_1 = require("lodash");
const undici_1 = require("undici");
const url_1 = require("url");
const helpers_1 = require("../helpers");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { RequestError } = require('@octokit/request-error');
// eslint-disable-next-line @typescript-eslint/no-var-requires
const CFError = require('cf-errors');
const logger = (0, helpers_1.createNewLogger)('codefresh:infra:git-providers:github');
const MAX_PER_PAGE = 100;
const MAX_CONCURRENT_REQUESTS = 4; // requests X pages each time
const MAX_RESULTS = 10000;
const GITHUB_CLOUD_API_URL = 'api.github.com';
const GITHUB_REST_ENDPOINT = '/api/v3';
const GITHUB_FINEGRAINED_TOKEN_LENGTH = 93;
const GITHUB_FINEGRAINED_TOKEN_PREFIX = 'github_pat_';
const UNAUTHORIZED_STATUS_CODE = 403;
const TOO_MANY_REQUESTS_STATUS_CODE = 429;
const statesMap = {
pending: 'pending',
running: 'pending',
success: 'success',
failure: 'failure',
error: 'error',
};
const scopesMap = {
repo_read: 'repo',
repo_write: 'repo',
repo_create: 'repo',
admin_repo_hook: 'admin:repo_hook',
};
const _cleanFilePath = (filepath) => {
const prefix = './';
const hasPrefix = filepath.indexOf(prefix) === 0;
const path = hasPrefix ? filepath.slice(prefix.length) : filepath;
return path;
};
const _toBranch = (rawBranch) => {
return {
id: rawBranch.name,
name: rawBranch.name,
commit: {
sha: (0, lodash_1.get)(rawBranch, 'commit.sha'),
commiter_name: (0, lodash_1.get)(rawBranch, 'commit.commit.author.name'),
message: (0, lodash_1.get)(rawBranch, 'commit.commit.message'),
url: (0, lodash_1.get)(rawBranch, 'commit.commit.url'),
}
};
};
const _toUser = (user) => {
return {
login: user.login,
email: user.email,
avatar_url: user.avatar_url,
web_url: user.html_url,
};
};
const _toRepo = (rawRepo) => {
return {
id: rawRepo.id.toString(),
provider: 'github',
name: rawRepo.name,
full_name: rawRepo.full_name,
private: rawRepo.private,
pushed_at: rawRepo.pushed_at,
open_issues: rawRepo.open_issues_count,
clone_url: rawRepo.clone_url,
ssh_url: rawRepo.ssh_url,
owner: {
login: (0, lodash_1.get)(rawRepo, 'owner.login', ''),
avatar_url: (0, lodash_1.get)(rawRepo, 'owner.avatar_url', ''),
creator: null,
},
org: (0, lodash_1.get)(rawRepo, 'organization.login', null),
default_branch: rawRepo.default_branch,
permissions: {
admin: (0, lodash_1.get)(rawRepo, 'permissions.admin', null),
},
webUrl: (0, lodash_1.get)(rawRepo, 'html_url', ''),
};
};
const _toWebhook = (rawWebhook, owner, repo) => {
return {
id: rawWebhook.id,
name: rawWebhook.name,
events: rawWebhook.events,
endpoint: (0, lodash_1.get)(rawWebhook, 'config.url', ''),
repo: `${owner}/${repo}`,
};
};
const _toPullRequest = (rawPullRequest) => {
return {
id: (0, lodash_1.get)(rawPullRequest, 'number', ''),
url: (0, lodash_1.get)(rawPullRequest, 'html_url', ''),
labels: (0, lodash_1.get)(rawPullRequest, 'labels', []).map((label) => {
return String((0, lodash_1.get)(label, 'name'));
}).join(', '),
title: (0, lodash_1.get)(rawPullRequest, 'title', ''),
body: (0, lodash_1.get)(rawPullRequest, 'body', ''),
state: (0, lodash_1.get)(rawPullRequest, 'state', ''),
isMerged: (0, lodash_1.get)(rawPullRequest, 'merged', false),
mergeCommitSHA: rawPullRequest.merge_commit_sha,
createdAt: (0, lodash_1.get)(rawPullRequest, 'created_at', ''),
userLogin: (0, lodash_1.get)(rawPullRequest, 'user.login', ''),
userAvatarUrl: (0, lodash_1.get)(rawPullRequest, 'user.avatar_url', ''),
isFork: (0, lodash_1.get)(rawPullRequest, 'head.repo.fork', false),
headRepo: (0, lodash_1.get)(rawPullRequest, 'head.repo.full_name', ''),
headRepoUrl: (0, lodash_1.get)(rawPullRequest, 'head.repo.clone_url', ''),
headBranch: (0, lodash_1.get)(rawPullRequest, 'head.ref', ''),
headCommitSHA: (0, lodash_1.get)(rawPullRequest, 'head.sha', ''),
baseRepo: (0, lodash_1.get)(rawPullRequest, 'base.repo.full_name', ''),
baseRepoUrl: (0, lodash_1.get)(rawPullRequest, 'base.repo.clone_url', ''),
baseBranch: (0, lodash_1.get)(rawPullRequest, 'base.ref', ''),
};
};
class Github {
static tokenResetMemory = new Map();
githubClient;
auth;
constructor(opt) {
const baseUrl = new url_1.URL(opt.apiUrl || `https://${opt.apiHost || GITHUB_CLOUD_API_URL}`);
if (opt.apiPathPrefix) {
baseUrl.pathname = opt.apiPathPrefix;
}
else if (baseUrl.hostname !== GITHUB_CLOUD_API_URL && baseUrl.pathname == '/') {
// only upadte pathname if there is no current pathname
baseUrl.pathname = GITHUB_REST_ENDPOINT;
}
let strBaseUrl = baseUrl.toString();
if (strBaseUrl.endsWith('/')) {
strBaseUrl = strBaseUrl.slice(0, -1);
}
// The option rejectUnauthorized: true is enabling an TLS verification.
// And we disabled it by default for EnvHttpProxyAgent for some reason - get(opt, 'insecure', false).
// We need to check the codebase and enable TLS verification. https://codefresh-io.atlassian.net/browse/CR-28794
const secure = process.env.NODE_TLS_REJECT_UNAUTHORIZED === '0' ? false : (0, lodash_1.get)(opt, 'insecure', false);
const agentOptions = {
pipelining: 0,
connections: 10,
keepAliveTimeout: opt.timeout || 10000,
connect: {
rejectUnauthorized: secure,
},
};
const agent = new undici_1.EnvHttpProxyAgent(agentOptions);
const myFetch = (url, options) => {
return (0, undici_1.fetch)(url, {
...options,
dispatcher: agent,
});
};
const GithubClient = rest_1.Octokit.plugin(plugin_retry_1.retry);
this.auth = {
username: 'github',
password: opt.password,
};
this.githubClient = new GithubClient({
userAgent: 'Codefresh',
auth: opt.password,
request: {
fetch: myFetch,
},
timeout: opt.timeout || 10000,
host: opt.apiHost || 'api.github.com',
log: logger,
debug: true,
baseUrl: strBaseUrl,
});
}
getName() {
return 'github';
}
async getPaginatedResults(apiFunc, args, limit = MAX_RESULTS, page = 1) {
const finalResults = [];
let startingPage = page;
let shouldStop = false;
while (!shouldStop) {
const neededResults = limit - finalResults.length;
// make up to MAX_PAGES_PER_CYCLE requests in parallel
const neededPages = Math.min((0, lodash_1.ceil)(neededResults / MAX_PER_PAGE), MAX_CONCURRENT_REQUESTS);
const pagePromises = (0, lodash_1.range)(startingPage, startingPage + neededPages).map((pageNumber) => apiFunc({
...args,
page: pageNumber,
per_page: MAX_PER_PAGE,
}));
// harvest results
const pagesResults = await Promise.all(pagePromises);
for (const res of pagesResults) {
const currentNeededResults = limit - finalResults.length;
const data = (0, lodash_1.get)(res, 'data', []);
finalResults.push(...data.slice(0, currentNeededResults));
if (limit === finalResults.length) {
shouldStop = true;
break; // got all the results we need, no reason to process other requests
}
if ((0, lodash_1.isEmpty)(data) || (0, lodash_1.size)(data) < MAX_PER_PAGE) {
shouldStop = true;
break; // last page, no reason to process other requests
}
}
// next batch
startingPage += neededPages;
}
return {
data: finalResults,
status: 200,
};
}
async fetchRawFile(opt) {
const { owner, repo, ref, path } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.getContent({ owner, repo, path: _cleanFilePath(path), ref }));
if (err) {
throw new CFError({
message: `Failed to retrieve file: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to retrieve file: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
const data = res.data;
const base64Content = data.content.replace(/\n/gi, '');
return Buffer.from(base64Content, 'base64').toString();
}
async getBranch(opt) {
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.getBranch(opt));
if (err) {
throw new CFError({
message: `Failed to retrieve branch: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to retrieve branch: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return _toBranch(res.data);
}
async getRepository(opt) {
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.get(opt));
if (err) {
throw new CFError({
message: `Failed to retrieve repository: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to retrieve repository: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return _toRepo(res.data);
}
async createRepository(opt) {
const { login } = await this.getUser();
let res, err;
if (login === opt.owner) {
[err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.createForAuthenticatedUser({ name: opt.repo, auto_init: opt.autoInit, private: opt.private }));
}
else {
[err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.createInOrg({ org: opt.owner, name: opt.repo, auto_init: opt.autoInit, private: opt.private }));
}
if (err) {
throw new CFError({
message: `Failed to create repository: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to create repository: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return _toRepo(res.data);
}
async listBranches(opt) {
const { owner, repo, limit, page } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.getPaginatedResults(this.githubClient.repos.listBranches, { owner, repo }, limit, page));
if (err) {
throw new CFError({
message: `Failed to list branches: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to list branches: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return res.data.map(_toBranch);
}
async createBranch(opt) {
const { owner, repo, branch, sha } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.rest.git.createRef({
owner,
repo,
ref: `refs/heads/${branch}`,
sha
}));
if (err) {
throw new CFError({
message: `Failed to create branch: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to create branch: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return _toBranch(res.data);
}
async listRepositoriesForOwner(opt) {
const { owner, sort, direction, limit, page } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.getPaginatedResults(this.githubClient.repos.listForUser, { username: owner, sort, direction }, limit, page));
if (err) {
throw new CFError({
message: `Failed to list repos: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to list repos: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return res.data.map(_toRepo);
}
async listRepositoriesWithAffiliation(opt) {
const { limit, page, ...args } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.getPaginatedResults(this.githubClient.repos.listForAuthenticatedUser, args, limit, page));
if (err) {
throw new CFError({
message: `Failed to list repos: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to list repos: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return res.data.map(_toRepo);
}
async listRepositoriesForOrganization(opt) {
const { limit, page, organization, sort, direction } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.getPaginatedResults(this.githubClient.repos.listForOrg, { org: organization, sort, direction }, limit, page));
if (err) {
throw new CFError({
message: `Failed to list repos: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to list repos: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return res.data.map(_toRepo);
}
async listWebhooks(opt) {
const { limit, page, ...args } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.getPaginatedResults(this.githubClient.repos.listWebhooks, args, limit, page));
if (err) {
throw new CFError({
message: `Failed to list repository webhooks: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to list repository webhooks: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return res.data.map((rawWebHook) => _toWebhook(rawWebHook, args.owner, args.repo));
}
async createRepositoryWebhook(opt) {
const { owner, repo, endpoint, secret } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.createWebhook({
owner,
repo,
name: 'web',
config: {
url: endpoint,
content_type: 'json',
secret,
},
events: [
'push',
'pull_request',
'release',
'issue_comment',
'repository'
],
active: true
}));
if (err) {
throw new CFError({
message: `Failed to create repository webhook: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to create repository webhook: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return _toWebhook(res.data, owner, repo);
}
async deleteRepositoryWebhook(opt) {
const { owner, repo, hookId } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.deleteWebhook({
owner,
repo,
hook_id: hookId,
}));
if (err) {
throw new CFError({
message: `Failed to delete repository webhook: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to delete repository webhook: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
}
async listOrganizations(opt) {
const { limit, page } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.getPaginatedResults(this.githubClient.orgs.listForAuthenticatedUser, {}, limit, page));
if (err) {
throw new CFError({
message: `Failed to list organizations: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to list organizations: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return res.data.map((org) => org.login);
}
async createCommitStatus(opt) {
const { owner, repo, sha, targetUrl, state, description, context } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.createCommitStatus({
owner,
repo,
sha,
target_url: targetUrl,
state: statesMap[state],
description,
context,
}));
if (err) {
throw new CFError({
message: `Failed to create commit status: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to create commit status: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
}
async getPullRequestFiles(opt) {
const { owner, repo, pullNumber } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.pulls.listFiles({
owner,
repo,
pull_number: pullNumber
}));
if (err) {
throw new CFError({
message: `Failed to get pull request files: ${JSON.stringify(opt)}`,
cause: err,
});
}
return res.data.map((file) => file.filename);
}
async getPullRequest(opt) {
const { owner, repo, pullNumber } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.pulls.get({
owner,
repo,
pull_number: pullNumber
}));
if (err) {
throw new CFError({
message: `Failed to get pull request files: ${JSON.stringify(opt)}`,
cause: err,
});
}
return _toPullRequest(res.data);
}
async searchMergedPullRequestByCommitSha(opt) {
const { owner, repo, sha } = opt;
try {
const q = this.constructSearchQuery({
sha,
repo: `${owner}/${repo}`,
type: 'pr',
is: 'merged',
});
const { data: { items } } = await this.githubClient.search.issuesAndPullRequests({
q
});
if (items.length === 0) {
return undefined;
}
return _toPullRequest(items[0]);
}
catch (error) {
throw new CFError({
message: `Failed to get pull request by sha: ${sha}`,
cause: error,
});
}
}
async createPullRequest(opt) {
const { owner, repo, title, body, head, base } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.pulls.create({
owner,
repo,
title,
body,
head,
base,
}));
if (err) {
throw new CFError({
message: `Failed to create pull request: ${JSON.stringify(opt)}`,
cause: err,
});
}
return _toPullRequest(res.data);
}
async getUser(opt) {
const [err, res] = await this.rateLimitAssertionWrapper(async () => {
if (opt?.username) {
return await this.githubClient.users.getByUsername({ username: opt.username });
}
else {
return await this.githubClient.users.getAuthenticated();
}
});
if (err) {
throw new CFError({
message: `Failed to get user: ${JSON.stringify(opt)}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to get user: ${JSON.stringify(opt)}, status code: ${res.status}`);
}
return _toUser(res.data);
}
async getUserByEmail(email) {
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.search.users({ q: email }));
if (err) {
throw new CFError({
message: `Failed to get user: ${email}`,
cause: err,
});
}
else if (res.status >= 400) {
throw new CFError(`Failed to get user: ${email}, status code: ${res.status}`);
}
if (res.data.items.length === 0) {
throw new CFError(`No user with email: ${email} was found on ${this.getName()}`);
}
return _toUser(res.data.items[0]);
}
async getRepositoryPermissions(opt) {
const { owner, repo } = opt;
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.repos.get({ owner, repo }));
if (err) {
if (err.status) {
if (err.status !== 404 && err.status !== 401) {
throw new CFError({
message: `Failed to get repository permissions: ${owner}/${repo}`,
cause: err
});
}
return { read: false, write: false };
}
throw new CFError({
message: `Failed to get repository permissions: ${owner}/${repo}: ${err}`,
cause: err,
});
}
return {
read: (0, lodash_1.get)(res.data, 'permissions.pull', false),
write: (0, lodash_1.get)(res.data, 'permissions.push', false),
};
}
async assertApiScopes(opt) {
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.request('/user'));
if (err) {
throw new CFError({
message: `ValidationError: failed asserting api scopes`,
cause: err
});
}
if (!res.headers['x-oauth-scopes']) {
throw new CFError({
message: `ValidationError: missing scopes: ${opt.scopes.toString()}`,
cause: err
});
}
const originalScopes = res.headers['x-oauth-scopes'].replace(/ /g, '').split(',');
const isValid = opt.scopes.every(val => originalScopes.includes(scopesMap[val]));
if (!isValid) {
throw new CFError({
message: `ValidationError: got scopes ${res.headers['x-oauth-scopes'].toString()} while expected: ${opt.scopes.toString()}`,
cause: err
});
}
}
async validateToken() {
const [err, res] = await this.rateLimitAssertionWrapper(() => this.githubClient.request('/user'));
if (err) {
throw new CFError({
message: `ValidationError: ${err}`,
cause: err
});
}
if (res.statusCode == 401) {
throw new CFError(`ValidationError: token is invalid`);
}
}
skipPermissionsValidation() {
if (this.isTokenFineGrained()) {
return {
skip: true,
reason: 'the token is Fine-Grained'
};
}
return { skip: false };
}
isTokenFineGrained() {
const token = this.auth.password;
return token.length === GITHUB_FINEGRAINED_TOKEN_LENGTH
&& token.startsWith(GITHUB_FINEGRAINED_TOKEN_PREFIX);
}
toOwnerRepo(fullRepoName) {
const [owner, ...repoParts] = fullRepoName.split('/');
return [owner, repoParts.join('/')];
}
getAuth() {
return this.auth;
}
isTokenMutable() {
return true;
}
requiresRepoToCheckTokenScopes() {
return false;
}
useAdminForUserPermission() {
return false;
}
constructSearchQuery(qualifiers) {
return Object.entries(qualifiers).map(([key, value]) => `${key}:${value}`).join('+');
}
getRateLimit(headers) {
if (!headers)
return undefined;
return {
limit: headers['x-ratelimit-limit'],
used: headers['x-ratelimit-used'],
remaining: headers['x-ratelimit-remaining'],
reset: headers['x-ratelimit-reset'],
};
}
createRateLimitError(resetTime) {
return new RequestError('Reached rate limit, skip api call', UNAUTHORIZED_STATUS_CODE, {
response: {
status: UNAUTHORIZED_STATUS_CODE,
url: 'https://api.github.com/rate_limit',
headers: {
'x-ratelimit-limit': '5000',
'x-ratelimit-used': '5000',
'x-ratelimit-remaining': '0',
'x-ratelimit-reset': String(resetTime),
},
data: {
message: 'API rate limit exceeded',
documentation_url: 'https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting',
},
},
request: {
method: 'GET',
url: 'https://api.github.com/rate_limit',
headers: {},
},
});
}
reachedRateLimit(err, res) {
const headers = err?.response?.headers ?? res?.headers;
const rateLimit = this.getRateLimit(headers);
if (!rateLimit) {
return 0;
}
const { remaining, reset } = rateLimit;
if (err &&
[UNAUTHORIZED_STATUS_CODE, TOO_MANY_REQUESTS_STATUS_CODE].includes(err?.status) &&
err.message?.includes('rate limit')) {
return reset;
}
else if (remaining <= 1) {
return reset;
}
else {
return 0;
}
}
async rateLimitAssertionWrapper(fn) {
try {
helpers_1.RateLimitManager.assert(this.auth.password, this.createRateLimitError);
}
catch (error) {
return [error, null];
}
const [err, res] = await (0, helpers_1.to)(fn());
helpers_1.RateLimitManager.set(this.auth.password, this.reachedRateLimit(err, res));
return [err, res];
}
}
exports.default = Github;
//# sourceMappingURL=github.js.map