renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
390 lines • 16.8 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.GithubHttp = exports.setBaseUrl = void 0;
const tslib_1 = require("tslib");
const is_1 = tslib_1.__importDefault(require("@sindresorhus/is"));
const luxon_1 = require("luxon");
const error_messages_1 = require("../../constants/error-messages");
const logger_1 = require("../../logger");
const external_host_error_1 = require("../../types/errors/external-host-error");
const repository_1 = require("../cache/repository");
const env_1 = require("../env");
const mask_1 = require("../mask");
const p = tslib_1.__importStar(require("../promises"));
const range_1 = require("../range");
const regex_1 = require("../regex");
const url_1 = require("../url");
const host_rules_1 = require("./host-rules");
const http_1 = require("./http");
const githubBaseUrl = 'https://api.github.com/';
let baseUrl = githubBaseUrl;
const setBaseUrl = (url) => {
baseUrl = url;
};
exports.setBaseUrl = setBaseUrl;
function handleGotError(err, url, opts) {
const path = url.toString();
let message = err.message || '';
const body = err.response?.body;
if (is_1.default.plainObject(body) && 'message' in body) {
message = String(body.message);
}
if (err.code === 'ERR_HTTP2_STREAM_ERROR' ||
err.code === 'ENOTFOUND' ||
err.code === 'ETIMEDOUT' ||
err.code === 'EAI_AGAIN' ||
err.code === 'ECONNRESET') {
logger_1.logger.debug({ err }, 'GitHub failure: RequestError');
return new external_host_error_1.ExternalHostError(err, 'github');
}
if (err.name === 'ParseError') {
logger_1.logger.debug({ err }, '');
return new external_host_error_1.ExternalHostError(err, 'github');
}
if (err.statusCode && err.statusCode >= 500 && err.statusCode < 600) {
logger_1.logger.debug({ err }, 'GitHub failure: 5xx');
return new external_host_error_1.ExternalHostError(err, 'github');
}
if (err.statusCode === 403 &&
message.startsWith('You have triggered an abuse detection mechanism')) {
logger_1.logger.debug({ err }, 'GitHub failure: abuse detection');
return new Error(error_messages_1.PLATFORM_RATE_LIMIT_EXCEEDED);
}
if (err.statusCode === 403 &&
message.startsWith('You have exceeded a secondary rate limit')) {
logger_1.logger.warn({ err }, 'GitHub failure: secondary rate limit');
return new Error(error_messages_1.PLATFORM_RATE_LIMIT_EXCEEDED);
}
if (err.statusCode === 403 && message.includes('Upgrade to GitHub Pro')) {
logger_1.logger.debug(`Endpoint: ${path}, needs paid GitHub plan`);
return err;
}
if (err.statusCode === 403 && message.includes('rate limit exceeded')) {
logger_1.logger.debug({ err }, 'GitHub failure: rate limit');
return new Error(error_messages_1.PLATFORM_RATE_LIMIT_EXCEEDED);
}
if (err.statusCode === 403 &&
message.startsWith('Resource not accessible by integration')) {
logger_1.logger.debug({ err }, 'GitHub failure: Resource not accessible by integration');
return new Error(error_messages_1.PLATFORM_INTEGRATION_UNAUTHORIZED);
}
if (err.statusCode === 401) {
// Warn once for github.com token if unauthorized
const hostname = (0, url_1.parseUrl)(url)?.hostname;
if (hostname === 'github.com' || hostname === 'api.github.com') {
logger_1.logger.once.warn('github.com token 401 unauthorized');
}
if (message.includes('Bad credentials')) {
const rateLimit = err.headers?.['x-ratelimit-limit'] ?? -1;
logger_1.logger.debug({
token: (0, mask_1.maskToken)(opts.token),
err,
}, 'GitHub failure: Bad credentials');
if (rateLimit === '60') {
return new external_host_error_1.ExternalHostError(err, 'github');
}
return new Error(error_messages_1.PLATFORM_BAD_CREDENTIALS);
}
}
if (err.statusCode === 422) {
if (message.includes('Review cannot be requested from pull request author')) {
return err;
}
else if (err.body?.errors?.find((e) => e.field === 'milestone')) {
return err;
}
else if (err.body?.errors?.find((e) => e.code === 'invalid')) {
logger_1.logger.debug({ err }, 'Received invalid response - aborting');
return new Error(error_messages_1.REPOSITORY_CHANGED);
}
else if (err.body?.errors?.find((e) => e.message?.startsWith('A pull request already exists'))) {
return err;
}
logger_1.logger.debug({ err }, '422 Error thrown from GitHub');
return new external_host_error_1.ExternalHostError(err, 'github');
}
if (err.statusCode === 410 &&
err.body?.message === 'Issues are disabled for this repo') {
return err;
}
return err;
}
function constructAcceptString(input) {
const defaultAccept = 'application/vnd.github.v3+json';
const acceptStrings = typeof input === 'string' ? input.split((0, regex_1.regEx)(/\s*,\s*/)) : [];
// TODO: regression of #6736
if (!acceptStrings.some((x) => x === defaultAccept) &&
(!acceptStrings.some((x) => x.startsWith('application/vnd.github.')) ||
acceptStrings.length < 2)) {
acceptStrings.push(defaultAccept);
}
return acceptStrings.join(', ');
}
const MAX_GRAPHQL_PAGE_SIZE = 100;
function getGraphqlPageSize(fieldName, defaultPageSize = MAX_GRAPHQL_PAGE_SIZE) {
const cache = (0, repository_1.getCache)();
const graphqlPageCache = cache?.platform?.github
?.graphqlPageCache;
const cachedRecord = graphqlPageCache?.[fieldName];
if (graphqlPageCache && cachedRecord) {
logger_1.logger.debug({ fieldName, ...cachedRecord }, 'GraphQL page size: found cached value');
const oldPageSize = cachedRecord.pageSize;
const now = luxon_1.DateTime.local();
const then = luxon_1.DateTime.fromISO(cachedRecord.pageLastResizedAt);
const expiry = then.plus({ hours: 24 });
if (now > expiry) {
const newPageSize = Math.min(oldPageSize * 2, MAX_GRAPHQL_PAGE_SIZE);
if (newPageSize < MAX_GRAPHQL_PAGE_SIZE) {
const timestamp = now.toISO();
logger_1.logger.debug({ fieldName, oldPageSize, newPageSize, timestamp }, 'GraphQL page size: expanding');
cachedRecord.pageLastResizedAt = timestamp;
cachedRecord.pageSize = newPageSize;
}
else {
logger_1.logger.debug({ fieldName, oldPageSize, newPageSize }, 'GraphQL page size: expanded to default page size');
delete graphqlPageCache[fieldName];
}
return newPageSize;
}
return oldPageSize;
}
return defaultPageSize;
}
function setGraphqlPageSize(fieldName, newPageSize) {
const oldPageSize = getGraphqlPageSize(fieldName);
if (newPageSize !== oldPageSize) {
const now = luxon_1.DateTime.local();
const pageLastResizedAt = now.toISO();
logger_1.logger.debug({ fieldName, oldPageSize, newPageSize, timestamp: pageLastResizedAt }, 'GraphQL page size: shrinking');
const cache = (0, repository_1.getCache)();
cache.platform ??= {};
cache.platform.github ??= {};
cache.platform.github.graphqlPageCache ??= {};
const graphqlPageCache = cache.platform.github
.graphqlPageCache;
graphqlPageCache[fieldName] = {
pageLastResizedAt,
pageSize: newPageSize,
};
}
}
function replaceUrlBase(url, baseUrl) {
const relativeUrl = `${url.pathname}${url.search}`;
return new URL(relativeUrl, baseUrl);
}
class GithubHttp extends http_1.HttpBase {
get baseUrl() {
return baseUrl;
}
constructor(hostType = 'github', options) {
super(hostType, options);
}
processOptions(url, opts) {
if (!opts.token) {
const authUrl = new URL(url);
if (opts.repository) {
// set authUrl to https://api.github.com/repos/org/repo or https://gihub.domain.com/api/v3/repos/org/repo
authUrl.hash = '';
authUrl.search = '';
authUrl.pathname = (0, url_1.joinUrlParts)(authUrl.pathname.startsWith('/api/v3') ? '/api/v3' : '', 'repos', `${opts.repository}`);
}
let readOnly = opts.readOnly;
const { method = 'get' } = opts;
if (readOnly === undefined &&
['get', 'head'].includes(method.toLowerCase())) {
readOnly = true;
}
const { token } = (0, host_rules_1.findMatchingRule)(authUrl.toString(), {
hostType: this.hostType,
readOnly,
});
opts.token = token;
}
const accept = constructAcceptString(opts.headers?.accept);
opts.headers = {
...opts.headers,
accept,
};
}
handleError(url, opts, err) {
throw handleGotError(err, url, opts);
}
async requestJsonUnsafe(method, options) {
const httpOptions = options.httpOptions ?? {};
const resolvedUrl = this.resolveUrl(options.url, httpOptions);
const opts = {
...options,
url: resolvedUrl,
};
const result = await super.requestJsonUnsafe(method, opts);
if (httpOptions.paginate) {
delete httpOptions.cacheProvider;
httpOptions.memCache = false;
// Check if result is paginated
const pageLimit = httpOptions.pageLimit ?? 10;
const linkHeader = (0, url_1.parseLinkHeader)(result?.headers?.link);
const next = linkHeader?.next;
const env = (0, env_1.getEnv)();
if (next?.url && linkHeader?.last?.page) {
let lastPage = parseInt(linkHeader.last.page);
if (!env.RENOVATE_PAGINATE_ALL && httpOptions.paginate !== 'all') {
lastPage = Math.min(pageLimit, lastPage);
}
const baseUrl = httpOptions.baseUrl ?? this.baseUrl;
const parsedUrl = new URL(next.url, baseUrl);
const rebasePagination = !!baseUrl &&
!!env.RENOVATE_X_REBASE_PAGINATION_LINKS &&
// Preserve github.com URLs for use cases like release notes
parsedUrl.origin !== 'https://api.github.com';
const firstPageUrl = rebasePagination
? replaceUrlBase(parsedUrl, baseUrl)
: parsedUrl;
const queue = [...(0, range_1.range)(2, lastPage)].map((pageNumber) => () => {
// copy before modifying searchParams
const nextUrl = new URL(firstPageUrl);
nextUrl.searchParams.set('page', String(pageNumber));
return super.requestJsonUnsafe(method, {
...opts,
url: nextUrl,
});
});
const pages = await p.all(queue);
if (httpOptions.paginationField && is_1.default.plainObject(result.body)) {
const paginatedResult = result.body[httpOptions.paginationField];
if (is_1.default.array(paginatedResult)) {
for (const nextPage of pages) {
if (is_1.default.plainObject(nextPage.body)) {
const nextPageResults = nextPage.body[httpOptions.paginationField];
if (is_1.default.array(nextPageResults)) {
paginatedResult.push(...nextPageResults);
}
}
}
}
}
else if (is_1.default.array(result.body)) {
for (const nextPage of pages) {
if (is_1.default.array(nextPage.body)) {
result.body.push(...nextPage.body);
}
}
}
}
}
return result;
}
async requestGraphql(query, options = {}) {
const path = 'graphql';
const { paginate, count = MAX_GRAPHQL_PAGE_SIZE, cursor = null } = options;
let { variables } = options;
if (paginate) {
variables = {
...variables,
count,
cursor,
};
}
const body = variables ? { query, variables } : { query };
const opts = {
baseUrl: baseUrl.replace('/v3/', '/'), // GHE uses unversioned graphql path
body,
headers: { accept: options?.acceptHeader },
readOnly: options.readOnly,
};
if (options.token) {
opts.token = options.token;
}
logger_1.logger.trace(`Performing Github GraphQL request`);
try {
const res = await this.postJson(path, opts);
return res?.body;
}
catch (err) {
logger_1.logger.debug({ err, query, options }, 'Unexpected GraphQL Error');
if (err instanceof external_host_error_1.ExternalHostError && count && count > 10) {
logger_1.logger.info('Reducing pagination count to workaround graphql errors');
return null;
}
throw handleGotError(err, path, opts);
}
}
async queryRepoField(query, fieldName, options = {}) {
const result = [];
const { paginate = true } = options;
let optimalCount = null;
let count = getGraphqlPageSize(fieldName, options.count ?? MAX_GRAPHQL_PAGE_SIZE);
let limit = options.limit ?? 1000;
let cursor = null;
let isIterating = true;
while (isIterating) {
const res = await this.requestGraphql(query, {
...options,
count: Math.min(count, limit),
cursor,
paginate,
});
const repositoryData = res?.data?.repository;
if (is_1.default.nonEmptyObject(repositoryData) &&
!is_1.default.nullOrUndefined(repositoryData[fieldName])) {
optimalCount = count;
const { nodes = [], edges = [], pageInfo, } = repositoryData[fieldName];
result.push(...nodes);
result.push(...edges);
limit = Math.max(0, limit - nodes.length - edges.length);
if (limit === 0) {
isIterating = false;
}
else if (paginate && pageInfo) {
const { hasNextPage, endCursor } = pageInfo;
if (hasNextPage && endCursor) {
cursor = endCursor;
}
else {
isIterating = false;
}
}
}
else {
count = Math.floor(count / 2);
if (count === 0) {
logger_1.logger.warn({ query, options, res }, 'Error fetching GraphQL nodes');
isIterating = false;
}
}
if (!paginate) {
isIterating = false;
}
}
if (optimalCount && optimalCount < MAX_GRAPHQL_PAGE_SIZE) {
setGraphqlPageSize(fieldName, optimalCount);
}
return result;
}
/**
* Get the raw text file from a URL.
* Only use this method to fetch text files.
*
* @param url Full API URL, contents path or path inside the repository to the file
* @param options
*
* @example url = 'https://api.github.com/repos/renovatebot/renovate/contents/package.json'
* @example url = 'renovatebot/renovate/contents/package.json'
* @example url = 'package.json' & options.repository = 'renovatebot/renovate'
*/
async getRawTextFile(url, options = {}) {
const newOptions = {
...options,
headers: {
accept: 'application/vnd.github.raw+json',
},
};
let newURL = url;
const httpRegex = (0, regex_1.regEx)(/^https?:\/\//);
if (options.repository && !httpRegex.test(options.repository)) {
newURL = (0, url_1.joinUrlParts)(options.repository, 'contents', url);
}
return await this.getText(newURL, newOptions);
}
}
exports.GithubHttp = GithubHttp;
//# sourceMappingURL=github.js.map