renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
241 lines • 9.12 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.GithubGraphqlDatasourceFetcher = void 0;
const tslib_1 = require("tslib");
const global_1 = require("../../../config/global");
const logger_1 = require("../../../logger");
const external_host_error_1 = require("../../../types/errors/external-host-error");
const memCache = tslib_1.__importStar(require("../../cache/memory"));
const packageCache = tslib_1.__importStar(require("../../cache/package"));
const url_1 = require("../url");
const memory_cache_strategy_1 = require("./cache-strategies/memory-cache-strategy");
const package_cache_strategy_1 = require("./cache-strategies/package-cache-strategy");
/**
* We know empirically that certain type of GraphQL errors
* can be fixed by shrinking page size.
*
* @see https://github.com/renovatebot/renovate/issues/16343
*/
function isUnknownGraphqlError(err) {
const { message } = err;
return message.startsWith('Something went wrong while executing your query.');
}
function canBeSolvedByShrinking(err) {
const errors = err instanceof AggregateError ? err.errors : [err];
return errors.some((e) => err instanceof external_host_error_1.ExternalHostError || isUnknownGraphqlError(e));
}
class GithubGraphqlDatasourceFetcher {
http;
datasourceAdapter;
static async query(config, http, adapter) {
const instance = new GithubGraphqlDatasourceFetcher(config, http, adapter);
const items = await instance.getItems();
return items;
}
baseUrl;
repoOwner;
repoName;
itemsPerQuery = 100;
queryCount = 0;
cursor = null;
isPersistent;
constructor(packageConfig, http, datasourceAdapter) {
this.http = http;
this.datasourceAdapter = datasourceAdapter;
const { packageName, registryUrl } = packageConfig;
[this.repoOwner, this.repoName] = packageName.split('/');
this.baseUrl = (0, url_1.getApiBaseUrl)(registryUrl).replace(/\/v3\/$/, '/'); // Replace for GHE
}
getCacheNs() {
return this.datasourceAdapter.key;
}
getCacheKey() {
return [this.baseUrl, this.repoOwner, this.repoName].join(':');
}
getRawQueryOptions() {
const baseUrl = this.baseUrl;
const repository = `${this.repoOwner}/${this.repoName}`;
const query = this.datasourceAdapter.query;
const variables = {
owner: this.repoOwner,
name: this.repoName,
count: this.itemsPerQuery,
cursor: this.cursor,
};
return {
baseUrl,
repository,
readOnly: true,
body: { query, variables },
};
}
async doRawQuery() {
const requestOptions = this.getRawQueryOptions();
let httpRes;
try {
httpRes = await this.http.postJson('/graphql', requestOptions);
}
catch (err) {
return [null, err];
}
const { body } = httpRes;
const { data, errors } = body;
if (errors?.length) {
if (errors.length === 1) {
const { message } = errors[0];
const err = new Error(message);
return [null, err];
}
else {
const errorInstances = errors.map(({ message }) => new Error(message));
const err = new AggregateError(errorInstances);
return [null, err];
}
}
if (!data) {
const msg = 'GitHub GraphQL datasource: failed to obtain data';
const err = new Error(msg);
return [null, err];
}
if (!data.repository) {
const msg = 'GitHub GraphQL datasource: failed to obtain repository data';
const err = new Error(msg);
return [null, err];
}
if (!data.repository.payload) {
const msg = 'GitHub GraphQL datasource: failed to obtain repository payload data';
const err = new Error(msg);
return [null, err];
}
this.queryCount += 1;
// For values other than explicit `false`,
// we assume that items can not be cached.
this.isPersistent ??= data.repository.isRepoPrivate === false;
const res = data.repository.payload;
return [res, null];
}
shrinkPageSize() {
if (this.itemsPerQuery === 100) {
this.itemsPerQuery = 50;
return true;
}
if (this.itemsPerQuery === 50) {
this.itemsPerQuery = 25;
return true;
}
return false;
}
hasReachedQueryLimit() {
return this.queryCount >= 100;
}
async doShrinkableQuery() {
let res = null;
let err = null;
while (!res) {
[res, err] = await this.doRawQuery();
if (err) {
if (!canBeSolvedByShrinking(err)) {
throw err;
}
const shrinkResult = this.shrinkPageSize();
if (!shrinkResult) {
throw err;
}
const { body, ...options } = this.getRawQueryOptions();
logger_1.logger.debug({ options, newSize: this.itemsPerQuery }, 'Shrinking GitHub GraphQL page size after error');
}
}
return res;
}
_cacheStrategy;
cacheStrategy() {
if (this._cacheStrategy) {
return this._cacheStrategy;
}
const cacheNs = this.getCacheNs();
const cacheKey = this.getCacheKey();
const cachePrivatePackages = global_1.GlobalConfig.get('cachePrivatePackages', false);
this._cacheStrategy =
cachePrivatePackages || this.isPersistent
? new package_cache_strategy_1.GithubGraphqlPackageCacheStrategy(cacheNs, cacheKey)
: new memory_cache_strategy_1.GithubGraphqlMemoryCacheStrategy(cacheNs, cacheKey);
return this._cacheStrategy;
}
/**
* This method is responsible for data synchronization.
* It also detects persistence of the package, based on the first page result.
*/
async doPaginatedFetch() {
let hasNextPage = true;
let isPaginationDone = false;
let nextCursor;
while (hasNextPage && !isPaginationDone && !this.hasReachedQueryLimit()) {
const queryResult = await this.doShrinkableQuery();
const resultItems = [];
for (const node of queryResult.nodes) {
const item = this.datasourceAdapter.transform(node);
if (!item) {
logger_1.logger.once.info({
packageName: `${this.repoOwner}/${this.repoName}`,
baseUrl: this.baseUrl,
}, `GitHub GraphQL datasource: skipping empty item`);
continue;
}
resultItems.push(item);
}
// It's important to call `getCacheStrategy()` after `doShrinkableQuery()`
// because `doShrinkableQuery()` may change `this.isCacheable`.
//
// Otherwise, cache items for public packages will never be persisted
// in long-term cache.
isPaginationDone = await this.cacheStrategy().reconcile(resultItems);
hasNextPage = !!queryResult?.pageInfo?.hasNextPage;
nextCursor = queryResult?.pageInfo?.endCursor;
if (hasNextPage && nextCursor) {
this.cursor = nextCursor;
}
}
if (this.isPersistent) {
await this.storePersistenceFlag(30);
}
}
async doCachedQuery() {
await this.loadPersistenceFlag();
if (!this.isPersistent) {
await this.doPaginatedFetch();
}
const res = await this.cacheStrategy().finalizeAndReturn();
if (res.length) {
return res;
}
delete this.isPersistent;
await this.doPaginatedFetch();
return this.cacheStrategy().finalizeAndReturn();
}
async loadPersistenceFlag() {
const ns = this.getCacheNs();
const key = `${this.getCacheKey()}:is-persistent`;
this.isPersistent = await packageCache.get(ns, key);
}
async storePersistenceFlag(minutes) {
const ns = this.getCacheNs();
const key = `${this.getCacheKey()}:is-persistent`;
await packageCache.set(ns, key, true, minutes);
}
/**
* This method ensures the only one query is executed
* to a particular package during single run.
*/
doUniqueQuery() {
const cacheKey = `github-pending:${this.getCacheNs()}:${this.getCacheKey()}`;
const resultPromise = memCache.get(cacheKey) ?? this.doCachedQuery();
memCache.set(cacheKey, resultPromise);
return resultPromise;
}
async getItems() {
const res = await this.doUniqueQuery();
return res;
}
}
exports.GithubGraphqlDatasourceFetcher = GithubGraphqlDatasourceFetcher;
//# sourceMappingURL=datasource-fetcher.js.map