renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
225 lines • 8.66 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.VersionsEndpointCache = exports.memCache = void 0;
const zod_1 = require("zod");
const logger_1 = require("../../../logger");
const date_1 = require("../../../util/date");
const http_1 = require("../../../util/http");
const regex_1 = require("../../../util/regex");
const result_1 = require("../../../util/result");
const schema_utils_1 = require("../../../util/schema-utils");
const string_1 = require("../../../util/string");
const url_1 = require("../../../util/url");
function getContentTail(content) {
return content.slice(-33);
}
function getContentHead(content) {
return content.slice(0, 33);
}
function stripContentHead(content) {
return content.slice(33);
}
function reconcilePackageVersions(packageVersions, versionLines) {
for (const line of versionLines) {
const packageName = (0, string_1.copystr)(line.packageName);
let versions = packageVersions.get(packageName) ?? [];
const { deletedVersions, addedVersions } = line;
if (deletedVersions.size > 0) {
versions = versions.filter((v) => !deletedVersions.has(v));
}
if (addedVersions.length > 0) {
const existingVersions = new Set(versions);
for (const addedVersion of addedVersions) {
if (!existingVersions.has(addedVersion)) {
const version = (0, string_1.copystr)(addedVersion);
versions.push(version);
}
}
}
packageVersions.set(packageName, versions);
}
return packageVersions;
}
function parseFullBody(body) {
const packageVersions = reconcilePackageVersions(new Map(), VersionLines.parse(body));
const syncedAt = new Date();
const contentLength = body.length;
const contentTail = getContentTail(body);
return result_1.Result.ok({
packageVersions,
syncedAt,
contentLength,
contentTail,
});
}
exports.memCache = new Map();
function cacheResult(registryUrl, result) {
const registryHostname = (0, url_1.parseUrl)(registryUrl)?.hostname;
if (registryHostname === 'rubygems.org') {
exports.memCache.set(registryUrl, result);
}
}
const VersionLines = zod_1.z
.string()
.transform((x) => x.split(regex_1.newlineRegex))
.pipe((0, schema_utils_1.LooseArray)(zod_1.z
.string()
.transform((line) => line.trim())
.refine((line) => line.length > 0)
.refine((line) => !line.startsWith('created_at:'))
.refine((line) => line !== '---')
.transform((line) => line.split(' '))
.pipe(zod_1.z.tuple([zod_1.z.string(), zod_1.z.string()]).rest(zod_1.z.string()))
.transform(([packageName, versions]) => {
const deletedVersions = new Set();
const addedVersions = [];
for (const version of versions.split(',')) {
if (version.startsWith('-')) {
deletedVersions.add(version.slice(1));
}
else {
addedVersions.push(version);
}
}
return { packageName, deletedVersions, addedVersions };
})));
function isStale(regCache) {
return (0, date_1.getElapsedMinutes)(regCache.syncedAt) >= 15;
}
class VersionsEndpointCache {
http;
constructor(http) {
this.http = http;
}
cacheRequests = new Map();
/**
* At any given time, there should only be one request for a given registryUrl.
*/
async getCache(registryUrl) {
const oldResult = exports.memCache.get(registryUrl);
if (!oldResult) {
const newResult = await this.fullSync(registryUrl);
cacheResult(registryUrl, newResult);
return newResult;
}
const { val: data } = oldResult.unwrap();
if (!data) {
return oldResult;
}
if (isStale(data)) {
exports.memCache.delete(registryUrl); // If no error is thrown, we'll re-set the cache
const newResult = await this.deltaSync(data, registryUrl);
cacheResult(registryUrl, newResult);
return newResult;
}
return oldResult;
}
async getVersions(registryUrl, packageName) {
/**
* Ensure that only one request for a given registryUrl is in flight at a time.
*/
let cacheRequest = this.cacheRequests.get(registryUrl);
if (!cacheRequest) {
cacheRequest = this.getCache(registryUrl);
this.cacheRequests.set(registryUrl, cacheRequest);
}
let cachedResult;
try {
cachedResult = await cacheRequest;
}
finally {
this.cacheRequests.delete(registryUrl);
}
const { val: cachedData } = cachedResult.unwrap();
if (!cachedData) {
logger_1.logger.debug({ packageName, registryUrl }, 'Rubygems: endpoint not supported');
return result_1.Result.err('unsupported-api');
}
const versions = cachedData.packageVersions.get(packageName);
if (!versions?.length) {
logger_1.logger.debug({ packageName, registryUrl }, 'Rubygems: versions not found');
return result_1.Result.err('package-not-found');
}
return result_1.Result.ok(versions);
}
async fullSync(registryUrl) {
try {
const url = `${registryUrl}/versions`;
const opts = { headers: { 'Accept-Encoding': 'gzip' } };
const { body } = await this.http.getText(url, opts);
return parseFullBody(body);
}
catch (err) {
if (err instanceof http_1.HttpError && err.response?.statusCode === 404) {
return result_1.Result.err('unsupported-api');
}
throw err;
}
}
async deltaSync(oldCache, registryUrl) {
try {
const url = `${registryUrl}/versions`;
const startByte = oldCache.contentLength - oldCache.contentTail.length;
const opts = {
headers: {
['Accept-Encoding']: 'deflate, compress, br', // Note: `gzip` usage breaks http client, when used with `Range` header
['Range']: `bytes=${startByte}-`,
},
};
const { statusCode, body } = await this.http.getText(url, opts);
/**
* Rubygems will return the full body instead of `416 Range Not Satisfiable`.
* In this case, status code will be 200 instead of 206.
*/
if (statusCode === 200) {
return parseFullBody(body);
}
/**
* We request data in range that overlaps previously fetched data.
* If the head of the response doesn't match the tail of the previous response,
* it means that the data we have is no longer valid.
* In this case we start over with a full sync.
*/
const contentHead = getContentHead(body);
if (contentHead !== oldCache.contentTail) {
return this.fullSync(registryUrl);
}
/**
* Update the cache with the new data.
*/
const delta = stripContentHead(body);
const packageVersions = reconcilePackageVersions(oldCache.packageVersions, VersionLines.parse(delta));
const syncedAt = new Date();
const contentLength = oldCache.contentLength + delta.length;
const contentTail = getContentTail(body);
return result_1.Result.ok({
packageVersions,
syncedAt,
contentLength,
contentTail,
});
}
catch (err) {
if (err instanceof http_1.HttpError) {
const responseStatus = err.response?.statusCode;
/**
* In case of `416 Range Not Satisfiable` we do a full sync.
* This is unlikely to happen in real life, but anyway.
*/
if (responseStatus === 416) {
return this.fullSync(registryUrl);
}
/**
* If the endpoint is not supported, we stop trying.
* This is unlikely to happen in real life, but still.
*/
if (responseStatus === 404) {
return result_1.Result.err('unsupported-api');
}
}
throw err;
}
}
}
exports.VersionsEndpointCache = VersionsEndpointCache;
//# sourceMappingURL=versions-endpoint-cache.js.map