renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
294 lines • 12.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.CrateDatasource = void 0;
const tslib_1 = require("tslib");
const simple_git_1 = tslib_1.__importDefault(require("simple-git"));
const upath_1 = tslib_1.__importDefault(require("upath"));
const global_1 = require("../../../config/global");
const logger_1 = require("../../../logger");
const memCache = tslib_1.__importStar(require("../../../util/cache/memory"));
const decorator_1 = require("../../../util/cache/package/decorator");
const utils_1 = require("../../../util/exec/utils");
const fs_1 = require("../../../util/fs");
const config_1 = require("../../../util/git/config");
const hash_1 = require("../../../util/hash");
const memory_http_cache_provider_1 = require("../../../util/http/cache/memory-http-cache-provider");
const regex_1 = require("../../../util/regex");
const url_1 = require("../../../util/url");
const cargoVersioning = tslib_1.__importStar(require("../../versioning/cargo"));
const datasource_1 = require("../datasource");
const schema_1 = require("./schema");
class CrateDatasource extends datasource_1.Datasource {
static id = 'crate';
constructor() {
super(CrateDatasource.id);
}
defaultRegistryUrls = ['https://crates.io'];
defaultVersioning = cargoVersioning.id;
static CRATES_IO_BASE_URL = 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/';
static CRATES_IO_API_BASE_URL = 'https://crates.io/api/v1/';
sourceUrlSupport = 'package';
sourceUrlNote = 'The source URL is determined from the `repository` field in the results.';
async getReleases({ packageName, registryUrl, }) {
/* v8 ignore next 6 -- should never happen */
if (!registryUrl) {
logger_1.logger.warn('crate datasource: No registryUrl specified, cannot perform getReleases');
return null;
}
const registryInfo = await CrateDatasource.fetchRegistryInfo({
packageName,
registryUrl,
});
if (!registryInfo) {
logger_1.logger.debug(`Could not fetch registry info from ${registryUrl}`);
return null;
}
const dependencyUrl = CrateDatasource.getDependencyUrl(registryInfo, packageName);
const payload = await this.fetchCrateRecordsPayload(registryInfo, packageName);
const lines = payload
.split(regex_1.newlineRegex) // break into lines
.map((line) => line.trim()) // remove whitespace
.filter((line) => line.length !== 0) // remove empty lines
.map((line) => JSON.parse(line)); // parse
const metadata = await this.getCrateMetadata(registryInfo, packageName);
const result = {
dependencyUrl,
releases: [],
};
if (metadata?.homepage) {
result.homepage = metadata.homepage;
}
if (metadata?.repository) {
result.sourceUrl = metadata.repository;
}
result.releases = lines
.map((version) => {
const release = {
version: version.vers.replace(/\+.*$/, ''),
};
if (version.yanked) {
release.isDeprecated = true;
}
if (version.rust_version) {
release.constraints = {
rust: [version.rust_version],
};
}
return release;
})
.filter((release) => release.version);
if (!result.releases.length) {
return null;
}
return result;
}
async getCrateMetadata(info, packageName) {
if (info.flavor !== 'crates.io') {
return null;
}
// The `?include=` suffix is required to avoid unnecessary database queries
// on the crates.io server. This lets us work around the regular request
// throttling of one request per second.
const crateUrl = `${CrateDatasource.CRATES_IO_API_BASE_URL}crates/${packageName}?include=`;
logger_1.logger.debug({ crateUrl, packageName, registryUrl: info.rawUrl }, 'downloading crate metadata');
try {
const response = await this.http.getJsonUnchecked(crateUrl);
return response.body.crate;
}
catch (err) {
logger_1.logger.warn({ err, packageName, registryUrl: info.rawUrl }, 'failed to download crate metadata');
}
return null;
}
async fetchCrateRecordsPayload(info, packageName) {
if (info.clonePath) {
const path = upath_1.default.join(info.clonePath, ...CrateDatasource.getIndexSuffix(packageName));
return (0, fs_1.readCacheFile)(path, 'utf8');
}
const baseUrl = info.flavor === 'crates.io'
? CrateDatasource.CRATES_IO_BASE_URL
: info.rawUrl;
if (info.flavor === 'crates.io' || info.isSparse) {
const packageSuffix = CrateDatasource.getIndexSuffix(packageName.toLowerCase());
const crateUrl = (0, url_1.joinUrlParts)(baseUrl, ...packageSuffix);
try {
return (await this.http.getText(crateUrl)).body;
}
catch (err) {
this.handleGenericErrors(err);
}
}
throw new Error(`unsupported crate registry flavor: ${info.flavor}`);
}
/**
* Computes the dependency URL for a crate, given
* registry information
*/
static getDependencyUrl(info, packageName) {
switch (info.flavor) {
case 'crates.io':
return `https://crates.io/crates/${packageName}`;
case 'cloudsmith': {
// input: https://dl.cloudsmith.io/basic/$org/$repo/cargo/index.git
const tokens = info.url.pathname.split('/');
const org = tokens[2];
const repo = tokens[3];
return `https://cloudsmith.io/~${org}/repos/${repo}/packages/detail/cargo/${packageName}`;
}
default:
return `${info.rawUrl}/${packageName}`;
}
}
/**
* Given a Git URL, computes a semi-human-readable name for a folder in which to
* clone the repository.
*/
static cacheDirFromUrl(url) {
const proto = url.protocol.replace((0, regex_1.regEx)(/:$/), '');
const host = url.hostname;
const hash = (0, hash_1.toSha256)(url.pathname).substring(0, 7);
return `crate-registry-${proto}-${host}-${hash}`;
}
static isSparseRegistry(url) {
const parsed = (0, url_1.parseUrl)(url);
if (!parsed) {
return false;
}
return parsed.protocol.startsWith('sparse+');
}
/**
* Fetches information about a registry, by url.
* If no url is given, assumes crates.io.
* If an url is given, assumes it's a valid Git repository
* url and clones it to cache.
*/
static async fetchRegistryInfo({ packageName, registryUrl, }) {
/* v8 ignore next 3 -- should never happen */
if (!registryUrl) {
return null;
}
const isSparseRegistry = CrateDatasource.isSparseRegistry(registryUrl);
const registryFetchUrl = isSparseRegistry
? registryUrl.replace(/^sparse\+/, '')
: registryUrl;
const url = (0, url_1.parseUrl)(registryFetchUrl);
if (!url) {
logger_1.logger.debug(`Could not parse registry URL ${registryFetchUrl}`);
return null;
}
let flavor;
if (url.hostname === 'crates.io') {
flavor = 'crates.io';
}
else if (url.hostname === 'dl.cloudsmith.io') {
flavor = 'cloudsmith';
}
else {
flavor = 'other';
}
const registry = {
flavor,
rawUrl: registryFetchUrl,
url,
isSparse: isSparseRegistry,
};
if (registry.flavor !== 'crates.io' &&
!global_1.GlobalConfig.get('allowCustomCrateRegistries')) {
logger_1.logger.warn('crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out');
return null;
}
if (registry.flavor !== 'crates.io' && !registry.isSparse) {
const cacheKey = `crate-datasource/registry-clone-path/${registryFetchUrl}`;
const cacheKeyForError = `crate-datasource/registry-clone-path/${registryFetchUrl}/error`;
// We need to ensure we don't run `git clone` in parallel. Therefore we store
// a promise of the running operation in the mem cache, which in the end resolves
// to the file path of the cloned repository.
const clonePathPromise = memCache.get(cacheKey);
let clonePath;
if (clonePathPromise) {
clonePath = await clonePathPromise;
}
else {
clonePath = upath_1.default.join((0, fs_1.privateCacheDir)(), CrateDatasource.cacheDirFromUrl(url));
logger_1.logger.info({ clonePath, registryFetchUrl }, `Cloning private cargo registry`);
const git = (0, simple_git_1.default)({
...(0, config_1.simpleGitConfig)(),
maxConcurrentProcesses: 1,
}).env((0, utils_1.getChildEnv)());
const clonePromise = git.clone(registryFetchUrl, clonePath, {
'--depth': 1,
});
memCache.set(cacheKey, clonePromise.then(() => clonePath).catch(() => null));
try {
await clonePromise;
}
catch (err) {
logger_1.logger.warn({ err, packageName, registryFetchUrl }, 'failed cloning git registry');
memCache.set(cacheKeyForError, err);
return null;
}
}
if (!clonePath) {
const err = memCache.get(cacheKeyForError);
logger_1.logger.warn({ err, packageName, registryFetchUrl }, 'Previous git clone failed, bailing out.');
return null;
}
registry.clonePath = clonePath;
}
return registry;
}
static areReleasesCacheable(registryUrl) {
// We only cache public releases, we don't want to cache private
// cloned data between runs.
return registryUrl === 'https://crates.io';
}
static getIndexSuffix(packageName) {
const len = packageName.length;
if (len === 1) {
return ['1', packageName];
}
if (len === 2) {
return ['2', packageName];
}
if (len === 3) {
return ['3', packageName[0], packageName];
}
return [packageName.slice(0, 2), packageName.slice(2, 4), packageName];
}
async postprocessRelease({ packageName, registryUrl }, release) {
if (registryUrl !== 'https://crates.io') {
return release;
}
const url = `https://crates.io/api/v1/crates/${packageName}/${release.version}`;
const { body: releaseTimestamp } = await this.http.getJson(url, { cacheProvider: memory_http_cache_provider_1.memCacheProvider }, schema_1.ReleaseTimestampSchema);
release.releaseTimestamp = releaseTimestamp;
return release;
}
}
exports.CrateDatasource = CrateDatasource;
tslib_1.__decorate([
(0, decorator_1.cache)({
namespace: `datasource-${CrateDatasource.id}`,
key: ({ registryUrl, packageName }) =>
// TODO: types (#22198)
`${registryUrl}/${packageName}`,
cacheable: ({ registryUrl }) => CrateDatasource.areReleasesCacheable(registryUrl),
})
], CrateDatasource.prototype, "getReleases", null);
tslib_1.__decorate([
(0, decorator_1.cache)({
namespace: `datasource-${CrateDatasource.id}-metadata`,
key: (info, packageName) => `${info.rawUrl}/${packageName}`,
cacheable: (info) => CrateDatasource.areReleasesCacheable(info.rawUrl),
ttlMinutes: 24 * 60, // 24 hours
})
], CrateDatasource.prototype, "getCrateMetadata", null);
tslib_1.__decorate([
(0, decorator_1.cache)({
namespace: `datasource-crate`,
key: ({ registryUrl, packageName }, { version }) => `postprocessRelease:${registryUrl}:${packageName}:${version}`,
ttlMinutes: 7 * 24 * 60,
cacheable: ({ registryUrl }, _) => registryUrl === 'https://crates.io',
})
], CrateDatasource.prototype, "postprocessRelease", null);
//# sourceMappingURL=index.js.map