renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
236 lines • 11.9 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.PypiDatasource = void 0;
const tslib_1 = require("tslib");
const node_url_1 = tslib_1.__importDefault(require("node:url"));
const is_1 = tslib_1.__importDefault(require("@sindresorhus/is"));
const changelog_filename_regex_1 = tslib_1.__importDefault(require("changelog-filename-regex"));
const logger_1 = require("../../../logger");
const array_1 = require("../../../util/array");
const env_1 = require("../../../util/env");
const html_1 = require("../../../util/html");
const regex_1 = require("../../../util/regex");
const timestamp_1 = require("../../../util/timestamp");
const url_1 = require("../../../util/url");
const pep440 = tslib_1.__importStar(require("../../versioning/pep440"));
const datasource_1 = require("../datasource");
const util_1 = require("../util");
const common_1 = require("./common");
class PypiDatasource extends datasource_1.Datasource {
static id = 'pypi';
constructor() {
super(PypiDatasource.id);
}
caching = true;
customRegistrySupport = true;
static defaultURL = (0, env_1.getEnv)().PIP_INDEX_URL ?? 'https://pypi.org/pypi/';
defaultRegistryUrls = [PypiDatasource.defaultURL];
defaultVersioning = pep440.id;
registryStrategy = 'merge';
releaseTimestampNote = 'The relase timestamp is determined from the `upload_time` field in the results.';
sourceUrlSupport = 'release';
sourceUrlNote = 'The source URL is determined from the `homepage` field if it is a github repository, else we use the `project_urls` field.';
async getReleases({ packageName, registryUrl, }) {
let dependency = null;
// TODO: null check (#22198)
const hostUrl = (0, url_1.ensureTrailingSlash)(registryUrl.replace('https://pypi.org/simple', 'https://pypi.org/pypi'));
const normalizedLookupName = (0, common_1.normalizePythonDepName)(packageName);
// not all simple indexes use this identifier, but most do
if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) {
logger_1.logger.trace({ packageName, hostUrl }, 'Looking up pypi simple dependency');
dependency = await this.getSimpleDependency(normalizedLookupName, hostUrl);
}
else {
logger_1.logger.trace({ packageName, hostUrl }, 'Looking up pypi api dependency');
try {
// we need to resolve early here so we can catch any 404s and fallback to a simple lookup
dependency = await this.getDependency(normalizedLookupName, hostUrl);
}
catch (err) {
// error contacting json-style api -- attempt to fallback to a simple-style api
logger_1.logger.trace({ packageName, hostUrl, err }, 'Looking up pypi simple dependency via fallback');
dependency = await this.getSimpleDependency(normalizedLookupName, hostUrl);
}
}
return dependency;
}
async getAuthHeaders(lookupUrl) {
const parsedUrl = (0, url_1.parseUrl)(lookupUrl);
if (!parsedUrl) {
logger_1.logger.once.debug({ lookupUrl }, 'Failed to parse URL');
return {};
}
if (parsedUrl.hostname.endsWith('.pkg.dev')) {
const auth = await (0, util_1.getGoogleAuthToken)();
if (auth) {
return { authorization: `Basic ${auth}` };
}
logger_1.logger.once.debug({ lookupUrl }, 'Could not get Google access token');
return {};
}
return {};
}
async getDependency(packageName, hostUrl) {
const lookupUrl = node_url_1.default.resolve(hostUrl, `${(0, common_1.normalizePythonDepName)(packageName)}/json`);
const dependency = { releases: [] };
logger_1.logger.trace({ lookupUrl }, 'Pypi api got lookup');
const headers = await this.getAuthHeaders(lookupUrl);
const rep = await this.http.getJsonUnchecked(lookupUrl, {
headers,
});
const dep = rep?.body;
if (!dep) {
logger_1.logger.trace({ dependency: packageName }, 'pip package not found');
return null;
}
if (rep.authorization) {
dependency.isPrivate = true;
}
logger_1.logger.trace({ lookupUrl }, 'Got pypi api result');
if (dep.info?.home_page) {
dependency.homepage = dep.info.home_page;
if ((0, common_1.isGitHubRepo)(dep.info.home_page)) {
dependency.sourceUrl = dep.info.home_page.replace('http://', 'https://');
}
}
if (dep.info?.project_urls) {
for (const [name, projectUrl] of Object.entries(dep.info.project_urls)) {
const lower = name.toLowerCase();
if (!dependency.sourceUrl &&
(lower.startsWith('repo') ||
lower === 'code' ||
lower === 'source' ||
(0, common_1.isGitHubRepo)(projectUrl))) {
dependency.sourceUrl = projectUrl;
}
if (!dependency.changelogUrl &&
([
'changelog',
'change log',
'changes',
'release notes',
'news',
"what's new",
].includes(lower) ||
changelog_filename_regex_1.default.exec(lower))) {
// from https://github.com/pypa/warehouse/blob/418c7511dc367fb410c71be139545d0134ccb0df/warehouse/templates/packaging/detail.html#L24
dependency.changelogUrl = projectUrl;
}
}
}
if (dep.releases) {
const versions = Object.keys(dep.releases);
dependency.releases = versions.map((version) => {
const releases = (0, array_1.coerceArray)(dep.releases?.[version]);
const { upload_time: releaseTimestamp } = releases[0] || {};
const isDeprecated = releases.some(({ yanked }) => yanked);
const result = {
version,
releaseTimestamp: (0, timestamp_1.asTimestamp)(releaseTimestamp),
};
if (isDeprecated) {
result.isDeprecated = isDeprecated;
}
// There may be multiple releases with different requires_python, so we return all in an array
const pythonConstraints = releases
.map(({ requires_python }) => requires_python)
.filter(is_1.default.string);
result.constraints = {
python: Array.from(new Set(pythonConstraints)),
};
return result;
});
}
return dependency;
}
static extractVersionFromLinkText(text, packageName) {
// source packages
const lcText = text.toLowerCase();
const normalizedSrcText = (0, common_1.normalizePythonDepName)(text);
const srcPrefix = `${packageName}-`;
// source distribution format: `{name}-{version}.tar.gz` (https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-file-name)
// binary distribution: `{distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl` (https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-name-convention)
// officially both `name` and `distribution` should be normalized and then the - replaced with _, but in reality this is not the case
// We therefore normalize the name we have (replacing `_-.` with -) and then check if the text starts with the normalized name
if (!normalizedSrcText.startsWith(srcPrefix)) {
return null;
}
// strip off the prefix using the prefix length as we may have normalized the srcPrefix/packageName
// We assume that neither the version nor the suffix contains multiple `-` like `0.1.2---rc1.tar.gz`
// and use the difference in length to strip off the prefix in case the name contains double `--` characters
const normalizedLengthDiff = lcText.length - normalizedSrcText.length;
const res = lcText.slice(srcPrefix.length + normalizedLengthDiff);
// source distribution
const srcSuffixes = ['.tar.gz', '.tar.bz2', '.tar.xz', '.zip', '.tgz'];
const srcSuffix = srcSuffixes.find((suffix) => lcText.endsWith(suffix));
if (srcSuffix) {
// strip off the suffix using character length
return res.slice(0, -srcSuffix.length);
}
// binary distribution
// for binary distributions the version is the first part after the removed distribution name
const wheelSuffix = '.whl';
if (lcText.endsWith(wheelSuffix) && lcText.split('-').length > 2) {
return res.split('-')[0];
}
return null;
}
static cleanSimpleHtml(html) {
return (html
.replace((0, regex_1.regEx)(/<\/?pre>/), '')
// Certain simple repositories like artifactory don't escape > and <
.replace((0, regex_1.regEx)(/data-requires-python="([^"]*?)>([^"]*?)"/g), 'data-requires-python="$1>$2"')
.replace((0, regex_1.regEx)(/data-requires-python="([^"]*?)<([^"]*?)"/g), 'data-requires-python="$1<$2"'));
}
async getSimpleDependency(packageName, hostUrl) {
const lookupUrl = node_url_1.default.resolve(hostUrl, (0, url_1.ensureTrailingSlash)((0, common_1.normalizePythonDepName)(packageName)));
const dependency = { releases: [] };
const headers = await this.getAuthHeaders(lookupUrl);
const response = await this.http.getText(lookupUrl, { headers });
const dep = response?.body;
if (!dep) {
logger_1.logger.trace({ dependency: packageName }, 'pip package not found');
return null;
}
if (response.authorization) {
dependency.isPrivate = true;
}
const root = (0, html_1.parse)(PypiDatasource.cleanSimpleHtml(dep));
const links = root.querySelectorAll('a');
const releases = {};
for (const link of Array.from(links)) {
const version = PypiDatasource.extractVersionFromLinkText(link.text?.trim(), packageName);
if (version) {
const release = {
yanked: link.hasAttribute('data-yanked'),
};
const requiresPython = link.getAttribute('data-requires-python');
if (requiresPython) {
release.requires_python = requiresPython;
}
if (!releases[version]) {
releases[version] = [];
}
releases[version].push(release);
}
}
const versions = Object.keys(releases);
dependency.releases = versions.map((version) => {
const versionReleases = (0, array_1.coerceArray)(releases[version]);
const isDeprecated = versionReleases.some(({ yanked }) => yanked);
const result = { version };
if (isDeprecated) {
result.isDeprecated = isDeprecated;
}
// There may be multiple releases with different requires_python, so we return all in an array
result.constraints = {
// TODO: string[] isn't allowed here
python: versionReleases.map(({ requires_python }) => requires_python),
};
return result;
});
return dependency;
}
}
exports.PypiDatasource = PypiDatasource;
//# sourceMappingURL=index.js.map