renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
193 lines (192 loc) • 7.91 kB
JavaScript
import { getEnv } from "../../../util/env.js";
import { regEx } from "../../../util/regex.js";
import { logger } from "../../../logger/index.js";
import { ensureTrailingSlash, parseUrl } from "../../../util/url.js";
import { coerceArray } from "../../../util/array.js";
import { id } from "../../versioning/pep440/index.js";
import { parse } from "../../../util/html.js";
import { asTimestamp } from "../../../util/timestamp.js";
import { Datasource } from "../datasource.js";
import { getGoogleAuthToken } from "../util.js";
import { isGitHubRepo, normalizePythonDepName } from "./common.js";
import { PypiResponse } from "./schema.js";
import { isString } from "@sindresorhus/is";
import changelogFilenameRegex from "changelog-filename-regex";
//#region lib/modules/datasource/pypi/index.ts
var PypiDatasource = class PypiDatasource extends Datasource {
static id = "pypi";
constructor() {
super(PypiDatasource.id);
}
caching = true;
customRegistrySupport = true;
static defaultURL = getEnv().PIP_INDEX_URL ?? "https://pypi.org/pypi/";
defaultRegistryUrls = [PypiDatasource.defaultURL];
defaultVersioning = id;
registryStrategy = "merge";
releaseTimestampSupport = true;
releaseTimestampNote = "The relase timestamp is determined from the `upload_time` field in the results. This field is not available when using the simple API.";
sourceUrlSupport = "release";
sourceUrlNote = "The source URL is determined from the `homepage` field if it is a github repository, else we use the `project_urls` field.";
async getReleases({ packageName, registryUrl }) {
let dependency = null;
const hostUrl = ensureTrailingSlash(registryUrl.replace("https://pypi.org/simple", "https://pypi.org/pypi"));
const normalizedLookupName = normalizePythonDepName(packageName);
if (hostUrl.endsWith("/simple/") || hostUrl.endsWith("/+simple/")) {
logger.trace({
packageName,
hostUrl
}, "Looking up pypi simple dependency");
dependency = await this.getSimpleDependency(normalizedLookupName, hostUrl);
} else {
logger.trace({
packageName,
hostUrl
}, "Looking up pypi api dependency");
try {
dependency = await this.getDependency(normalizedLookupName, hostUrl);
} catch (err) {
logger.trace({
packageName,
hostUrl,
err
}, "Looking up pypi simple dependency via fallback");
dependency = await this.getSimpleDependency(normalizedLookupName, hostUrl);
}
}
return dependency;
}
sanitizeLookupUrl(lookupUrl, parsedUrl) {
if (!parsedUrl.username && !parsedUrl.password) return lookupUrl;
parsedUrl.username = "";
parsedUrl.password = "";
return parsedUrl.toString();
}
async getAuthHeaders(lookupUrl) {
const parsedUrl = parseUrl(lookupUrl);
// v8 ignore if -- TODO: refactor to cover this branch through public behavior again
if (!parsedUrl) {
logger.once.debug({ lookupUrl }, "Failed to parse URL");
return {
headers: {},
lookupUrl
};
}
if (parsedUrl.hostname.endsWith(".pkg.dev")) {
const auth = await getGoogleAuthToken();
if (auth) {
const sanitizedLookupUrl = this.sanitizeLookupUrl(lookupUrl, parsedUrl);
return {
headers: { authorization: `Basic ${auth}` },
lookupUrl: sanitizedLookupUrl
};
}
logger.once.debug({ lookupUrl }, "Could not get Google access token");
return {
headers: {},
lookupUrl
};
}
return {
headers: {},
lookupUrl
};
}
async getDependency(packageName, hostUrl) {
const lookupUrl = new URL(`${normalizePythonDepName(packageName)}/json`, hostUrl).href;
const dependency = { releases: [] };
logger.trace({ lookupUrl }, "Pypi api got lookup");
const { headers, lookupUrl: sanitizedUrl } = await this.getAuthHeaders(lookupUrl);
const rep = await this.http.getJson(sanitizedUrl, { headers }, PypiResponse);
const dep = rep?.body;
if (rep.authorization) dependency.isPrivate = true;
logger.trace({ lookupUrl }, "Got pypi api result");
if (dep.info?.home_page) {
dependency.homepage = dep.info.home_page;
if (isGitHubRepo(dep.info.home_page)) dependency.sourceUrl = dep.info.home_page.replace("http://", "https://");
}
if (dep.info?.project_urls) for (const [name, projectUrl] of Object.entries(dep.info.project_urls)) {
const lower = name.toLowerCase();
if (projectUrl && !dependency.sourceUrl && (lower.startsWith("repo") || lower === "code" || lower === "source" || isGitHubRepo(projectUrl))) dependency.sourceUrl = projectUrl;
if (!dependency.changelogUrl && ([
"changelog",
"change log",
"changes",
"release notes",
"news",
"what's new"
].includes(lower) || changelogFilenameRegex.exec(lower))) dependency.changelogUrl = projectUrl;
}
if (dep.releases) dependency.releases = Object.keys(dep.releases).map((version) => {
const releases = coerceArray(dep.releases?.[version]);
const { upload_time: releaseTimestamp } = releases[0] || {};
const isDeprecated = releases.some(({ yanked }) => yanked);
const result = {
version,
releaseTimestamp: asTimestamp(releaseTimestamp)
};
if (isDeprecated) result.isDeprecated = isDeprecated;
const pythonConstraints = releases.map(({ requires_python }) => requires_python).filter(isString);
result.constraints = { python: Array.from(new Set(pythonConstraints)) };
return result;
});
return dependency;
}
static extractVersionFromLinkText(text, packageName) {
const lcText = text.toLowerCase();
const normalizedSrcText = normalizePythonDepName(text);
const srcPrefix = `${packageName}-`;
if (!normalizedSrcText.startsWith(srcPrefix)) return null;
const normalizedLengthDiff = lcText.length - normalizedSrcText.length;
const res = lcText.slice(srcPrefix.length + normalizedLengthDiff);
const srcSuffix = [
".tar.gz",
".tar.bz2",
".tar.xz",
".zip",
".tgz"
].find((suffix) => lcText.endsWith(suffix));
if (srcSuffix) return res.slice(0, -srcSuffix.length);
if (lcText.endsWith(".whl") && lcText.split("-").length > 2) return res.split("-")[0];
return null;
}
static cleanSimpleHtml(html) {
return html.replace(regEx(/<\/?pre>/), "").replace(regEx(/data-requires-python="([^"]*?)>([^"]*?)"/g), "data-requires-python=\"$1>$2\"").replace(regEx(/data-requires-python="([^"]*?)<([^"]*?)"/g), "data-requires-python=\"$1<$2\"");
}
async getSimpleDependency(packageName, hostUrl) {
const lookupUrl = new URL(ensureTrailingSlash(normalizePythonDepName(packageName)), hostUrl).href;
const dependency = { releases: [] };
const { headers, lookupUrl: sanitizedUrl } = await this.getAuthHeaders(lookupUrl);
const response = await this.http.getText(sanitizedUrl, { headers });
const dep = response?.body;
if (!dep) {
logger.trace({ dependency: packageName }, "pip package not found");
return null;
}
if (response.authorization) dependency.isPrivate = true;
const links = parse(PypiDatasource.cleanSimpleHtml(dep)).querySelectorAll("a");
const releases = {};
for (const link of Array.from(links)) {
const version = PypiDatasource.extractVersionFromLinkText(link.text?.trim(), packageName);
if (version) {
const release = { yanked: link.hasAttribute("data-yanked") };
const requiresPython = link.getAttribute("data-requires-python");
if (requiresPython) release.requires_python = requiresPython;
if (!releases[version]) releases[version] = [];
releases[version].push(release);
}
}
dependency.releases = Object.keys(releases).map((version) => {
const versionReleases = coerceArray(releases[version]);
const isDeprecated = versionReleases.some(({ yanked }) => yanked);
const result = { version };
if (isDeprecated) result.isDeprecated = isDeprecated;
result.constraints = { python: versionReleases.map(({ requires_python }) => requires_python) };
return result;
});
return dependency;
}
};
//#endregion
export { PypiDatasource };
//# sourceMappingURL=index.js.map