renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
365 lines • 16.5 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadHttpProtocol = downloadHttpProtocol;
exports.downloadHttpContent = downloadHttpContent;
exports.downloadS3Protocol = downloadS3Protocol;
exports.downloadArtifactRegistryProtocol = downloadArtifactRegistryProtocol;
exports.getMavenUrl = getMavenUrl;
exports.downloadMaven = downloadMaven;
exports.downloadMavenXml = downloadMavenXml;
exports.getDependencyParts = getDependencyParts;
exports.createUrlForDependencyPom = createUrlForDependencyPom;
exports.getDependencyInfo = getDependencyInfo;
const node_stream_1 = require("node:stream");
const client_s3_1 = require("@aws-sdk/client-s3");
const xmldoc_1 = require("xmldoc");
const error_messages_1 = require("../../../constants/error-messages");
const logger_1 = require("../../../logger");
const external_host_error_1 = require("../../../types/errors/external-host-error");
const http_1 = require("../../../util/http");
const package_http_cache_provider_1 = require("../../../util/http/cache/package-http-cache-provider");
const regex_1 = require("../../../util/regex");
const result_1 = require("../../../util/result");
const s3_1 = require("../../../util/s3");
const streams_1 = require("../../../util/streams");
const timestamp_1 = require("../../../util/timestamp");
const url_1 = require("../../../util/url");
const util_1 = require("../util");
const common_1 = require("./common");
function getHost(url) {
return (0, url_1.parseUrl)(url)?.host;
}
function isTemporaryError(err) {
if (err.code === 'ECONNRESET') {
return true;
}
if (err.response) {
const status = err.response.statusCode;
return status === 429 || (status >= 500 && status < 600);
}
return false;
}
function isHostError(err) {
return err.code === 'ETIMEDOUT';
}
function isNotFoundError(err) {
return err.code === 'ENOTFOUND' || err.response?.statusCode === 404;
}
function isPermissionsIssue(err) {
const status = err.response?.statusCode;
return status === 401 || status === 403;
}
function isConnectionError(err) {
return (err.code === 'EAI_AGAIN' ||
err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' ||
err.code === 'ECONNREFUSED');
}
function isUnsupportedHostError(err) {
return err.name === 'UnsupportedProtocolError';
}
const cacheProvider = new package_http_cache_provider_1.PackageHttpCacheProvider({
namespace: 'datasource-maven:cache-provider',
softTtlMinutes: 15,
checkAuthorizationHeader: true,
checkCacheControlHeader: false, // Maven doesn't respond with `cache-control` headers
});
async function downloadHttpProtocol(http, pkgUrl, opts = {}) {
const url = pkgUrl.toString();
const fetchResult = await result_1.Result.wrap(http.getText(url, { ...opts, cacheProvider }))
.transform((res) => {
const result = { data: res.body };
if (!res.authorization) {
result.isCacheable = true;
}
const lastModified = (0, timestamp_1.asTimestamp)(res?.headers?.['last-modified']);
if (lastModified) {
result.lastModified = lastModified;
}
return result;
})
.catch((err) => {
/* v8 ignore start: never happens, needs for type narrowing */
if (!(err instanceof http_1.HttpError)) {
return result_1.Result.err({ type: 'unknown', err });
} /* v8 ignore stop */
const failedUrl = url;
if (err.message === error_messages_1.HOST_DISABLED) {
logger_1.logger.trace({ failedUrl }, 'Host disabled');
return result_1.Result.err({ type: 'host-disabled' });
}
if (isNotFoundError(err)) {
logger_1.logger.trace({ failedUrl }, `Url not found`);
return result_1.Result.err({ type: 'not-found' });
}
if (isHostError(err)) {
logger_1.logger.debug(`Cannot connect to host ${failedUrl}`);
return result_1.Result.err({ type: 'host-error' });
}
if (isPermissionsIssue(err)) {
logger_1.logger.debug(`Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`);
return result_1.Result.err({ type: 'permission-issue' });
}
if (isTemporaryError(err)) {
logger_1.logger.debug({ failedUrl, err }, 'Temporary error');
if (getHost(url) === getHost(common_1.MAVEN_REPO)) {
return result_1.Result.err({ type: 'maven-central-temporary-error', err });
}
else {
return result_1.Result.err({ type: 'temporary-error' });
}
}
if (isConnectionError(err)) {
logger_1.logger.debug(`Connection refused to maven registry ${failedUrl}`);
return result_1.Result.err({ type: 'connection-error' });
}
if (isUnsupportedHostError(err)) {
logger_1.logger.debug(`Unsupported host ${failedUrl}`);
return result_1.Result.err({ type: 'unsupported-host' });
}
logger_1.logger.info({ failedUrl, err }, 'Unknown HTTP download error');
return result_1.Result.err({ type: 'unknown', err });
});
const { err } = fetchResult.unwrap();
if (err?.type === 'maven-central-temporary-error') {
throw new external_host_error_1.ExternalHostError(err.err);
}
return fetchResult;
}
async function downloadHttpContent(http, pkgUrl, opts = {}) {
const fetchResult = await downloadHttpProtocol(http, pkgUrl, opts);
return fetchResult.transform(({ data }) => data).unwrapOrNull();
}
function isS3NotFound(err) {
return err.message === 'NotFound' || err.message === 'NoSuchKey';
}
async function downloadS3Protocol(pkgUrl) {
logger_1.logger.trace({ url: pkgUrl.toString() }, `Attempting to load S3 dependency`);
const s3Url = (0, s3_1.parseS3Url)(pkgUrl);
if (!s3Url) {
return result_1.Result.err({ type: 'invalid-url' });
}
return await result_1.Result.wrap(() => {
const command = new client_s3_1.GetObjectCommand(s3Url);
const client = (0, s3_1.getS3Client)();
return client.send(command);
})
.transform(async ({ Body, LastModified, DeleteMarker, }) => {
if (DeleteMarker) {
logger_1.logger.trace({ failedUrl: pkgUrl.toString() }, 'Maven S3 lookup error: DeleteMarker encountered');
return result_1.Result.err({ type: 'not-found' });
}
if (!(Body instanceof node_stream_1.Readable)) {
logger_1.logger.debug({ failedUrl: pkgUrl.toString() }, 'Maven S3 lookup error: unsupported Body type');
return result_1.Result.err({ type: 'unsupported-format' });
}
const data = await (0, streams_1.streamToString)(Body);
const result = { data };
const lastModified = (0, timestamp_1.asTimestamp)(LastModified);
if (lastModified) {
result.lastModified = lastModified;
}
return result_1.Result.ok(result);
})
.catch((err) => {
if (!(err instanceof Error)) {
return result_1.Result.err(err);
}
const failedUrl = pkgUrl.toString();
if (err.name === 'CredentialsProviderError') {
logger_1.logger.debug({ failedUrl }, 'Maven S3 lookup error: credentials provider error, check "AWS_ACCESS_KEY_ID" and "AWS_SECRET_ACCESS_KEY" variables');
return result_1.Result.err({ type: 'credentials-error' });
}
if (err.message === 'Region is missing') {
logger_1.logger.debug({ failedUrl }, 'Maven S3 lookup error: missing region, check "AWS_REGION" variable');
return result_1.Result.err({ type: 'missing-aws-region' });
}
if (isS3NotFound(err)) {
logger_1.logger.trace({ failedUrl }, 'Maven S3 lookup error: object not found');
return result_1.Result.err({ type: 'not-found' });
}
logger_1.logger.debug({ failedUrl, err }, 'Maven S3 lookup error: unknown error');
return result_1.Result.err({ type: 'unknown', err });
});
}
async function downloadArtifactRegistryProtocol(http, pkgUrl) {
const opts = {};
const host = pkgUrl.host;
const path = pkgUrl.pathname;
logger_1.logger.trace({ host, path }, `Using google auth for Maven repository`);
const auth = await (0, util_1.getGoogleAuthToken)();
if (auth) {
opts.headers = { authorization: `Basic ${auth}` };
}
else {
logger_1.logger.once.debug({ host, path }, 'Could not get Google access token, using no auth');
}
const url = pkgUrl.toString().replace('artifactregistry:', 'https:');
return downloadHttpProtocol(http, url, opts);
}
function containsPlaceholder(str) {
return (0, regex_1.regEx)(/\${.*?}/g).test(str);
}
function getMavenUrl(dependency, repoUrl, path) {
return new URL(`${dependency.dependencyUrl}/${path}`, (0, url_1.ensureTrailingSlash)(repoUrl));
}
async function downloadMaven(http, url) {
const protocol = url.protocol;
let result = result_1.Result.err({ type: 'unsupported-protocol' });
if ((0, url_1.isHttpUrl)(url)) {
result = await downloadHttpProtocol(http, url);
}
if (protocol === 'artifactregistry:') {
result = await downloadArtifactRegistryProtocol(http, url);
}
if (protocol === 's3:') {
result = await downloadS3Protocol(url);
}
return result.onError((err) => {
if (err.type === 'unsupported-protocol') {
logger_1.logger.debug({ url: url.toString() }, `Maven lookup error: unsupported protocol (${protocol})`);
}
});
}
async function downloadMavenXml(http, url) {
const rawResult = await downloadMaven(http, url);
return rawResult.transform((result) => {
try {
return result_1.Result.ok({
...result,
data: new xmldoc_1.XmlDocument(result.data),
});
}
catch (err) {
return result_1.Result.err({ type: 'xml-parse-error', err });
}
});
}
function getDependencyParts(packageName) {
const [group, name] = packageName.split(':');
const dependencyUrl = `${group.replace((0, regex_1.regEx)(/\./g), '/')}/${name}`;
return {
display: packageName,
group,
name,
dependencyUrl,
};
}
function extractSnapshotVersion(metadata) {
// Parse the maven-metadata.xml for the snapshot version and determine
// the fixed version of the latest deployed snapshot.
// The metadata descriptor can be found at
// https://maven.apache.org/ref/3.3.3/maven-repository-metadata/repository-metadata.html
//
// Basically, we need to replace -SNAPSHOT with the artifact timestanp & build number,
// so for example 1.0.0-SNAPSHOT will become 1.0.0-<timestamp>-<buildNumber>
const version = metadata
.descendantWithPath('version')
?.val?.replace('-SNAPSHOT', '');
const snapshot = metadata.descendantWithPath('versioning.snapshot');
const timestamp = snapshot?.childNamed('timestamp')?.val;
const build = snapshot?.childNamed('buildNumber')?.val;
// If we weren't able to parse out the required 3 version elements,
// return null because we can't determine the fixed version of the latest snapshot.
if (!version || !timestamp || !build) {
return null;
}
return `${version}-${timestamp}-${build}`;
}
async function getSnapshotFullVersion(http, version, dependency, repoUrl) {
// To determine what actual files are available for the snapshot, first we have to fetch and parse
// the metadata located at http://<repo>/<group>/<artifact>/<version-SNAPSHOT>/maven-metadata.xml
const metadataUrl = getMavenUrl(dependency, repoUrl, `${version}/maven-metadata.xml`);
const metadataXmlResult = await downloadMavenXml(http, metadataUrl);
return metadataXmlResult
.transform(({ data }) => result_1.Result.wrapNullable(extractSnapshotVersion(data), {
type: 'snapshot-extract-error',
}))
.unwrapOrNull();
}
function isSnapshotVersion(version) {
if (version.endsWith('-SNAPSHOT')) {
return true;
}
return false;
}
async function createUrlForDependencyPom(http, version, dependency, repoUrl) {
if (isSnapshotVersion(version)) {
// By default, Maven snapshots are deployed to the repository with fixed file names.
// Resolve the full, actual pom file name for the version.
const fullVersion = await getSnapshotFullVersion(http, version, dependency, repoUrl);
// If we were able to resolve the version, use that, otherwise fall back to using -SNAPSHOT
if (fullVersion !== null) {
// TODO: types (#22198)
return `${version}/${dependency.name}-${fullVersion}.pom`;
}
}
// TODO: types (#22198)
return `${version}/${dependency.name}-${version}.pom`;
}
async function getDependencyInfo(http, dependency, repoUrl, version, recursionLimit = 5) {
const path = await createUrlForDependencyPom(http, version, dependency, repoUrl);
const pomUrl = getMavenUrl(dependency, repoUrl, path);
const pomXmlResult = await downloadMavenXml(http, pomUrl);
const dependencyInfoResult = await pomXmlResult.transform(async ({ data: pomContent }) => {
const result = {};
const homepage = pomContent.valueWithPath('url');
if (homepage && !containsPlaceholder(homepage)) {
result.homepage = homepage;
}
const sourceUrl = pomContent.valueWithPath('scm.url');
if (sourceUrl && !containsPlaceholder(sourceUrl)) {
result.sourceUrl = sourceUrl
.replace((0, regex_1.regEx)(/^scm:/), '')
.replace((0, regex_1.regEx)(/^git:/), '')
.replace((0, regex_1.regEx)(/^git@github.com:/), 'https://github.com/')
.replace((0, regex_1.regEx)(/^git@github.com\//), 'https://github.com/');
if (result.sourceUrl.startsWith('//')) {
// most likely the result of us stripping scm:, git: etc
// going with prepending https: here which should result in potential information retrival
result.sourceUrl = `https:${result.sourceUrl}`;
}
}
const relocation = pomContent.descendantWithPath('distributionManagement.relocation');
if (relocation) {
const relocationGroup = relocation.valueWithPath('groupId') ?? dependency.group;
const relocationName = relocation.valueWithPath('artifactId') ?? dependency.name;
result.replacementName = `${relocationGroup}:${relocationName}`;
const relocationVersion = relocation.valueWithPath('version');
result.replacementVersion = relocationVersion ?? version;
const relocationMessage = relocation.valueWithPath('message');
if (relocationMessage) {
result.deprecationMessage = relocationMessage;
}
}
const groupId = pomContent.valueWithPath('groupId');
if (groupId) {
result.packageScope = groupId;
}
const parent = pomContent.childNamed('parent');
if (recursionLimit > 0 &&
parent &&
(!result.sourceUrl || !result.homepage)) {
// if we found a parent and are missing some information
// trying to get the scm/homepage information from it
const [parentGroupId, parentArtifactId, parentVersion] = [
'groupId',
'artifactId',
'version',
].map((k) => parent.valueWithPath(k)?.replace(/\s+/g, ''));
if (parentGroupId && parentArtifactId && parentVersion) {
const parentDisplayId = `${parentGroupId}:${parentArtifactId}`;
const parentDependency = getDependencyParts(parentDisplayId);
const parentInformation = await getDependencyInfo(http, parentDependency, repoUrl, parentVersion, recursionLimit - 1);
if (!result.sourceUrl && parentInformation.sourceUrl) {
result.sourceUrl = parentInformation.sourceUrl;
}
if (!result.homepage && parentInformation.homepage) {
result.homepage = parentInformation.homepage;
}
}
}
return result;
});
return dependencyInfoResult.unwrapOr({});
}
//# sourceMappingURL=util.js.map