renovate
Version:
Automated dependency updates. Flexible so you don't need to be.
277 lines • 12.8 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.DebDatasource = void 0;
const tslib_1 = require("tslib");
const readline_1 = tslib_1.__importDefault(require("readline"));
const nanoid_1 = require("nanoid");
const upath_1 = tslib_1.__importDefault(require("upath"));
const logger_1 = require("../../../logger");
const decorator_1 = require("../../../util/cache/package/decorator");
const fs = tslib_1.__importStar(require("../../../util/fs"));
const hash_1 = require("../../../util/hash");
const url_1 = require("../../../util/url");
const datasource_1 = require("../datasource");
const checksum_1 = require("./checksum");
const common_1 = require("./common");
const file_1 = require("./file");
const release_1 = require("./release");
const url_2 = require("./url");
class DebDatasource extends datasource_1.Datasource {
static id = 'deb';
constructor() {
super(DebDatasource.id);
}
/**
* Users are able to specify custom Debian repositories as long as they follow
* the Debian package repository format as specified here
* @see{https://wiki.debian.org/DebianRepository/Format}
*/
customRegistrySupport = true;
/**
* Users can specify multiple upstream repositories and the datasource will aggregate the release
* @example
* When specifying multiple dependencies both internal and external dependencies from internal/external artifactory
*/
registryStrategy = 'merge';
/**
* The original apt source list file format is
* deb uri distribution [component1] [component2] [...]
* @see{https://wiki.debian.org/DebianRepository/Format}
*
* However, for Renovate, we require the registry URLs to be
* valid URLs which is why the parameters are encoded in the URL.
*
* The following query parameters are required:
* - components: comma separated list of components
* - suite: stable, oldstable or other alias for a release, either this or release must be given like buster
* - binaryArch: e.g. amd64 resolves to http://deb.debian.org/debian/dists/stable/non-free/binary-amd64/
*/
defaultRegistryUrls = [
'https://deb.debian.org/debian?suite=stable&components=main,contrib,non-free&binaryArch=amd64',
];
defaultVersioning = 'deb';
/**
* Downloads and extracts a package file from a component URL.
*
* @param componentUrl - The URL of the component.
* @returns The path to the extracted file and the last modification timestamp.
* @throws Will throw an error if no valid compression method is found.
*/
async downloadAndExtractPackage(componentUrl) {
const packageUrlHash = (0, hash_1.toSha256)(componentUrl);
const fullCacheDir = await fs.ensureCacheDir(common_1.cacheSubDir);
const extractedFile = upath_1.default.join(fullCacheDir, `${packageUrlHash}.txt`);
let lastTimestamp = await (0, file_1.getFileCreationTime)(extractedFile);
const compression = 'gz';
const compressedFile = upath_1.default.join(fullCacheDir, `${(0, nanoid_1.nanoid)()}_${packageUrlHash}.${compression}`);
const wasUpdated = await this.downloadPackageFile(componentUrl, compression, compressedFile, lastTimestamp);
if (wasUpdated || !lastTimestamp) {
try {
await (0, file_1.extract)(compressedFile, compression, extractedFile);
lastTimestamp = await (0, file_1.getFileCreationTime)(extractedFile);
}
catch (error) {
logger_1.logger.warn({
compressedFile,
componentUrl,
compression,
error: error.message,
}, 'Failed to extract package file from compressed file');
}
finally {
await fs.rmCache(compressedFile);
}
}
if (!lastTimestamp) {
//extracting went wrong
throw new Error('Missing metadata in extracted package index file!');
}
return { extractedFile, lastTimestamp };
}
/**
* Downloads a package file if it has been modified since the last download timestamp.
*
* @param basePackageUrl - The base URL of the package.
* @param compression - The compression method used (e.g., 'gz').
* @param compressedFile - The path where the compressed file will be saved.
* @param lastDownloadTimestamp - The timestamp of the last download.
* @returns True if the file was downloaded, otherwise false.
*/
async downloadPackageFile(basePackageUrl, compression, compressedFile, lastDownloadTimestamp) {
const baseSuiteUrl = (0, url_2.getBaseSuiteUrl)(basePackageUrl);
const packageUrl = (0, url_1.joinUrlParts)(basePackageUrl, `Packages.${compression}`);
let needsToDownload = true;
if (lastDownloadTimestamp) {
needsToDownload = await this.checkIfModified(packageUrl, lastDownloadTimestamp);
}
if (!needsToDownload) {
logger_1.logger.debug(`No need to download ${packageUrl}, file is up to date.`);
return false;
}
const readStream = this.http.stream(packageUrl);
const writeStream = fs.createCacheWriteStream(compressedFile);
await fs.pipeline(readStream, writeStream);
logger_1.logger.debug({ url: packageUrl, targetFile: compressedFile }, 'Downloading Debian package file');
let inReleaseContent = '';
try {
inReleaseContent = await this.fetchInReleaseFile(baseSuiteUrl);
}
catch (error) {
// This is expected to fail for Artifactory if GPG verification is not enabled
logger_1.logger.debug({ url: baseSuiteUrl, err: error }, 'Could not fetch InRelease file');
}
if (inReleaseContent) {
const actualChecksum = await (0, checksum_1.computeFileChecksum)(compressedFile);
const expectedChecksum = (0, checksum_1.parseChecksumsFromInRelease)(inReleaseContent,
// path to the Package.gz file
packageUrl.replace(`${baseSuiteUrl}/`, ''));
if (actualChecksum !== expectedChecksum) {
await fs.rmCache(compressedFile);
throw new Error('SHA256 checksum validation failed');
}
}
return needsToDownload;
}
/**
* Fetches the content of the InRelease file from the given base suite URL.
*
* @param baseReleaseUrl - The base URL of the suite (e.g., 'https://deb.debian.org/debian/dists/bullseye').
* @returns resolves to the content of the InRelease file.
* @throws An error if the InRelease file could not be downloaded.
*/
async fetchInReleaseFile(baseReleaseUrl) {
const inReleaseUrl = (0, url_1.joinUrlParts)(baseReleaseUrl, 'InRelease');
const response = await this.http.getText(inReleaseUrl);
return response.body;
}
/**
* Checks if a packageUrl content has been modified since the specified timestamp.
*
* @param packageUrl - The URL to check.
* @param lastDownloadTimestamp - The timestamp of the last download.
* @returns True if the content has been modified, otherwise false.
* @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since
*/
async checkIfModified(packageUrl, lastDownloadTimestamp) {
const options = {
headers: {
'If-Modified-Since': lastDownloadTimestamp.toUTCString(),
},
};
try {
const response = await this.http.head(packageUrl, options);
return response.statusCode !== 304;
}
catch (error) {
logger_1.logger.warn({ packageUrl, lastDownloadTimestamp, errorMessage: error.message }, 'Could not determine if package file is modified since last download');
return true; // Assume it needs to be downloaded if check fails
}
}
/**
* Parses the extracted package index file.
*
* @param extractedFile - The path to the extracted package file.
* @param lastTimestamp - The timestamp of the last modification.
* @returns a list of packages with minimal Metadata.
*/
async parseExtractedPackageIndex(extractedFile, _lastTimestamp) {
// read line by line to avoid high memory consumption as the extracted Packages
// files can be multiple MBs in size
const rl = readline_1.default.createInterface({
input: fs.createCacheReadStream(extractedFile),
terminal: false,
});
let currentPackage = {};
// A Package Index can contain multiple Versions of the package on private Artifactory (e.g. Jfrog)
const allPackages = {};
for await (const line of rl) {
if (line === '') {
// All information of the package are available, add to the list of packages
if (common_1.requiredPackageKeys.every((key) => key in currentPackage)) {
if (!allPackages[currentPackage.Package]) {
allPackages[currentPackage.Package] = [];
}
allPackages[currentPackage.Package].push(currentPackage);
currentPackage = {};
}
}
else {
for (const key of common_1.packageKeys) {
if (line.startsWith(`${key}:`)) {
currentPackage[key] = line.substring(key.length + 1).trim();
break;
}
}
}
}
// Check the last package after file reading is complete
if (common_1.requiredPackageKeys.every((key) => key in currentPackage)) {
if (!allPackages[currentPackage.Package]) {
allPackages[currentPackage.Package] = [];
}
allPackages[currentPackage.Package].push(currentPackage);
}
return allPackages;
}
async getPackageIndex(componentUrl) {
const { extractedFile, lastTimestamp } = await this.downloadAndExtractPackage(componentUrl);
return await this.parseExtractedPackageIndex(extractedFile, lastTimestamp);
}
/**
* Fetches the release information for a given package from the registry URL.
*
* @param config - Configuration for fetching releases.
* @returns The release result if the package is found, otherwise null.
*/
async getReleases({ registryUrl, packageName, }) {
/* v8 ignore next 3 -- should never happen */
if (!registryUrl) {
return null;
}
const componentUrls = (0, url_2.constructComponentUrls)(registryUrl);
let aggregatedRelease = null;
for (const componentUrl of componentUrls) {
try {
const packageIndex = await this.getPackageIndex(componentUrl);
const parsedPackages = packageIndex[packageName];
if (parsedPackages) {
const newRelease = (0, release_1.formatReleaseResult)(parsedPackages);
if (aggregatedRelease === null) {
aggregatedRelease = newRelease;
}
else {
if (!(0, release_1.releaseMetaInformationMatches)(aggregatedRelease, newRelease)) {
logger_1.logger.warn({ packageName }, 'Package occurred in more than one repository with different meta information. Aggregating releases anyway.');
}
aggregatedRelease.releases.push(...newRelease.releases);
}
}
}
catch (error) {
logger_1.logger.debug({ componentUrl, error }, 'Skipping component due to an error');
}
}
return aggregatedRelease;
}
}
exports.DebDatasource = DebDatasource;
tslib_1.__decorate([
(0, decorator_1.cache)({
namespace: `datasource-${DebDatasource.id}`,
key: (extractedFile, lastTimestamp) => `${extractedFile}:${lastTimestamp.getTime()}`,
ttlMinutes: 24 * 60,
})
], DebDatasource.prototype, "parseExtractedPackageIndex", null);
tslib_1.__decorate([
(0, decorator_1.cache)({
namespace: `datasource-${DebDatasource.id}`,
key: (componentUrl) => componentUrl,
})
], DebDatasource.prototype, "getPackageIndex", null);
tslib_1.__decorate([
(0, decorator_1.cache)({
namespace: `datasource-${DebDatasource.id}`,
key: ({ registryUrl, packageName }) => `${registryUrl}:${packageName}`,
})
], DebDatasource.prototype, "getReleases", null);
//# sourceMappingURL=index.js.map