turborepo-remote-cache
Version:
Turborepo remote cache server
163 lines • 14.1 kB
JavaScript
import { posix as posixPath } from 'path';
import { Readable, pipeline as pipelineCallback } from 'stream';
import { promisify } from 'util';
import { notImplemented } from '@hapi/boom';
import { STORAGE_PROVIDERS } from '../../../env.js';
import { createAzureBlobStorage, } from './azure-blob-storage.js';
import { cleanStaleLocalArtifacts } from './clean-local.js';
import { createGoogleCloudStorage, } from './google-cloud-storage.js';
import { createLocal, getLocalRootPath, } from './local.js';
import { createS3 } from './s3.js';
const pipeline = promisify(pipelineCallback);
const TURBO_CACHE_FOLDER_NAME = 'turborepocache';
const TURBO_CACHE_USE_TMP_FOLDER = true;
export function getArtifactPath(team, artifactId) {
return posixPath.join(team, artifactId);
}
export class ArtifactNotFoundError extends Error {
constructor(artifactPath) {
super(`Artifact ${artifactPath} doesn't exist.`);
this.name = 'ArtifactNotFoundError';
}
}
function createStorageLocation(provider, providerOptions) {
const { path = TURBO_CACHE_FOLDER_NAME, useTmp = TURBO_CACHE_USE_TMP_FOLDER, } = providerOptions;
switch (provider) {
case STORAGE_PROVIDERS.LOCAL: {
return createLocal({ path, useTmp });
}
case STORAGE_PROVIDERS.S3:
case STORAGE_PROVIDERS.s3: {
const { accessKey, secretKey, region, endpoint, maxSockets } = providerOptions;
return createS3({
accessKey,
secretKey,
bucket: path,
region,
endpoint,
maxSockets,
s3OptionsPassthrough: {
forcePathStyle: true,
},
});
}
case STORAGE_PROVIDERS.MINIO: {
const { accessKey, secretKey, region, endpoint, maxSockets } = providerOptions;
return createS3({
accessKey,
secretKey,
bucket: path,
region,
endpoint,
maxSockets,
s3OptionsPassthrough: {
forcePathStyle: true,
},
});
}
case STORAGE_PROVIDERS.GOOGLE_CLOUD_STORAGE: {
const { clientEmail, privateKey, projectId } = providerOptions;
return createGoogleCloudStorage({
bucket: path,
clientEmail,
privateKey,
projectId,
});
}
case STORAGE_PROVIDERS.AZURE_BLOB_STORAGE: {
const { connectionString } = providerOptions;
return createAzureBlobStorage({ containerName: path, connectionString });
}
default:
throw new Error(`Unsupported storage provider '${provider}'. Please select one of the following: ${Object.values(STORAGE_PROVIDERS).join(', ')}!`);
}
}
export function createLocation(provider, providerOptions) {
const location = createStorageLocation(provider, providerOptions);
function getArtifactTagPath(artifactId, team) {
return getArtifactPath(team, `${artifactId}.tag`);
}
async function getCachedArtifact(artifactId, team) {
return new Promise((resolve, reject) => {
const artifactPath = getArtifactPath(team, artifactId);
location.exists(artifactPath, (err, exists) => {
if (err) {
return reject(err);
}
if (!exists) {
return reject(new ArtifactNotFoundError(artifactPath));
}
resolve(location.createReadStream(artifactPath));
});
});
}
async function existsCachedArtifact(artifactId, team) {
return new Promise((resolve, reject) => {
const artifactPath = getArtifactPath(team, artifactId);
location.exists(artifactPath, (err, exists) => {
if (err) {
return reject(err);
}
if (!exists) {
return reject(new ArtifactNotFoundError(artifactPath));
}
resolve();
});
});
}
async function createCachedArtifact(artifactId, team, artifact) {
return pipeline(artifact, location.createWriteStream(getArtifactPath(team, artifactId)));
}
async function getCachedArtifactTag(artifactId, team) {
return new Promise((resolve, reject) => {
const tagPath = getArtifactTagPath(artifactId, team);
location.exists(tagPath, (err, exists) => {
if (err) {
return reject(err);
}
if (!exists) {
return reject(new ArtifactNotFoundError(tagPath));
}
const stream = location.createReadStream(tagPath);
const chunks = [];
stream.on('data', (chunk) => chunks.push(chunk));
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
stream.on('error', reject);
});
});
}
async function existsCachedArtifactTag(artifactId, team) {
return new Promise((resolve, reject) => {
const tagPath = getArtifactTagPath(artifactId, team);
location.exists(tagPath, (err, exists) => {
if (err) {
return reject(err);
}
if (!exists) {
return reject(new ArtifactNotFoundError(tagPath));
}
resolve();
});
});
}
async function createCachedArtifactTag(artifactId, team, tag) {
return pipeline(Readable.from(tag), location.createWriteStream(getArtifactTagPath(artifactId, team)));
}
async function cleanStaleArtifacts(team, olderThanDays) {
if (provider !== STORAGE_PROVIDERS.LOCAL) {
throw notImplemented('Clean is only supported for the local storage provider');
}
const { path = TURBO_CACHE_FOLDER_NAME, useTmp = TURBO_CACHE_USE_TMP_FOLDER, } = providerOptions;
return cleanStaleLocalArtifacts(getLocalRootPath({ path, useTmp }), team, olderThanDays);
}
return {
getCachedArtifact,
createCachedArtifact,
existsCachedArtifact,
getCachedArtifactTag,
existsCachedArtifactTag,
createCachedArtifactTag,
cleanStaleArtifacts,
};
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9zcmMvcGx1Z2lucy9yZW1vdGUtY2FjaGUvc3RvcmFnZS9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxPQUFPLEVBQUUsS0FBSyxJQUFJLFNBQVMsRUFBRSxNQUFNLE1BQU0sQ0FBQTtBQUN6QyxPQUFPLEVBQUUsUUFBUSxFQUFFLFFBQVEsSUFBSSxnQkFBZ0IsRUFBRSxNQUFNLFFBQVEsQ0FBQTtBQUMvRCxPQUFPLEVBQUUsU0FBUyxFQUFFLE1BQU0sTUFBTSxDQUFBO0FBQ2hDLE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSxZQUFZLENBQUE7QUFDM0MsT0FBTyxFQUFFLGlCQUFpQixFQUFFLE1BQU0saUJBQWlCLENBQUE7QUFDbkQsT0FBTyxFQUVMLHNCQUFzQixHQUN2QixNQUFNLHlCQUF5QixDQUFBO0FBQ2hDLE9BQU8sRUFBRSx3QkFBd0IsRUFBRSxNQUFNLGtCQUFrQixDQUFBO0FBQzNELE9BQU8sRUFFTCx3QkFBd0IsR0FDekIsTUFBTSwyQkFBMkIsQ0FBQTtBQUNsQyxPQUFPLEVBRUwsV0FBVyxFQUNYLGdCQUFnQixHQUNqQixNQUFNLFlBQVksQ0FBQTtBQUNuQixPQUFPLEVBQTRCLFFBQVEsRUFBRSxNQUFNLFNBQVMsQ0FBQTtBQUU1RCxNQUFNLFFBQVEsR0FBRyxTQUFTLENBQUMsZ0JBQWdCLENBQUMsQ0FBQTtBQUM1QyxNQUFNLHVCQUF1QixHQUFHLGdCQUF5QixDQUFBO0FBQ3pELE1BQU0sMEJBQTBCLEdBQUcsSUFBYSxDQUFBO0FBV2hELE1BQU0sVUFBVSxlQUFlLENBQUMsSUFBWSxFQUFFLFVBQWtCO0lBQzlELE9BQU8sU0FBUyxDQUFDLElBQUksQ0FBQyxJQUFJLEVBQUUsVUFBVSxDQUFDLENBQUE7QUFDekMsQ0FBQztBQXlCRCxNQUFNLE9BQU8scUJBQXNCLFNBQVEsS0FBSztJQUM5QyxZQUFZLFlBQW9CO1FBQzlCLEtBQUssQ0FBQyxZQUFZLFlBQVksaUJBQWlCLENBQUMsQ0FBQTtRQUNoRCxJQUFJLENBQUMsSUFBSSxHQUFHLHVCQUF1QixDQUFBO0lBQ3JDLENBQUM7Q0FDRjtBQVlELFNBQVMscUJBQXFCLENBQzVCLFFBQWtCLEVBQ2xCLGVBQTBDO0lBRTFDLE1BQU0sRUFDSixJQUFJLEdBQUcsdUJBQXVCLEVBQzlCLE1BQU0sR0FBRywwQkFBMEIsR0FDcEMsR0FBRyxlQUFlLENBQUE7SUFFbkIsUUFBUSxRQUFRLEVBQUUsQ0FBQztRQUNqQixLQUFLLGlCQUFpQixDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUM7WUFDN0IsT0FBTyxXQUFXLENBQUMsRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLENBQUMsQ0FBQTtRQUN0QyxDQUFDO1FBQ0QsS0FBSyxpQkFBaUIsQ0FBQyxFQUFFLENBQUM7UUFDMUIsS0FBSyxpQkFBaUIsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDO1lBQzFCLE1BQU0sRUFBRSxTQUFTLEVBQUUsU0FBUyxFQUFFLE1BQU0sRUFBRSxRQUFRLEVBQUUsVUFBVSxFQUFFLEdBQzFELGVBQTRCLENBQUE7WUFDOUIsT0FBTyxRQUFRLENBQUM7Z0JBQ2QsU0FBUztnQkFDVCxTQUFTO2dCQUNULE1BQU0sRUFBRSxJQUFJO2dCQUNaLE1BQU07Z0JBQ04sUUFBUTtnQkFDUixVQUFVO2dCQUNWLG9CQUFvQixFQUFFO29CQUNwQixjQUFjLEVBQUUsSUFBSTtpQkFDckI7YUFDRixDQUFDLENBQUE7UUFDSixDQUFDO1FBQ0QsS0FBSyxpQkFBaUIsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDO1lBQzdCLE1BQU0sRUFBRSxTQUFTLEVBQUUsU0FBUyxFQUFFLE1BQU0sRUFBRSxRQUFRLEVBQUUsVUFBVSxFQUFFLEdBQzFELGVBQTRCLENBQUE7WUFDOUIsT0FBTyxRQUFRLENBQUM7Z0JBQ2QsU0FBUztnQkFDVCxTQUFTO2dCQUNULE1BQU0sRUFBRSxJQUFJO2dCQUNaLE1BQU07Z0JBQ04sUUFBUTtnQkFDUixVQUFVO2dCQUNWLG9CQUFvQixFQUFFO29CQUNwQixjQUFjLEVBQUUsSUFBSTtpQkFDckI7YUFDRixDQUFDLENBQUE7UUFDSixDQUFDO1FBQ0QsS0FBSyxpQkFBaUIsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDLENBQUM7WUFDNUMsTUFBTSxFQUFFLFdBQVcsRUFBRSxVQUFVLEVBQUUsU0FBUyxFQUFFLEdBQzFDLGVBQTRDLENBQUE7WUFDOUMsT0FBTyx3QkFBd0IsQ0FBQztnQkFDOUIsTUFBTSxFQUFFLElBQUk7Z0JBQ1osV0FBVztnQkFDWCxVQUFVO2dCQUNWLFNBQVM7YUFDVixDQUFDLENBQUE7UUFDSixDQUFDO1FBQ0QsS0FBSyxpQkFBaUIsQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDLENBQUM7WUFDMUMsTUFBTSxFQUFFLGdCQUFnQixFQUFFLEdBQUcsZUFBMEMsQ0FBQTtZQUN2RSxPQUFPLHNCQUFzQixDQUFDLEVBQUUsYUFBYSxFQUFFLElBQUksRUFBRSxnQkFBZ0IsRUFBRSxDQUFDLENBQUE7UUFDMUUsQ0FBQztRQUNEO1lBQ0UsTUFBTSxJQUFJLEtBQUssQ0FDYixpQ0FBaUMsUUFBUSwwQ0FBMEMsTUFBTSxDQUFDLE1BQU0sQ0FDOUYsaUJBQWlCLENBQ2xCLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQ2hCLENBQUE7SUFDTCxDQUFDO0FBQ0gsQ0FBQztBQUVELE1BQU0sVUFBVSxjQUFjLENBQzVCLFFBQWtCLEVBQ2xCLGVBQTBDO0lBRTFDLE1BQU0sUUFBUSxHQUFHLHFCQUFxQixDQUFDLFFBQVEsRUFBRSxlQUFlLENBQUMsQ0FBQTtJQUVqRSxTQUFTLGtCQUFrQixDQUFDLFVBQWtCLEVBQUUsSUFBWTtRQUMxRCxPQUFPLGVBQWUsQ0FBQyxJQUFJLEVBQUUsR0FBRyxVQUFVLE1BQU0sQ0FBQyxDQUFBO0lBQ25ELENBQUM7SUFFRCxLQUFLLFVBQVUsaUJBQWlCLENBQUMsVUFBa0IsRUFBRSxJQUFZO1FBQy9ELE9BQU8sSUFBSSxPQUFPLENBQUMsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLEVBQUU7WUFDckMsTUFBTSxZQUFZLEdBQUcsZUFBZSxDQUFDLElBQUksRUFBRSxVQUFVLENBQUMsQ0FBQTtZQUN0RCxRQUFRLENBQUMsTUFBTSxDQUFDLFlBQVksRUFBRSxDQUFDLEdBQUcsRUFBRSxNQUFNLEVBQUUsRUFBRTtnQkFDNUMsSUFBSSxHQUFHLEVBQUUsQ0FBQztvQkFDUixPQUFPLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQTtnQkFDcEIsQ0FBQztnQkFDRCxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7b0JBQ1osT0FBTyxNQUFNLENBQUMsSUFBSSxxQkFBcUIsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFBO2dCQUN4RCxDQUFDO2dCQUNELE9BQU8sQ0FBQyxRQUFRLENBQUMsZ0JBQWdCLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQTtZQUNsRCxDQUFDLENBQUMsQ0FBQTtRQUNKLENBQUMsQ0FBQyxDQUFBO0lBQ0osQ0FBQztJQUVELEtBQUssVUFBVSxvQkFBb0IsQ0FBQyxVQUFrQixFQUFFLElBQVk7UUFDbEUsT0FBTyxJQUFJLE9BQU8sQ0FBTyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsRUFBRTtZQUMzQyxNQUFNLFlBQVksR0FBRyxlQUFlLENBQUMsSUFBSSxFQUFFLFVBQVUsQ0FBQyxDQUFBO1lBQ3RELFFBQVEsQ0FBQyxNQUFNLENBQUMsWUFBWSxFQUFFLENBQUMsR0FBRyxFQUFFLE1BQU0sRUFBRSxFQUFFO2dCQUM1QyxJQUFJLEdBQUcsRUFBRSxDQUFDO29CQUNSLE9BQU8sTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFBO2dCQUNwQixDQUFDO2dCQUNELElBQUksQ0FBQyxNQUFNLEVBQUUsQ0FBQztvQkFDWixPQUFPLE1BQU0sQ0FBQyxJQUFJLHFCQUFxQixDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUE7Z0JBQ3hELENBQUM7Z0JBQ0QsT0FBTyxFQUFFLENBQUE7WUFDWCxDQUFDLENBQUMsQ0FBQTtRQUNKLENBQUMsQ0FBQyxDQUFBO0lBQ0osQ0FBQztJQUVELEtBQUssVUFBVSxvQkFBb0IsQ0FDakMsVUFBa0IsRUFDbEIsSUFBWSxFQUNaLFFBQWtCO1FBRWxCLE9BQU8sUUFBUSxDQUNiLFFBQVEsRUFDUixRQUFRLENBQUMsaUJBQWlCLENBQUMsZUFBZSxDQUFDLElBQUksRUFBRSxVQUFVLENBQUMsQ0FBQyxDQUM5RCxDQUFBO0lBQ0gsQ0FBQztJQUVELEtBQUssVUFBVSxvQkFBb0IsQ0FDakMsVUFBa0IsRUFDbEIsSUFBWTtRQUVaLE9BQU8sSUFBSSxPQUFPLENBQUMsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLEVBQUU7WUFDckMsTUFBTSxPQUFPLEdBQUcsa0JBQWtCLENBQUMsVUFBVSxFQUFFLElBQUksQ0FBQyxDQUFBO1lBQ3BELFFBQVEsQ0FBQyxNQUFNLENBQUMsT0FBTyxFQUFFLENBQUMsR0FBRyxFQUFFLE1BQU0sRUFBRSxFQUFFO2dCQUN2QyxJQUFJLEdBQUcsRUFBRSxDQUFDO29CQUNSLE9BQU8sTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFBO2dCQUNwQixDQUFDO2dCQUNELElBQUksQ0FBQyxNQUFNLEVBQUUsQ0FBQztvQkFDWixPQUFPLE1BQU0sQ0FBQyxJQUFJLHFCQUFxQixDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUE7Z0JBQ25ELENBQUM7Z0JBQ0QsTUFBTSxNQUFNLEdBQUcsUUFBUSxDQUFDLGdCQUFnQixDQUFDLE9BQU8sQ0FBQyxDQUFBO2dCQUNqRCxNQUFNLE1BQU0sR0FBYSxFQUFFLENBQUE7Z0JBQzNCLE1BQU0sQ0FBQyxFQUFFLENBQUMsTUFBTSxFQUFFLENBQUMsS0FBYSxFQUFFLEVBQUUsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUE7Z0JBQ3hELE1BQU0sQ0FBQyxFQUFFLENBQUMsS0FBSyxFQUFFLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUE7Z0JBQ3ZFLE1BQU0sQ0FBQyxFQUFFLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxDQUFBO1lBQzVCLENBQUMsQ0FBQyxDQUFBO1FBQ0osQ0FBQyxDQUFDLENBQUE7SUFDSixDQUFDO0lBRUQsS0FBSyxVQUFVLHVCQUF1QixDQUNwQyxVQUFrQixFQUNsQixJQUFZO1FBRVosT0FBTyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsRUFBRTtZQUNyQyxNQUFNLE9BQU8sR0FBRyxrQkFBa0IsQ0FBQyxVQUFVLEVBQUUsSUFBSSxDQUFDLENBQUE7WUFDcEQsUUFBUSxDQUFDLE1BQU0sQ0FBQyxPQUFPLEVBQUUsQ0FBQyxHQUFHLEVBQUUsTUFBTSxFQUFFLEVBQUU7Z0JBQ3ZDLElBQUksR0FBRyxFQUFFLENBQUM7b0JBQ1IsT0FBTyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUE7Z0JBQ3BCLENBQUM7Z0JBQ0QsSUFBSSxDQUFDLE1BQU0sRUFBRSxDQUFDO29CQUNaLE9BQU8sTUFBTSxDQUFDLElBQUkscUJBQXFCLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQTtnQkFDbkQsQ0FBQztnQkFDRCxPQUFPLEVBQUUsQ0FBQTtZQUNYLENBQUMsQ0FBQyxDQUFBO1FBQ0osQ0FBQyxDQUFDLENBQUE7SUFDSixDQUFDO0lBRUQsS0FBSyxVQUFVLHVCQUF1QixDQUNwQyxVQUFrQixFQUNsQixJQUFZLEVBQ1osR0FBVztRQUVYLE9BQU8sUUFBUSxDQUNiLFFBQVEsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQ2xCLFFBQVEsQ0FBQyxpQkFBaUIsQ0FBQyxrQkFBa0IsQ0FBQyxVQUFVLEVBQUUsSUFBSSxDQUFDLENBQUMsQ0FDakUsQ0FBQTtJQUNILENBQUM7SUFFRCxLQUFLLFVBQVUsbUJBQW1CLENBQUMsSUFBWSxFQUFFLGFBQXFCO1FBQ3BFLElBQUksUUFBUSxLQUFLLGlCQUFpQixDQUFDLEtBQUssRUFBRSxDQUFDO1lBQ3pDLE1BQU0sY0FBYyxDQUNsQix3REFBd0QsQ0FDekQsQ0FBQTtRQUNILENBQUM7UUFFRCxNQUFNLEVBQ0osSUFBSSxHQUFHLHVCQUF1QixFQUM5QixNQUFNLEdBQUcsMEJBQTBCLEdBQ3BDLEdBQUcsZUFBZSxDQUFBO1FBRW5CLE9BQU8sd0JBQXdCLENBQzdCLGdCQUFnQixDQUFDLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxDQUFDLEVBQ2xDLElBQUksRUFDSixhQUFhLENBQ2QsQ0FBQTtJQUNILENBQUM7SUFFRCxPQUFPO1FBQ0wsaUJBQWlCO1FBQ2pCLG9CQUFvQjtRQUNwQixvQkFBb0I7UUFDcEIsb0JBQW9CO1FBQ3BCLHVCQUF1QjtRQUN2Qix1QkFBdUI7UUFDdkIsbUJBQW1CO0tBQ3BCLENBQUE7QUFDSCxDQUFDIn0=