@factorialco/shadowdog
Version:
<img src="https://raw.githubusercontent.com/factorialco/shadowdog/refs/heads/main/logo.png" alt="drawing" width="100"/>
286 lines (285 loc) • 15.6 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const chalk_1 = __importDefault(require("chalk"));
const child_process_1 = require("child_process");
const fs_extra_1 = __importDefault(require("fs-extra"));
const minio = __importStar(require("minio"));
const os = __importStar(require("os"));
const path_1 = __importDefault(require("path"));
const tar = __importStar(require("tar"));
const zlib = __importStar(require("zlib"));
const utils_1 = require("../utils");
const createClient = () => {
const { AWS_PROFILE } = process.env;
if (AWS_PROFILE) {
try {
const credentials = JSON.parse((0, child_process_1.execSync)(`aws configure export-credentials --profile "${AWS_PROFILE}"`).toString());
return new minio.Client({
endPoint: 's3.amazonaws.com',
useSSL: true,
accessKey: credentials.AccessKeyId,
secretKey: credentials.SecretAccessKey,
sessionToken: credentials.SessionToken,
region: (0, child_process_1.execSync)(`aws configure get region --profile "${AWS_PROFILE}"`).toString().trim(),
});
}
catch {
(0, utils_1.logMessage)(`🌐 Not able to create a client for remote cache because of failing authentication with AWS_PROFILE`);
return null;
}
}
const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION } = process.env;
if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY || !AWS_REGION) {
(0, utils_1.logMessage)(`🌐 Not able to create a client for remote cache because of missing AWS credentials`);
return null;
}
return new minio.Client({
endPoint: 's3.amazonaws.com',
useSSL: true,
accessKey: AWS_ACCESS_KEY_ID,
secretKey: AWS_SECRET_ACCESS_KEY,
region: AWS_REGION,
});
};
const saveRemoteCache = async (client, bucket, stream, objectName, artifact) => {
var _a;
try {
await client.putObject(bucket, objectName, stream, stream.readableLength, {
output: artifact.output,
extra: (_a = process.env.SHADOWDOG_REMOTE_CACHE_EXTRA) !== null && _a !== void 0 ? _a : '',
});
}
catch (error) {
(0, utils_1.logMessage)(`🌐 Not able to store artifact '${chalk_1.default.blue(artifact.output)}' -> '${chalk_1.default.green(objectName)}' in remote cache`);
(0, utils_1.logError)(error);
return false;
}
return true;
};
const restoreRemoteCache = async (client, bucket, objectName, artifact, outputPath) => {
const stream = await client.getObject(bucket, objectName);
const targetPath = outputPath !== null && outputPath !== void 0 ? outputPath : path_1.default.join(process.cwd(), artifact.output, '..');
fs_extra_1.default.mkdirpSync(targetPath);
return new Promise((resolve, reject) => {
const extractStream = stream.pipe(tar.extract({
cwd: targetPath,
filter: (filePath) => filterFn(artifact.ignore, artifact.output, filePath),
}));
extractStream.on('end', () => {
resolve(null);
});
extractStream.on('error', reject);
});
};
const filterFn = (ignore, outputPath, filePath) => {
if (!ignore) {
return true;
}
const keep = !ignore.includes(path_1.default.join(outputPath, '..', filePath));
if (!keep) {
(0, utils_1.logMessage)(`🗜️ Ignored file '${chalk_1.default.blue(filePath)}' during compression because of the ignore list`);
}
return keep;
};
const restoreCache = async (client, commandConfig, currentCache, pluginOptions) => {
// Check if we can reuse some artifacts from the cache
const promisesToGenerate = commandConfig.artifacts.map(async (artifact) => {
const start = Date.now();
const cacheFileName = (0, utils_1.computeFileCacheName)(currentCache, artifact.output);
const cacheFilePath = path_1.default.join(pluginOptions.path, `${cacheFileName}.tar.gz`);
try {
// Check if artifact exists in remote cache by attempting to get object stats
try {
await client.statObject(pluginOptions.bucketName, cacheFilePath);
}
catch {
// Object doesn't exist in remote cache
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`🌐 Not able to reuse artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from remote cache because of cache ${chalk_1.default.bgRed('MISS')} ${chalk_1.default.cyan(`(${seconds}s)`)}`);
return artifact;
}
const artifactPath = path_1.default.join(process.cwd(), artifact.output);
const artifactExists = await fs_extra_1.default.exists(artifactPath);
// Double-check: verify that the file doesn't exist or that the content doesn't match the computed SHA
if (artifactExists) {
// Extract to a temporary location to compute its SHA
const tempOutputPath = path_1.default.join(os.tmpdir(), `shadowdog-remote-cache-temp-${cacheFileName}-${Date.now()}`);
try {
// Extract cache to temp location
await restoreRemoteCache(client, pluginOptions.bucketName, cacheFilePath, artifact, tempOutputPath);
// Find the extracted artifact in temp location
// The artifact is extracted with its basename preserved
const artifactBasename = path_1.default.basename(artifact.output);
const extractedArtifactPath = path_1.default.join(tempOutputPath, artifactBasename);
// Check if extracted artifact exists (it should)
if (await fs_extra_1.default.exists(extractedArtifactPath)) {
// Compute SHA of cached artifact (from temp location)
// Use absolute path since computeArtifactContentSha expects relative to cwd
const cachedSha = (0, utils_1.computeArtifactContentSha)(path_1.default.relative(process.cwd(), extractedArtifactPath));
// Compute SHA of existing artifact
const existingSha = (0, utils_1.computeArtifactContentSha)(artifact.output);
// Clean up temp location
await fs_extra_1.default.remove(tempOutputPath);
// If SHAs match, skip restore (artifact is already correct)
if (cachedSha !== null && existingSha !== null && cachedSha === existingSha) {
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`🌐 Skipping restore of artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' because existing file matches cached content (SHA: ${chalk_1.default.cyan(cachedSha)}) ${chalk_1.default.cyan(`(${seconds}s)`)}`);
return null;
}
// SHAs don't match or one is null, proceed with restore
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`🌐 Reusing artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from remote cache because of cache ${chalk_1.default.bgGreen('HIT')} (existing SHA: ${chalk_1.default.cyan(existingSha !== null && existingSha !== void 0 ? existingSha : 'N/A')}, cached SHA: ${chalk_1.default.cyan(cachedSha !== null && cachedSha !== void 0 ? cachedSha : 'N/A')}) ${chalk_1.default.cyan(`(${seconds}s)`)}`);
}
else {
// Extracted artifact not found in expected location, proceed with restore
await fs_extra_1.default.remove(tempOutputPath);
(0, utils_1.logMessage)(`⚠️ Could not find extracted artifact in temp location for '${chalk_1.default.blue(artifact.output)}', proceeding with restore`);
}
}
catch {
// If temp extraction fails, try direct restore
try {
await fs_extra_1.default.remove(tempOutputPath);
}
catch {
// Ignore cleanup errors
}
(0, utils_1.logMessage)(`⚠️ Could not verify SHA for artifact '${chalk_1.default.blue(artifact.output)}', proceeding with restore`);
}
}
else {
// Artifact doesn't exist, proceed with restore
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`🌐 Reusing artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from remote cache because of cache ${chalk_1.default.bgGreen('HIT')} ${chalk_1.default.cyan(`(${seconds}s)`)}`);
}
// When replaceOnRestore is enabled, remove existing artifact before
// restoring from cache to prevent stale files from persisting
// (tar extract is additive and won't remove files that exist on
// disk but not in the tarball)
if (artifact.replaceOnRestore) {
const artifactFullPath = path_1.default.join(process.cwd(), artifact.output);
if (await fs_extra_1.default.exists(artifactFullPath)) {
await fs_extra_1.default.remove(artifactFullPath);
}
}
// Restore the artifact to its final location
await restoreRemoteCache(client, pluginOptions.bucketName, cacheFilePath, artifact);
return null;
}
catch (error) {
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`🌐 Not able to reuse artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from remote cache because of cache ${chalk_1.default.bgRed('MISS')} ${chalk_1.default.cyan(`(${seconds}s)`)}`);
(0, utils_1.logError)(error);
}
// If we can't reuse the artifact, we return it so it can be generated
return artifact;
});
const artifactToGenerate = await Promise.all(promisesToGenerate);
if (commandConfig.artifacts &&
commandConfig.artifacts.length > 0 &&
artifactToGenerate.filter(Boolean).length === 0 // Filtering out the artifacts that were reused from cache
) {
(0, utils_1.logMessage)(`⤵️ Skipping command '${chalk_1.default.yellow(commandConfig.command)}' generation because all artifacts were reused from remote cache`);
return true;
}
return false;
};
const middleware = async ({ config, files, environment, next, abort, options, }) => {
if (process.env.SHADOWDOG_DISABLE_REMOTE_CACHE) {
return next();
}
const client = createClient();
if (!client) {
return await next();
}
const readCache = process.env.SHADOWDOG_REMOTE_CACHE_READ
? process.env.SHADOWDOG_REMOTE_CACHE_READ === 'true'
: options.read;
const writeCache = process.env.SHADOWDOG_REMOTE_CACHE_WRITE
? process.env.SHADOWDOG_REMOTE_CACHE_WRITE === 'true'
: options.write;
const currentCache = (0, utils_1.computeCache)(files, environment, config.command);
if (readCache) {
const hasBeenRestored = await restoreCache(client, config, currentCache, options);
if (hasBeenRestored) {
return abort();
}
}
await next();
if (writeCache) {
return Promise.all(config.artifacts.map(async (artifact) => {
if (!fs_extra_1.default.existsSync(path_1.default.join(process.cwd(), artifact.output))) {
(0, utils_1.logMessage)(`🌐 Not able to store artifact '${chalk_1.default.blue(artifact.output)}' in remote cache because is not present`);
return;
}
const cacheFileName = (0, utils_1.computeFileCacheName)(currentCache, artifact.output);
const cacheFilePath = path_1.default.join(options.path, `${cacheFileName}.tar.gz`);
const sourceCacheFilePath = path_1.default.join(process.cwd(), artifact.output);
try {
const tarStream = tar.create({
gzip: false,
cwd: path_1.default.dirname(sourceCacheFilePath),
filter: (filePath) => filterFn(artifact.ignore, artifact.output, filePath),
}, [path_1.default.basename(sourceCacheFilePath)]);
tarStream.on('error', (error) => {
(0, utils_1.logError)(error);
});
const gzipStream = zlib.createGzip();
gzipStream.on('error', (error) => {
(0, utils_1.logError)(error);
});
const stream = tarStream.pipe(gzipStream);
stream.on('error', (error) => {
(0, utils_1.logError)(error);
});
(0, utils_1.logMessage)(`🌐 Storing artifact '${chalk_1.default.blue(artifact.output)}' in remote cache with value '${chalk_1.default.green(cacheFileName)}'`);
await saveRemoteCache(client, options.bucketName, stream, cacheFilePath, artifact);
}
catch (error) {
(0, utils_1.logMessage)(`🚫 An error ocurred while storing cache for artifact '${artifact.output}' with id '${chalk_1.default.green(cacheFileName)}`);
(0, utils_1.logError)(error);
}
})).catch((error) => {
(0, utils_1.logError)(error);
});
}
};
exports.default = {
middleware,
};