@factorialco/shadowdog
Version:
<img src="https://raw.githubusercontent.com/factorialco/shadowdog/refs/heads/main/logo.png" alt="drawing" width="100"/>
239 lines (238 loc) • 12.7 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.compressArtifact = void 0;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const tar = __importStar(require("tar"));
const zlib = __importStar(require("zlib"));
const chalk_1 = __importDefault(require("chalk"));
const utils_1 = require("../utils");
const compressArtifact = (folderPath, outputPath, filter) => {
return new Promise((resolve, reject) => {
const tarStream = tar.c({
gzip: false,
cwd: path.dirname(folderPath),
filter,
}, [path.basename(folderPath)]);
const gzipStream = zlib.createGzip();
const writeStream = fs.createWriteStream(outputPath);
writeStream.on('finish', () => {
resolve(null);
});
writeStream.on('error', reject);
gzipStream.on('error', reject);
tarStream.on('error', reject);
tarStream.pipe(gzipStream).pipe(writeStream);
});
};
exports.compressArtifact = compressArtifact;
const decompressArtifact = (tarGzPath, outputPath, filter) => {
return new Promise((resolve, reject) => {
fs.mkdirpSync(outputPath);
const readStream = fs.createReadStream(tarGzPath);
const unzipStream = zlib.createGunzip();
const tarExtractStream = tar.x({ cwd: outputPath, filter });
tarExtractStream.on('finish', () => {
resolve(null);
});
tarExtractStream.on('error', (err) => {
reject(err);
});
unzipStream.on('error', (err) => {
reject(err);
});
readStream.pipe(unzipStream).pipe(tarExtractStream);
});
};
const restoreCache = async (commandConfig, currentCache, { path: cachePath }) => {
// Check if we can reuse some artifacts from the cache
const promisesToGenerate = commandConfig.artifacts.map(async (artifact) => {
const start = Date.now();
const cacheFileName = (0, utils_1.computeFileCacheName)(currentCache, artifact.output);
const cacheFilePath = path.join(cachePath, `${cacheFileName}.tar.gz`);
// First, we check if the artifact is in the local file system cache
if (fs.existsSync(cacheFilePath)) {
const artifactPath = path.join(process.cwd(), artifact.output);
const artifactExists = await fs.exists(artifactPath);
// Double-check: verify that the file doesn't exist or that the content doesn't match the computed SHA
if (artifactExists) {
// Extract to a temporary location to compute its SHA
const tempOutputPath = path.join(cachePath, `.temp-${cacheFileName}-${Date.now()}`);
try {
// Extract cache to temp location
await decompressArtifact(cacheFilePath, tempOutputPath, (filePath) => filterFn(artifact.ignore, artifact.output, filePath));
// Find the extracted artifact in temp location
// The artifact is extracted with its basename preserved
const artifactBasename = path.basename(artifact.output);
const extractedArtifactPath = path.join(tempOutputPath, artifactBasename);
// Check if extracted artifact exists (it should)
if (await fs.exists(extractedArtifactPath)) {
// Compute SHA of cached artifact (from temp location)
// Use absolute path since computeArtifactContentSha expects relative to cwd
const cachedSha = (0, utils_1.computeArtifactContentSha)(path.relative(process.cwd(), extractedArtifactPath));
// Compute SHA of existing artifact
const existingSha = (0, utils_1.computeArtifactContentSha)(artifact.output);
// Clean up temp location
await fs.remove(tempOutputPath);
// If SHAs match, skip restore (artifact is already correct)
if (cachedSha !== null && existingSha !== null && cachedSha === existingSha) {
(0, utils_1.logMessage)(`📦 Skipping restore of artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' because existing file matches cached content (SHA: ${chalk_1.default.cyan(cachedSha)})`);
return null;
}
// SHAs don't match or one is null, proceed with restore
(0, utils_1.logMessage)(`📦 Reusing artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from local cache because of cache ${chalk_1.default.bgGreen('HIT')} (existing SHA: ${chalk_1.default.cyan(existingSha !== null && existingSha !== void 0 ? existingSha : 'N/A')}, cached SHA: ${chalk_1.default.cyan(cachedSha !== null && cachedSha !== void 0 ? cachedSha : 'N/A')})`);
}
else {
// Extracted artifact not found in expected location, proceed with restore
await fs.remove(tempOutputPath);
(0, utils_1.logMessage)(`⚠️ Could not find extracted artifact in temp location for '${chalk_1.default.blue(artifact.output)}', proceeding with restore`);
}
}
catch {
// If temp extraction fails, try direct restore
try {
await fs.remove(tempOutputPath);
}
catch {
// Ignore cleanup errors
}
(0, utils_1.logMessage)(`⚠️ Could not verify SHA for artifact '${chalk_1.default.blue(artifact.output)}', proceeding with restore`);
}
}
else {
// Artifact doesn't exist, proceed with restore
(0, utils_1.logMessage)(`📦 Reusing artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from local cache because of cache ${chalk_1.default.bgGreen('HIT')}`);
}
try {
// When replaceOnRestore is enabled, remove existing artifact before
// restoring from cache to prevent stale files from persisting
// (tar extract is additive and won't remove files that exist on
// disk but not in the tarball)
if (artifact.replaceOnRestore) {
const artifactFullPath = path.join(process.cwd(), artifact.output);
if (await fs.exists(artifactFullPath)) {
await fs.remove(artifactFullPath);
}
}
await decompressArtifact(cacheFilePath, path.join(process.cwd(), artifact.output, '..'), (filePath) => filterFn(artifact.ignore, artifact.output, filePath));
}
catch (error) {
(0, utils_1.logMessage)(`🚫 An error ocurred while restoring cache for artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}'`);
(0, utils_1.logError)(error);
return artifact;
}
return null;
}
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`📦 Not able to reuse artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from cache because of cache ${chalk_1.default.bgRed('MISS')} ${chalk_1.default.cyan(`(${seconds}s)`)}`);
// If we can't reuse the artifact, we return it so it can be generated
return artifact;
});
const artifactToGenerate = await Promise.all(promisesToGenerate);
if (commandConfig.artifacts &&
commandConfig.artifacts.length > 0 &&
artifactToGenerate.filter(Boolean).length === 0 // Filtering out the artifacts that were reused from cache
) {
(0, utils_1.logMessage)(`⤵️ Skipping command '${chalk_1.default.yellow(commandConfig.command)}' generation because all artifacts were reused from cache`);
return true;
}
return false;
};
const filterFn = (ignore, outputPath, filePath) => {
if (!ignore) {
return true;
}
const keep = !ignore.includes(path.join(outputPath, '..', filePath));
if (!keep) {
(0, utils_1.logMessage)(`🗜️ Ignored file '${chalk_1.default.blue(filePath)}' during compression because of the ignore list`);
}
return keep;
};
const middleware = async ({ files, environment, config, next, abort, options, }) => {
var _a;
if (process.env.SHADOWDOG_DISABLE_LOCAL_CACHE) {
return next();
}
const readCache = process.env.SHADOWDOG_LOCAL_CACHE_READ
? process.env.SHADOWDOG_LOCAL_CACHE_READ === 'true'
: options.read;
const writeCache = process.env.SHADOWDOG_LOCAL_CACHE_WRITE
? process.env.SHADOWDOG_LOCAL_CACHE_WRITE === 'true'
: options.write;
const cachePath = (_a = process.env.SHADOWDOG_LOCAL_CACHE_PATH) !== null && _a !== void 0 ? _a : options.path;
const currentCache = (0, utils_1.computeCache)(files, environment, config.command);
fs.mkdirpSync(cachePath);
if (readCache) {
const hasBeenRestored = await restoreCache(config, currentCache, {
...options,
path: cachePath,
});
if (hasBeenRestored) {
return abort();
}
}
await next();
if (writeCache) {
return Promise.all(config.artifacts.map(async (artifact) => {
const start = Date.now();
const sourceCacheFilePath = path.join(process.cwd(), artifact.output);
const exists = await fs.exists(sourceCacheFilePath);
if (!exists) {
(0, utils_1.logMessage)(`📦 Not able to store artifact '${chalk_1.default.blue(artifact.output)}' in cache because is not present`);
return;
}
const cacheFileName = (0, utils_1.computeFileCacheName)(currentCache, artifact.output);
const cacheFilePath = path.join(cachePath, `${cacheFileName}.tar.gz`);
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`📦 Storing artifact '${chalk_1.default.blue(artifact.output)}' in cache with value '${chalk_1.default.green(cacheFileName)}' ${chalk_1.default.cyan(`(${seconds}s)`)}`);
try {
await (0, exports.compressArtifact)(sourceCacheFilePath, cacheFilePath, (filePath) => filterFn(artifact.ignore, artifact.output, filePath));
}
catch (error) {
(0, utils_1.logMessage)(`🚫 An error ocurred while storing cache for artifact '${artifact.output}' with id '${chalk_1.default.green(cacheFileName)}'`);
(0, utils_1.logError)(error);
}
})).catch((error) => {
(0, utils_1.logError)(error);
});
}
};
exports.default = {
middleware,
};