@factorialco/shadowdog
Version:
<img src="https://raw.githubusercontent.com/factorialco/shadowdog/refs/heads/main/logo.png" alt="drawing" width="100"/>
212 lines (211 loc) • 9.97 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const chalk_1 = __importDefault(require("chalk"));
const child_process_1 = require("child_process");
const fs_extra_1 = __importDefault(require("fs-extra"));
const minio = __importStar(require("minio"));
const path_1 = __importDefault(require("path"));
const tar = __importStar(require("tar"));
const zlib = __importStar(require("zlib"));
const utils_1 = require("../utils");
const createClient = () => {
const { AWS_PROFILE } = process.env;
if (AWS_PROFILE) {
try {
const credentials = JSON.parse((0, child_process_1.execSync)(`aws configure export-credentials --profile "${AWS_PROFILE}"`).toString());
return new minio.Client({
endPoint: 's3.amazonaws.com',
useSSL: true,
accessKey: credentials.AccessKeyId,
secretKey: credentials.SecretAccessKey,
sessionToken: credentials.SessionToken,
region: (0, child_process_1.execSync)(`aws configure get region --profile "${AWS_PROFILE}"`).toString().trim(),
});
}
catch {
(0, utils_1.logMessage)(`🌐 Not able to create a client for remote cache because of failing authentication with AWS_PROFILE`);
return null;
}
}
const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION } = process.env;
if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY || !AWS_REGION) {
(0, utils_1.logMessage)(`🌐 Not able to create a client for remote cache because of missing AWS credentials`);
return null;
}
return new minio.Client({
endPoint: 's3.amazonaws.com',
useSSL: true,
accessKey: AWS_ACCESS_KEY_ID,
secretKey: AWS_SECRET_ACCESS_KEY,
region: AWS_REGION,
});
};
const saveRemoteCache = async (client, bucket, stream, objectName, artifact) => {
var _a;
try {
await client.putObject(bucket, objectName, stream, stream.readableLength, {
output: artifact.output,
extra: (_a = process.env.SHADOWDOG_REMOTE_CACHE_EXTRA) !== null && _a !== void 0 ? _a : '',
});
}
catch (error) {
(0, utils_1.logMessage)(`🌐 Not able to store artifact '${chalk_1.default.blue(artifact.output)}' -> '${chalk_1.default.green(objectName)}' in remote cache`);
(0, utils_1.logError)(error);
return false;
}
return true;
};
const restoreRemoteCache = async (client, bucket, objectName, artifact) => {
const stream = await client.getObject(bucket, objectName);
const outputPath = path_1.default.join(process.cwd(), artifact.output, '..');
fs_extra_1.default.mkdirpSync(outputPath);
return new Promise((resolve, reject) => {
const extractStream = stream.pipe(tar.extract({
cwd: outputPath,
filter: (filePath) => filterFn(artifact.ignore, artifact.output, filePath),
}));
extractStream.on('end', () => {
resolve(null);
});
extractStream.on('error', reject);
});
};
const filterFn = (ignore, outputPath, filePath) => {
if (!ignore) {
return true;
}
const keep = !ignore.includes(path_1.default.join(outputPath, '..', filePath));
if (!keep) {
(0, utils_1.logMessage)(`🗜️ Ignored file '${chalk_1.default.blue(filePath)}' during compression because of the ignore list`);
}
return keep;
};
const restoreCache = async (client, commandConfig, currentCache, pluginOptions) => {
// Check if we can reuse some artifacts from the cache
const promisesToGenerate = commandConfig.artifacts.map(async (artifact) => {
const start = Date.now();
const cacheFileName = (0, utils_1.computeFileCacheName)(currentCache, artifact.output);
const cacheFilePath = path_1.default.join(pluginOptions.path, `${cacheFileName}.tar.gz`);
try {
await restoreRemoteCache(client, pluginOptions.bucketName, cacheFilePath, artifact);
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`🌐 Reusing artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from remote cache because of cache ${chalk_1.default.bgGreen('HIT')} ${chalk_1.default.cyan(`(${seconds}s)`)}`);
return null;
}
catch (error) {
const seconds = ((Date.now() - start) / 1000).toFixed(2);
(0, utils_1.logMessage)(`🌐 Not able to reuse artifact '${chalk_1.default.blue(artifact.output)}' with id '${chalk_1.default.green(cacheFileName)}' from remote cache because of cache ${chalk_1.default.bgRed('MISS')} ${chalk_1.default.cyan(`(${seconds}s)`)}`);
(0, utils_1.logError)(error);
}
// If we can't reuse the artifact, we return it so it can be generated
return artifact;
});
const artifactToGenerate = await Promise.all(promisesToGenerate);
if (commandConfig.artifacts &&
commandConfig.artifacts.length > 0 &&
artifactToGenerate.filter(Boolean).length === 0 // Filtering out the artifacts that were reused from cache
) {
(0, utils_1.logMessage)(`⤵️ Skipping command '${chalk_1.default.yellow(commandConfig.command)}' generation because all artifacts were reused from remote cache`);
return true;
}
return false;
};
const middleware = async ({ config, files, invalidators, next, abort, options, }) => {
if (process.env.SHADOWDOG_DISABLE_REMOTE_CACHE) {
return next();
}
const client = createClient();
if (!client) {
return await next();
}
const readCache = process.env.SHADOWDOG_REMOTE_CACHE_READ
? process.env.SHADOWDOG_REMOTE_CACHE_READ === 'true'
: options.read;
const writeCache = process.env.SHADOWDOG_REMOTE_CACHE_WRITE
? process.env.SHADOWDOG_REMOTE_CACHE_WRITE === 'true'
: options.write;
const currentCache = (0, utils_1.computeCache)([...files, ...invalidators.files], invalidators.environment);
if (readCache) {
const hasBeenRestored = await restoreCache(client, config, currentCache, options);
if (hasBeenRestored) {
return abort();
}
}
await next();
if (writeCache) {
return Promise.all(config.artifacts.map(async (artifact) => {
if (!fs_extra_1.default.existsSync(path_1.default.join(process.cwd(), artifact.output))) {
(0, utils_1.logMessage)(`🌐 Not able to store artifact '${chalk_1.default.blue(artifact.output)}' in remote cache because is not present`);
return;
}
const cacheFileName = (0, utils_1.computeFileCacheName)(currentCache, artifact.output);
const cacheFilePath = path_1.default.join(options.path, `${cacheFileName}.tar.gz`);
const sourceCacheFilePath = path_1.default.join(process.cwd(), artifact.output);
try {
const tarStream = tar.create({
gzip: false,
cwd: path_1.default.dirname(sourceCacheFilePath),
filter: (filePath) => filterFn(artifact.ignore, artifact.output, filePath),
}, [path_1.default.basename(sourceCacheFilePath)]);
tarStream.on('error', (error) => {
(0, utils_1.logError)(error);
});
const gzipStream = zlib.createGzip();
gzipStream.on('error', (error) => {
(0, utils_1.logError)(error);
});
const stream = tarStream.pipe(gzipStream);
stream.on('error', (error) => {
(0, utils_1.logError)(error);
});
(0, utils_1.logMessage)(`🌐 Storing artifact '${chalk_1.default.blue(artifact.output)}' in remote cache with value '${chalk_1.default.green(cacheFileName)}'`);
await saveRemoteCache(client, options.bucketName, stream, cacheFilePath, artifact);
}
catch (error) {
(0, utils_1.logMessage)(`🚫 An error ocurred while storing cache for artifact '${artifact.output}' with id '${chalk_1.default.green(cacheFileName)}`);
(0, utils_1.logError)(error);
}
})).catch((error) => {
(0, utils_1.logError)(error);
});
}
};
exports.default = {
middleware,
};