ultimate-jekyll-manager
Version:
Ultimate Jekyll dependency manager
598 lines (503 loc) • 21.2 kB
JavaScript
// Libraries
const Manager = new (require('../../build.js'));
const logger = Manager.logger('imagemin');
const { src, dest, watch, series } = require('gulp');
const glob = require('glob').globSync;
const responsive = require('gulp-responsive-modern');
const sharp = require('sharp');
const path = require('path');
const { Transform } = require('stream');
const jetpack = require('fs-jetpack');
const GitHubCache = require('./utils/github-cache');
// Load package
const rootPathProject = Manager.getRootPath('project');
const ujmConfig = Manager.getUJMConfig();
// Settings
const CACHE_DIR = '.temp/cache/imagemin';
const CACHE_BRANCH = 'cache-uj-imagemin';
const MAX_SOURCE_DIMENSION = 4096;
const REWRITE_QUALITY = 80;
// Variables
let githubCache;
// Supported image extensions
const ALL_IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp'];
const RESPONSIVE_EXTENSIONS = new Set(['jpg', 'jpeg', 'png']);
// Globs include upper- and lower-case variants so micromatch (used by gulp-responsive-modern)
// matches files like IMG_3119.JPG. micromatch is strictly case-sensitive even on case-insensitive
// filesystems, so we expand each extension to both cases rather than relying on a nocase flag.
const expandCases = (exts) => exts.flatMap((ext) => [ext, ext.toUpperCase()]);
const ALL_IMAGE_GLOB = `*.{${expandCases(ALL_IMAGE_EXTENSIONS).join(',')}}`;
const RESPONSIVE_GLOB = `*.{${expandCases([...RESPONSIVE_EXTENSIONS]).join(',')}}`;
// Glob
const input = [
// Files to include
`src/assets/images/**/${ALL_IMAGE_GLOB}`,
// Files to exclude
// '!dist/**',
];
const output = 'dist/assets/images';
const delay = 250;
// Set index
let index = -1;
// Picture sizes configuration
const PICTURE_SIZES = [
{ width: 1024, suffix: '-1024px', formats: ['original', 'webp'] },
{ width: 640, suffix: '-640px', formats: ['original', 'webp'] },
{ width: 320, suffix: '-320px', formats: ['original', 'webp'] },
{ width: null, suffix: '', formats: ['original', 'webp'] }, // Original size
];
// Get responsive configs once
const responsiveConfigs = getResponsiveConfigs();
// Main task
async function imagemin(complete) {
// Increment index
index++;
// Log
logger.log('Starting...');
Manager.logMemory(logger, 'Start');
// Skip in dev mode - only run during builds
if (!Manager.isBuildMode()) {
logger.log('⏭️ Skipping imagemin in dev mode');
return complete();
}
// Skip if disabled in config
if (ujmConfig?.imagemin?.enabled === false) {
logger.log('⏭️ Skipping imagemin - disabled in ultimate-jekyll-manager.json');
return complete();
}
// Track timing
const startTime = Date.now();
// Initialize cache on first run
if (index === 0) {
// Log responsive configurations
logger.log('📏 Responsive configurations:', responsiveConfigs);
// Initialize cache
githubCache = await initializeCache();
}
// Short circuit if no GitHub credentials
if (!githubCache || !githubCache.hasCredentials()) {
logger.log('⏭️ Skipping imagemin - no GitHub cache credentials');
return complete();
}
// Track statistics
const stats = {
totalImages: 0,
fromCache: 0,
optimized: 0,
cachedFiles: [],
optimizedFiles: [],
sizeBefore: 0,
sizeAfter: 0,
savedBytes: 0
};
// Get all images
const files = glob(input);
if (files.length === 0) {
logger.log('Found 0 images to process');
return complete();
}
stats.totalImages = files.length;
logger.log(`Found ${files.length} images to process`);
// Load metadata
const metaPath = path.join(CACHE_DIR, 'meta.json');
let meta = githubCache ? githubCache.loadMetadata(metaPath) : {};
// Clean metadata of deleted files
if (githubCache) {
githubCache.cleanDeletedFromMetadata(meta, files, rootPathProject);
}
// Optionally rewrite oversized source images on disk (opt-in via UJ_IMAGEMIN_REWRITE_SOURCES=true).
// Caps longest dimension at MAX_SOURCE_DIMENSION so gulp-responsive-modern + sharp don't stall
// on huge inputs. Runs BEFORE determineFilesToProcess so cached-but-oversized images get
// rewritten too; the new on-disk content hashes differently than the stored meta hash, so
// determineFilesToProcess naturally picks the rewritten image up for re-optimization.
if (process.env.UJ_IMAGEMIN_REWRITE_SOURCES === 'true') {
await rewriteOversizedSources(files);
}
// Determine what needs processing
const { filesToProcess, validCachePaths } = await determineFilesToProcess(files, meta, githubCache, stats);
// Handle case where all files are from cache
if (filesToProcess.length === 0) {
logger.log('✅ All images from cache');
// Calculate timing
const endTime = Date.now();
const elapsedMs = endTime - startTime;
// Log statistics
logImageStatistics(stats, startTime, endTime);
await handleCacheOnlyUpdate(githubCache, metaPath, meta, validCachePaths, files.length, stats, { startTime, endTime, elapsedMs });
return complete();
}
// Calculate expected output count for progress tracking
const expectedOutputs = filesToProcess.reduce((count, file) => {
const ext = path.extname(file).slice(1).toLowerCase();
return count + (RESPONSIVE_EXTENSIONS.has(ext) ? responsiveConfigs.length : 1);
}, 0);
logger.log(`🔄 Processing ${filesToProcess.length} images (${expectedOutputs} output files)`);
stats.optimized = filesToProcess.length;
// Track sizes for optimization
for (const file of filesToProcess) {
const fileStats = jetpack.inspect(file);
if (fileStats) {
stats.sizeBefore += fileStats.size;
}
stats.optimizedFiles.push(path.relative(rootPathProject, file));
}
// Progress counter
let processedOutputs = 0;
// Process images.
//
// CRITICAL: this function is `async`, which means returning a stream from it yields a
// Promise<Stream> to gulp — gulp resolves the task immediately on the Promise rather than
// waiting for the stream's 'finish' event. Downstream tasks (jekyll, audit, etc.) then start
// before imagemin has actually written its outputs to disk, and the build "succeeds" while
// silently shipping a partial _site/. We must explicitly await stream completion + cache push
// before returning, so gulp sees the real completion.
//
// This await only ever runs in build mode — dev mode short-circuits via `!Manager.isBuildMode()`
// above (so `npm start` never blocks on this), letting BrowserSync reload as images land later.
await new Promise((resolve, reject) => {
src(filesToProcess, { base: 'src/assets/images' })
.pipe(lowercaseExtTransform())
.pipe(responsive({
[`**/${RESPONSIVE_GLOB}`]: responsiveConfigs
}, {
quality: 80,
progressive: true,
withMetadata: false,
withoutEnlargement: false,
skipOnEnlargement: false,
errorOnUnusedImage: false,
passThroughUnused: true,
}))
.on('error', reject)
.pipe(dest(output))
.on('error', reject)
.on('data', (file) => {
// Progress tracking
processedOutputs++;
const relativePath = path.relative(path.join(rootPathProject, output), file.path);
logger.log(`🖼️ ${processedOutputs}/${expectedOutputs}: ${relativePath}`);
// Save to cache
const cachePath = path.join(CACHE_DIR, 'images', relativePath);
jetpack.copy(file.path, cachePath, { overwrite: true });
// Track size after optimization
const fileStats = jetpack.inspect(file.path);
if (fileStats) {
stats.sizeAfter += fileStats.size;
}
})
.on('finish', resolve);
});
// Calculate final statistics
stats.savedBytes = stats.sizeBefore - stats.sizeAfter;
// Calculate timing
const endTime = Date.now();
const elapsedMs = endTime - startTime;
// Log statistics
logImageStatistics(stats, startTime, endTime);
// Save metadata and push cache
if (githubCache && githubCache.hasCredentials()) {
githubCache.saveMetadata(metaPath, meta);
logger.log(`📊 Updating cache with ${stats.optimized} new optimizations and README stats...`);
// Collect all cache files to push (metadata will be auto-included)
const allCacheFiles = glob(path.join(CACHE_DIR, '**/*'), { nodir: true });
// Push to GitHub with atomic replacement
await githubCache.pushBranch(allCacheFiles, {
validFiles: validCachePaths,
stats: {
timestamp: new Date().toISOString(),
sourceCount: files.length,
cachedCount: allCacheFiles.length - 1,
processedNow: stats.optimized,
fromCache: stats.fromCache,
newlyProcessed: stats.optimized,
timing: {
startTime,
endTime,
elapsedMs
},
imageStats: {
totalImages: stats.totalImages,
optimized: stats.optimized,
skipped: stats.fromCache,
totalSizeBefore: stats.sizeBefore,
totalSizeAfter: stats.sizeAfter,
totalSaved: stats.savedBytes
},
details: `Optimized ${stats.optimized} images, ${stats.fromCache} from cache\n\n### Files from cache:\n${stats.cachedFiles.length > 0 ? stats.cachedFiles.map(f => `- ${f}`).join('\n') : 'None'}\n\n### Newly optimized files:\n${stats.optimizedFiles.length > 0 ? stats.optimizedFiles.map(f => `- ${f}`).join('\n') : 'None'}`
}
});
}
logger.log('✅ Finished!');
return complete();
}
// Watcher task
function imageminWatcher(complete) {
// Quit if in build mode
if (Manager.isBuildMode()) {
logger.log('[watcher] Skipping watcher in build mode');
return complete();
}
// Log
logger.log('[watcher] Watching for changes...');
// Watch for changes
watch(input, { delay: delay, dot: true }, imagemin)
.on('change', (path) => {
logger.log(`[watcher] File changed (${path})`);
});
// Complete
return complete();
}
// Default Task
module.exports = series(
imagemin,
imageminWatcher
);
// ============================================================================
// Helper Functions
// ============================================================================
// Rewrite oversized source images in place, capping longest dimension at MAX_SOURCE_DIMENSION.
// Only affects files whose decoded longest side exceeds the cap. Cache invalidation is implicit:
// the new content hashes differently than the previously-cached entry, so determineFilesToProcess
// will pick affected files up for re-optimization on its own.
async function rewriteOversizedSources(files) {
const responsiveFiles = files.filter(f => RESPONSIVE_EXTENSIONS.has(path.extname(f).slice(1).toLowerCase()));
if (responsiveFiles.length === 0) {
return;
}
logger.log(`🔍 Checking ${responsiveFiles.length} source images for oversize (>${MAX_SOURCE_DIMENSION}px longest side)...`);
let rewritten = 0;
for (const file of responsiveFiles) {
const metadata = await sharp(file).metadata();
const longest = Math.max(metadata.width || 0, metadata.height || 0);
if (longest <= MAX_SOURCE_DIMENSION) {
continue;
}
const ext = path.extname(file).slice(1).toLowerCase();
const sizeBefore = jetpack.inspect(file)?.size || 0;
// Resize, encode to a buffer (sharp can't write back to its own input file directly), then overwrite.
const pipeline = sharp(file).resize({
width: MAX_SOURCE_DIMENSION,
height: MAX_SOURCE_DIMENSION,
fit: 'inside',
withoutEnlargement: true,
});
const buffer = ext === 'png'
? await pipeline.png({ quality: REWRITE_QUALITY }).toBuffer()
: await pipeline.jpeg({ quality: REWRITE_QUALITY, progressive: true, mozjpeg: true }).toBuffer();
jetpack.write(file, buffer);
const sizeAfter = buffer.length;
// No cache bookkeeping needed: the rewritten file has new content, so the next
// determineFilesToProcess() call computes a different hash than the stored meta hash and
// naturally treats this image as needing reprocessing. Stored meta will be overwritten
// with the new hash when determineFilesToProcess() runs.
rewritten++;
logger.log(`✂️ Rewrote ${path.relative(rootPathProject, file)}: ${metadata.width}x${metadata.height} → max ${MAX_SOURCE_DIMENSION}px, ${formatBytes(sizeBefore)} → ${formatBytes(sizeAfter)}`);
}
if (rewritten === 0) {
logger.log(`✅ No oversized sources found (all within ${MAX_SOURCE_DIMENSION}px)`);
} else {
logger.log(`✂️ Rewrote ${rewritten} oversized source image(s)`);
}
}
// Lowercase the extension on each Vinyl file's path before piping into gulp-responsive-modern.
// gulp-responsive-modern's lib/format.js uses a case-sensitive switch on path.extname() and returns
// the string 'unsupported' for anything else, which then crashes sharp.toFormat(). Files saved
// straight off a camera (IMG_3119.JPG) hit this. Rewriting the Vinyl path in-stream keeps the
// on-disk file untouched while letting the plugin recognize the format.
function lowercaseExtTransform() {
return new Transform({
objectMode: true,
transform(file, _enc, cb) {
const ext = path.extname(file.path);
if (ext && ext !== ext.toLowerCase()) {
file.path = file.path.slice(0, -ext.length) + ext.toLowerCase();
}
cb(null, file);
},
});
}
// Build responsive configurations from PICTURE_SIZES
function getResponsiveConfigs() {
const configs = [];
PICTURE_SIZES.forEach(size => {
size.formats.forEach(format => {
const config = {};
if (size.width) {
config.width = size.width;
}
config.rename = { suffix: size.suffix };
if (format === 'webp') {
config.format = 'webp';
}
configs.push(config);
});
});
return configs;
}
// Initialize or get cache
async function initializeCache() {
const useCache = process.env.UJ_IMAGEMIN_CACHE !== 'false';
if (!useCache) {
return null;
}
const cache = new GitHubCache({
branchName: CACHE_BRANCH,
cacheDir: CACHE_DIR,
logger: logger,
cacheType: 'Image',
description: 'processed image cache for faster builds'
});
// Fetch cache from GitHub if credentials available
if (cache.hasCredentials()) {
await cache.fetchBranch();
logger.log(`📦 Cache initialized with ${glob(path.join(CACHE_DIR, '**/*'), { nodir: true }).length} files`);
} else {
logger.log('📦 Cache enabled (local only - no GitHub credentials)');
}
return cache;
}
// Determine which files need processing
async function determineFilesToProcess(files, meta, githubCache, stats) {
const filesToProcess = [];
const validCachePaths = new Set();
for (const file of files) {
const relativePath = path.relative(rootPathProject, file);
const hash = githubCache ? githubCache.calculateHash(file) : null;
// Track expected outputs for this file
const baseName = path.basename(file, path.extname(file));
const dirName = path.dirname(relativePath).replace(/^src\/assets\/images\/?/, '');
const originalExt = path.extname(file).slice(1); // Remove the dot
// Only generate responsive outputs for supported formats
// Other formats (svg, gif, webp) pass through as-is
const outputs = [];
if (RESPONSIVE_EXTENSIONS.has(originalExt.toLowerCase())) {
PICTURE_SIZES.forEach(size => {
size.formats.forEach(format => {
if (format === 'original') {
outputs.push(`${baseName}${size.suffix}.${originalExt}`);
} else if (format === 'webp') {
outputs.push(`${baseName}${size.suffix}.webp`);
}
});
});
} else {
outputs.push(`${baseName}.${originalExt}`);
}
// Track as valid cache files
outputs.forEach(name => validCachePaths.add(path.join('images', dirName, name)));
// Check if cached and all outputs exist
const useCached = meta[relativePath]?.hash === hash;
if (useCached) {
const allExist = outputs.every(name =>
jetpack.exists(path.join(CACHE_DIR, 'images', dirName, name))
);
if (allExist) {
// Copy from cache to output
outputs.forEach(name => {
const src = path.join(CACHE_DIR, 'images', dirName, name);
const dst = path.join(output, dirName, name);
jetpack.copy(src, dst, { overwrite: true });
});
logger.log(`📦 ${stats.fromCache + 1}/${files.length} from cache: ${relativePath}`);
stats.fromCache++;
stats.cachedFiles.push(relativePath);
// Track size of cached files
outputs.forEach(name => {
const cachePath = path.join(CACHE_DIR, 'images', dirName, name);
const fileStats = jetpack.inspect(cachePath);
if (fileStats) {
stats.sizeAfter += fileStats.size;
}
});
// Track original size
const originalStats = jetpack.inspect(file);
if (originalStats) {
stats.sizeBefore += originalStats.size;
}
continue;
}
}
// Needs processing
filesToProcess.push(file);
meta[relativePath] = { hash, timestamp: new Date().toISOString() };
}
return { filesToProcess, validCachePaths, cacheStats: stats };
}
// Handle cache-only update (when no files need processing)
async function handleCacheOnlyUpdate(githubCache, metaPath, meta, validCachePaths, fileCount, stats, timing) {
if (!githubCache || !githubCache.hasCredentials()) {
return;
}
// Save metadata
githubCache.saveMetadata(metaPath, meta);
// ALWAYS update README with latest stats, even if no orphans
logger.log(`📊 Updating cache README with latest statistics...`);
// Collect all valid cache files
const allCacheFiles = glob(path.join(CACHE_DIR, '**/*'), { nodir: true });
// Push to GitHub (will update README even if no file changes)
await githubCache.pushBranch(allCacheFiles, {
validFiles: validCachePaths,
stats: {
timestamp: new Date().toISOString(),
sourceCount: fileCount,
cachedCount: allCacheFiles.length - 1, // Subtract meta.json
processedNow: stats.totalImages,
fromCache: stats.fromCache,
newlyProcessed: stats.optimized,
timing: timing,
imageStats: {
totalImages: stats.totalImages,
optimized: stats.optimized,
skipped: stats.fromCache,
totalSizeBefore: stats.sizeBefore,
totalSizeAfter: stats.sizeAfter,
totalSaved: stats.sizeBefore - stats.sizeAfter
},
details: `All ${fileCount} images served from cache`
}
});
}
// Log image statistics
function logImageStatistics(stats, startTime, endTime) {
const elapsedMs = endTime - startTime;
const elapsedSeconds = Math.floor(elapsedMs / 1000);
const elapsedMinutes = Math.floor(elapsedSeconds / 60);
const elapsedFormatted = elapsedMinutes > 0
? `${elapsedMinutes}m ${elapsedSeconds % 60}s`
: `${elapsedSeconds}s`;
logger.log('\n📊 Image Optimization Statistics:');
logger.log('═══════════════════════════════════════');
// Timing
logger.log('⏱️ Timing:');
logger.log(` Start time: ${new Date(startTime).toLocaleTimeString()}`);
logger.log(` End time: ${new Date(endTime).toLocaleTimeString()}`);
logger.log(` Total elapsed: ${elapsedFormatted}`);
// File processing stats
logger.log('\n📁 File Processing:');
logger.log(` Total images: ${stats.totalImages}`);
logger.log(` From cache: ${stats.fromCache} (${((stats.fromCache / stats.totalImages) * 100).toFixed(1)}%)`);
logger.log(` Newly optimized: ${stats.optimized} (${((stats.optimized / stats.totalImages) * 100).toFixed(1)}%)`);
// Size reduction stats
if (stats.sizeBefore > 0 && stats.sizeAfter > 0) {
const savedPercent = ((stats.savedBytes / stats.sizeBefore) * 100).toFixed(1);
const label = stats.savedBytes < 0 ? 'Total added' : 'Total saved';
logger.log('\n💾 Size Reduction:');
logger.log(` Original size: ${formatBytes(stats.sizeBefore)}`);
logger.log(` Optimized size: ${formatBytes(stats.sizeAfter)}`);
logger.log(` ${label}: ${formatBytes(Math.abs(stats.savedBytes))} (${savedPercent}%)`);
}
logger.log('═══════════════════════════════════════\n');
}
// Helper to format bytes. Handles negative inputs — when responsive variants (8 per source)
// sum to more than the cached original, savedBytes goes negative; without the absolute-value
// guard, Math.log(negative) is NaN and the suffix index becomes NaN -> "NaN undefined".
function formatBytes(bytes, decimals = 2) {
if (bytes === 0) return '0 Bytes';
const sign = bytes < 0 ? '-' : '';
const abs = Math.abs(bytes);
const k = 1024;
const dm = decimals < 0 ? 0 : decimals;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.min(Math.floor(Math.log(abs) / Math.log(k)), sizes.length - 1);
return sign + parseFloat((abs / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
}