hard-source-webpack-plugin-fixed-hashbug/index.js

Hard cache the source of modules in webpack. Patched version fixing the hash bug

const crypto = require('crypto');
const fs = require('graceful-fs');
const path = require('path');

const lodash = require('lodash');
const _mkdirp = require('mkdirp');
const _rimraf = require('rimraf');
const nodeObjectHash = require('node-object-hash');
const findCacheDir = require('find-cache-dir');

const envHash = require('./lib/envHash');
const defaultConfigHash = require('./lib/defaultConfigHash');
const promisify = require('./lib/util/promisify');
const relateContext = require('./lib/util/relate-context');
const pluginCompat = require('./lib/util/plugin-compat');
const logMessages = require('./lib/util/log-messages');

const LoggerFactory = require('./lib/loggerFactory');

const cachePrefix = require('./lib/util').cachePrefix;

const CacheSerializerFactory = require('./lib/CacheSerializerFactory');
const ExcludeModulePlugin = require('./lib/ExcludeModulePlugin');
const HardSourceLevelDbSerializerPlugin = require('./lib/SerializerLeveldbPlugin');
const SerializerAppend2Plugin = require('./lib/SerializerAppend2Plugin');
const SerializerAppendPlugin = require('./lib/SerializerAppendPlugin');
const SerializerCacachePlugin = require('./lib/SerializerCacachePlugin');
const SerializerJsonPlugin = require('./lib/SerializerJsonPlugin');

const hardSourceVersion = require('./package.json').version;

function requestHash(request) {
  return crypto
    .createHash('sha1')
    .update(request)
    .digest()
    .hexSlice();
}

const mkdirp = promisify(_mkdirp, { context: _mkdirp });
mkdirp.sync = _mkdirp.sync.bind(_mkdirp);
const rimraf = promisify(_rimraf);
rimraf.sync = _rimraf.sync.bind(_rimraf);
const fsReadFile = promisify(fs.readFile, { context: fs });
const fsWriteFile = promisify(fs.writeFile, { context: fs });

const bulkFsTask = (array, each) =>
  new Promise((resolve, reject) => {
    let ops = 0;
    const out = [];
    array.forEach((item, i) => {
      out[i] = each(item, (back, callback) => {
        ops++;
        return (err, value) => {
          try {
            out[i] = back(err, value, out[i]);
          } catch (e) {
            return reject(e);
          }

          ops--;
          if (ops === 0) {
            resolve(out);
          }
        };
      });
    });
    if (ops === 0) {
      resolve(out);
    }
  });

const compilerContext = relateContext.compilerContext;
const relateNormalPath = relateContext.relateNormalPath;
const contextNormalPath = relateContext.contextNormalPath;
const contextNormalPathSet = relateContext.contextNormalPathSet;

function relateNormalRequest(compiler, key) {
  return key
    .split('!')
    .map(subkey => relateNormalPath(compiler, subkey))
    .join('!');
}

function relateNormalModuleId(compiler, id) {
  return id.substring(0, 24) + relateNormalRequest(compiler, id.substring(24));
}

function contextNormalRequest(compiler, key) {
  return key
    .split('!')
    .map(subkey => contextNormalPath(compiler, subkey))
    .join('!');
}

function contextNormalModuleId(compiler, id) {
  return id.substring(0, 24) + contextNormalRequest(compiler, id.substring(24));
}

function contextNormalLoaders(compiler, loaders) {
  return loaders.map(loader =>
    Object.assign({}, loader, {
      loader: contextNormalPath(compiler, loader.loader),
    }),
  );
}

function contextNormalPathArray(compiler, paths) {
  return paths.map(subpath => contextNormalPath(compiler, subpath));
}

class HardSourceWebpackPlugin {
  constructor(options) {
    this.options = options || {};
  }

  getPath(dirName, suffix) {
    const confighashIndex = dirName.search(/\[confighash\]/);
    if (confighashIndex !== -1) {
      dirName = dirName.replace(/\[confighash\]/, this.configHash);
    }
    let cachePath = path.resolve(
      process.cwd(),
      this.compilerOutputOptions.path,
      dirName,
    );
    if (suffix) {
      cachePath = path.join(cachePath, suffix);
    }
    return cachePath;
  }

  getCachePath(suffix) {
    return this.getPath(this.options.cacheDirectory, suffix);
  }

  apply(compiler) {
    const options = this.options;
    let active = true;

    const logger = new LoggerFactory(compiler).create();

    const loggerCore = logger.from('core');
    logger.lock();

    const compilerHooks = pluginCompat.hooks(compiler);

    if (!compiler.options.cache) {
      compiler.options.cache = true;
    }

    if (!options.cacheDirectory) {
      options.cacheDirectory = path.resolve(
        findCacheDir({
          name: 'hard-source',
          cwd: compiler.options.context || process.cwd(),
        }),
        '[confighash]',
      );
    }

    this.compilerOutputOptions = compiler.options.output;
    if (!options.configHash) {
      options.configHash = defaultConfigHash;
    }
    if (options.configHash) {
      if (typeof options.configHash === 'string') {
        this.configHash = options.configHash;
      } else if (typeof options.configHash === 'function') {
        this.configHash = options.configHash(compiler.options);
      }
      compiler.__hardSource_configHash = this.configHash;
      compiler.__hardSource_shortConfigHash = this.configHash.substring(0, 8);
    }
    const configHashInDirectory =
      options.cacheDirectory.search(/\[confighash\]/) !== -1;
    if (configHashInDirectory && !this.configHash) {
      logMessages.configHashSetButNotUsed(compiler, {
        cacheDirectory: options.cacheDirectory,
      });
      active = false;

      function unlockLogger() {
        logger.unlock();
      }
      compilerHooks.watchRun.tap('HardSource - index', unlockLogger);
      compilerHooks.run.tap('HardSource - index', unlockLogger);
      return;
    }

    let environmentHasher = null;
    if (typeof options.environmentHash !== 'undefined') {
      if (options.environmentHash === false) {
        environmentHasher = () => Promise.resolve('');
      } else if (typeof options.environmentHash === 'string') {
        environmentHasher = () => Promise.resolve(options.environmentHash);
      } else if (typeof options.environmentHash === 'object') {
        environmentHasher = () => envHash(options.environmentHash);
        environmentHasher.inputs = () =>
          envHash.inputs(options.environmentHash);
      } else if (typeof options.environmentHash === 'function') {
        environmentHasher = () => Promise.resolve(options.environmentHash());
        if (options.environmentHash.inputs) {
          environmentHasher.inputs = () =>
            Promise.resolve(options.environmentHasher.inputs());
        }
      }
    }
    if (!environmentHasher) {
      environmentHasher = envHash;
    }

    const cacheDirPath = this.getCachePath();
    const cacheAssetDirPath = path.join(cacheDirPath, 'assets');
    const resolveCachePath = path.join(cacheDirPath, 'resolve.json');

    let currentStamp = '';

    const cacheSerializerFactory = new CacheSerializerFactory(compiler);
    let createSerializers = true;
    let cacheRead = false;

    const _this = this;

    pluginCompat.register(compiler, '_hardSourceCreateSerializer', 'sync', [
      'cacheSerializerFactory',
      'cacheDirPath',
    ]);
    pluginCompat.register(compiler, '_hardSourceResetCache', 'sync', []);
    pluginCompat.register(compiler, '_hardSourceReadCache', 'asyncParallel', [
      'relativeHelpers',
    ]);
    pluginCompat.register(
      compiler,
      '_hardSourceVerifyCache',
      'asyncParallel',
      [],
    );
    pluginCompat.register(compiler, '_hardSourceWriteCache', 'asyncParallel', [
      'compilation',
      'relativeHelpers',
    ]);

    if (configHashInDirectory) {
      const PruneCachesSystem = require('./lib/SystemPruneCaches');

      new PruneCachesSystem(
        path.dirname(cacheDirPath),
        options.cachePrune,
      ).apply(compiler);
    }

    function runReadOrReset(_compiler) {
      logger.unlock();

      if (!active) {
        return Promise.resolve();
      }

      try {
        fs.statSync(cacheAssetDirPath);
      } catch (_) {
        mkdirp.sync(cacheAssetDirPath);
        logMessages.configHashFirstBuild(compiler, {
          cacheDirPath,
          configHash: compiler.__hardSource_configHash,
        });
      }
      const start = Date.now();

      if (createSerializers) {
        createSerializers = false;
        try {
          compilerHooks._hardSourceCreateSerializer.call(
            cacheSerializerFactory,
            cacheDirPath,
          );
        } catch (err) {
          return Promise.reject(err);
        }
      }

      return Promise.all([
        fsReadFile(path.join(cacheDirPath, 'stamp'), 'utf8').catch(() => ''),

        environmentHasher(),

        fsReadFile(path.join(cacheDirPath, 'version'), 'utf8').catch(() => ''),

        environmentHasher.inputs ? environmentHasher.inputs() : null,
      ]).then(([stamp, hash, versionStamp, hashInputs]) => {
        if (!configHashInDirectory && options.configHash) {
          hash += `_${_this.configHash}`;
        }

        if (hashInputs && !cacheRead) {
          logMessages.environmentInputs(compiler, { inputs: hashInputs });
        }

        currentStamp = hash;
        if (!hash || hash !== stamp || hardSourceVersion !== versionStamp) {
          if (hash && stamp) {
            if (configHashInDirectory) {
              logMessages.environmentHashChanged(compiler);
            } else {
              logMessages.configHashChanged(compiler);
            }
          } else if (versionStamp && hardSourceVersion !== versionStamp) {
            logMessages.hardSourceVersionChanged(compiler);
          }

          // Reset the cache, we can't use it do to an environment change.
          pluginCompat.call(compiler, '_hardSourceResetCache', []);

          return rimraf(cacheDirPath);
        }

        if (cacheRead) {
          return Promise.resolve();
        }
        cacheRead = true;

        logMessages.configHashBuildWith(compiler, {
          cacheDirPath,
          configHash: compiler.__hardSource_configHash,
        });

        function contextKeys(compiler, fn) {
          return source => {
            const dest = {};
            Object.keys(source).forEach(key => {
              dest[fn(compiler, key)] = source[key];
            });
            return dest;
          };
        }

        function contextValues(compiler, fn) {
          return source => {
            const dest = {};
            Object.keys(source).forEach(key => {
              const value = fn(compiler, source[key], key);
              if (value) {
                dest[key] = value;
              } else {
                delete dest[key];
              }
            });
            return dest;
          };
        }

        function copyWithDeser(dest, source) {
          Object.keys(source).forEach(key => {
            const item = source[key];
            dest[key] = typeof item === 'string' ? JSON.parse(item) : item;
          });
        }

        return Promise.all([
          compilerHooks._hardSourceReadCache.promise({
            contextKeys,
            contextValues,
            contextNormalPath,
            contextNormalRequest,
            contextNormalModuleId,
            copyWithDeser,
          }),
        ])
          .catch(error => {
            logMessages.serialBadCache(compiler, error);

            return rimraf(cacheDirPath);
          })
          .then(() => {
            // console.log('cache in', Date.now() - start);
          });
      });
    }

    compilerHooks.watchRun.tapPromise(
      'HardSource - index - readOrReset',
      runReadOrReset,
    );
    compilerHooks.run.tapPromise(
      'HardSource - index - readOrReset',
      runReadOrReset,
    );

    const detectModule = path => {
      try {
        require(path);
        return true;
      } catch (_) {
        return false;
      }
    };

    const webpackFeatures = {
      concatenatedModule: detectModule(
        'webpack/lib/optimize/ConcatenatedModule',
      ),
      generator: detectModule('webpack/lib/JavascriptGenerator'),
    };

    let schemasVersion = 2;
    if (webpackFeatures.concatenatedModule) {
      schemasVersion = 3;
    }
    if (webpackFeatures.generator) {
      schemasVersion = 4;
    }

    const ArchetypeSystem = require('./lib/SystemArchetype');
    const ParitySystem = require('./lib/SystemParity');

    const AssetCache = require('./lib/CacheAsset');
    const ModuleCache = require('./lib/CacheModule');

    const EnhancedResolveCache = require('./lib/CacheEnhancedResolve');
    const Md5Cache = require('./lib/CacheMd5');
    const ModuleResolverCache = require('./lib/CacheModuleResolver');

    const TransformCompilationPlugin = require('./lib/TransformCompilationPlugin');
    const TransformAssetPlugin = require('./lib/TransformAssetPlugin');
    let TransformConcatenationModulePlugin;
    if (webpackFeatures.concatenatedModule) {
      TransformConcatenationModulePlugin = require('./lib/TransformConcatenationModulePlugin');
    }
    const TransformNormalModulePlugin = require('./lib/TransformNormalModulePlugin');
    const TransformNormalModuleFactoryPlugin = require('./lib/TransformNormalModuleFactoryPlugin');
    const TransformModuleAssetsPlugin = require('./lib/TransformModuleAssetsPlugin');
    const TransformModuleErrorsPlugin = require('./lib/TransformModuleErrorsPlugin');
    const SupportExtractTextPlugin = require('./lib/SupportExtractTextPlugin');
    let SupportMiniCssExtractPlugin;
    if (webpackFeatures.generator) {
      SupportMiniCssExtractPlugin = require('./lib/SupportMiniCssExtractPlugin');
    }
    const TransformDependencyBlockPlugin = require('./lib/TransformDependencyBlockPlugin');
    const TransformBasicDependencyPlugin = require('./lib/TransformBasicDependencyPlugin');
    let HardHarmonyDependencyPlugin;
    const TransformSourcePlugin = require('./lib/TransformSourcePlugin');
    const TransformParserPlugin = require('./lib/TransformParserPlugin');
    let TransformGeneratorPlugin;
    if (webpackFeatures.generator) {
      TransformGeneratorPlugin = require('./lib/TransformGeneratorPlugin');
    }

    const ChalkLoggerPlugin = require('./lib/ChalkLoggerPlugin');

    new ArchetypeSystem().apply(compiler);
    new ParitySystem().apply(compiler);

    new AssetCache().apply(compiler);
    new ModuleCache().apply(compiler);

    new EnhancedResolveCache().apply(compiler);
    new Md5Cache().apply(compiler);
    new ModuleResolverCache().apply(compiler);

    new TransformCompilationPlugin().apply(compiler);

    new TransformAssetPlugin().apply(compiler);

    new TransformNormalModulePlugin({
      schema: schemasVersion,
    }).apply(compiler);
    new TransformNormalModuleFactoryPlugin().apply(compiler);

    if (TransformConcatenationModulePlugin) {
      new TransformConcatenationModulePlugin().apply(compiler);
    }

    new TransformModuleAssetsPlugin().apply(compiler);
    new TransformModuleErrorsPlugin().apply(compiler);
    new SupportExtractTextPlugin().apply(compiler);

    if (SupportMiniCssExtractPlugin) {
      new SupportMiniCssExtractPlugin().apply(compiler);
    }

    new TransformDependencyBlockPlugin({
      schema: schemasVersion,
    }).apply(compiler);

    new TransformBasicDependencyPlugin({
      schema: schemasVersion,
    }).apply(compiler);

    new TransformSourcePlugin({
      schema: schemasVersion,
    }).apply(compiler);

    new TransformParserPlugin({
      schema: schemasVersion,
    }).apply(compiler);

    if (TransformGeneratorPlugin) {
      new TransformGeneratorPlugin({
        schema: schemasVersion,
      }).apply(compiler);
    }

    new ChalkLoggerPlugin(this.options.info).apply(compiler);

    function runVerify(_compiler) {
      if (!active) {
        return Promise.resolve();
      }

      const stats = {};
      return pluginCompat.promise(compiler, '_hardSourceVerifyCache', []);
    }

    compilerHooks.watchRun.tapPromise('HardSource - index - verify', runVerify);
    compilerHooks.run.tapPromise('HardSource - index - verify', runVerify);

    let freeze;

    compilerHooks._hardSourceMethods.tap('HardSource - index', methods => {
      freeze = methods.freeze;
    });

    compilerHooks.afterCompile.tapPromise('HardSource - index', compilation => {
      if (!active) {
        return Promise.resolve();
      }

      const startCacheTime = Date.now();

      const identifierPrefix = cachePrefix(compilation);
      if (identifierPrefix !== null) {
        freeze('Compilation', null, compilation, {
          compilation,
        });
      }

      return Promise.all([
        mkdirp(cacheDirPath).then(() =>
          Promise.all([
            fsWriteFile(path.join(cacheDirPath, 'stamp'), currentStamp, 'utf8'),
            fsWriteFile(
              path.join(cacheDirPath, 'version'),
              hardSourceVersion,
              'utf8',
            ),
          ]),
        ),
        pluginCompat.promise(compiler, '_hardSourceWriteCache', [
          compilation,
          {
            relateNormalPath,
            relateNormalRequest,
            relateNormalModuleId,

            contextNormalPath,
            contextNormalRequest,
            contextNormalModuleId,
          },
        ]),
      ]).then(() => {
        // console.log('cache out', Date.now() - startCacheTime);
      });
    });
  }
}

module.exports = HardSourceWebpackPlugin;

HardSourceWebpackPlugin.ExcludeModulePlugin = ExcludeModulePlugin;
HardSourceWebpackPlugin.HardSourceLevelDbSerializerPlugin = HardSourceLevelDbSerializerPlugin;
HardSourceWebpackPlugin.LevelDbSerializerPlugin = HardSourceLevelDbSerializerPlugin;
HardSourceWebpackPlugin.SerializerAppend2Plugin = SerializerAppend2Plugin;
HardSourceWebpackPlugin.SerializerAppendPlugin = SerializerAppendPlugin;
HardSourceWebpackPlugin.SerializerCacachePlugin = SerializerCacachePlugin;
HardSourceWebpackPlugin.SerializerJsonPlugin = SerializerJsonPlugin;

Object.defineProperty(HardSourceWebpackPlugin, 'ParallelModulePlugin', {
  get() {
    return require('./lib/ParallelModulePlugin');
  },
});