webext-store-incompat-fixer/index.js

Package which clones a packed webextension and fixes incompatibilities with certain extension stores

/* eslint-disable no-restricted-syntax */

import fs from 'node:fs';
import JSZip from 'jszip';

const storeTheming = {
  chrome: '\u001B[33m',
  edge: '\u001B[32m',
  firefox: '\u001B[31;1m',
  opera: '\u001B[31m',
  safari: '\u001B[34m',
  samsung: '\u001B[34m',
  whale: '\u001B[36m',
};

const whaleLocaleFallbacks = {
  'bs-Cyrl': 'sr',
  'ca-valencia': 'ca',
  'sh-Cyrl': 'sr',
  'sr-Cyrl': 'sr',
  'sr-Latn': 'sh',
  'zh-Hans': 'zh_CN',
  'zh-Hant': 'zh_TW',
  'zh-Hant-HK': 'zh_HK',
};

function stringifyInOriginalFormat (originalString, newJson) {
  let space = null;
  if (originalString.indexOf('\t') !== -1) {
    space = '\t';
  } else if (originalString.indexOf('  ') !== -1) {
    space = 2;
  }
  return JSON.stringify(newJson, null, space);
}

function handleEdgeLocaleExclusions ({
  zip, manifestString, manifestJson, params,
}) {
  const defaultLocale = manifestJson.default_locale;
  const messagesMatch = manifestString.match(/__MSG_(.+?)__/g);

  let changed = false;

  if (!defaultLocale || !messagesMatch) {
    return Promise.resolve(changed);
  }

  const messagesToRemove = messagesMatch.map((item) => item.match(/__MSG_(.+)__/)[1]);

  if (messagesToRemove.length === 0) {
    return Promise.resolve(changed);
  }

  const forcedInclusions = Array.isArray(params.edgeLocaleInclusions)
    ? params.edgeLocaleInclusions
    : [];
  forcedInclusions.push(defaultLocale);

  const updatePromises = [];
  const localeRegex = /^_locales\/(.+)\/messages\.json$/;

  zip.file(localeRegex).forEach((file) => {
    const fileName = file.name;
    const localeId = fileName.match(localeRegex)[1];
    if (forcedInclusions.includes(localeId)) return;

    const readPromise = file.async('string').then((result) => {
      const localeJson = JSON.parse(result);
      let messageFound = false;

      messagesToRemove.forEach((messageId) => {
        if (localeJson[messageId]) {
          messageFound = true;
          delete localeJson[messageId];
        }
      });

      if (!messageFound) return null;

      changed = true;
      const localeString = stringifyInOriginalFormat(result, localeJson);
      return zip.file(fileName, localeString);
    });

    updatePromises.push(readPromise);
  });

  return Promise.all(updatePromises).then(() => changed);
}

function removeIncompatibleWeblocFiles ({
  zip, packageChanges,
}) {
  const incompatibleFiles = /^(.*).webloc$/;
  zip.file(incompatibleFiles).forEach((file) => {
    packageChanges.push(`Removed ${file.name} as it is not allowed to be included for the Opera store.`);
    zip.remove(file.name);
  });
}

async function handleWhaleLocaleIssues ({
  zip, manifestString, manifestJson, packageChanges, whaleMessageLimit = Infinity,
}) {
  // handle incompatible locale files
  const incompatLocaleMatch = /^_locales\/([a-zA-Z0-9]+[_-]([a-zA-Z0-9_-]+))\/messages\.json$/;
  const countryCodeMatch = /^[A-Z]{2}|[0-9]{3}$/;
  const promises = zip.file(incompatLocaleMatch).map((file) => {
    const localeIdMatch = file.name.match(incompatLocaleMatch);
    const fullLocaleId = localeIdMatch[1];
    const canonicalLocaleId = fullLocaleId.replaceAll('_', '-');

    const subTagPart = localeIdMatch[2];
    if (!countryCodeMatch.test(subTagPart)) {
      const fallbackLocaleId = whaleLocaleFallbacks[canonicalLocaleId];
      const fallbackName = `_locales/${fallbackLocaleId}/messages.json`;

      if (!fallbackLocaleId || zip.file(fallbackName)) {
        packageChanges.push(`Removed ${fullLocaleId} from package. See: https://forum.whale.naver.com/topic/39749/`);
      } else {
        packageChanges.push(`Renamed ${fullLocaleId} translations to ${fallbackLocaleId}. See: https://forum.whale.naver.com/topic/39749/`);
        zip.file(fallbackName, file.async('uint8array'));
      }
      zip.remove(`_locales/${fullLocaleId}`);
    }
    return null;
  });

  await Promise.all(promises);

  if (whaleMessageLimit === Infinity) return;

  const anyLocaleMatch = /^_locales\/([a-zA-Z0-9_]+)\/messages\.json$/;

  const files = zip.file(anyLocaleMatch).sort((a, b) => {
    return incompatLocaleMatch.test(a.name) - incompatLocaleMatch.test(b.name);
  });

  const defaultLocale = manifestJson.default_locale || 'en';
  for (const file of files) {
    const fullLocaleId = file.name.match(anyLocaleMatch)[1];
    if (fullLocaleId === defaultLocale) continue;

    const subLocaleIdMatch = file.name.match(incompatLocaleMatch);
    const subTagPart = subLocaleIdMatch && subLocaleIdMatch[2];

    await file.async('string').then(async (text) => {
      const specificMessages = JSON.parse(text);
      const allKeys = Object.keys(specificMessages);
      const initialMessageCount = allKeys.length;
      if (initialMessageCount < whaleMessageLimit) return;

      const comparisonTag = subTagPart ? fullLocaleId.split(/[_-]/, 1)[0] : defaultLocale;

      const comparisonFile = zip.file(`_locales/${comparisonTag}/messages.json`);

      let removedDuplicateFallbackMessages = 0;
      let removedNonDuplicateMessages = 0;

      if (comparisonFile) {
        const comparisonText = await comparisonFile.async('string');
        const comparisonMessages = JSON.parse(comparisonText);

        for (const messageId of allKeys) {
          if (initialMessageCount - removedDuplicateFallbackMessages < whaleMessageLimit) break;

          if (manifestString.includes(messageId)) continue;
          const specificMessageData = specificMessages[messageId];
          const comparisonMessageData = comparisonMessages[messageId];
          if (!comparisonMessageData) continue;
          if (JSON.stringify(specificMessageData) !== JSON.stringify(comparisonMessageData)) continue;
          removedDuplicateFallbackMessages += 1;
          delete specificMessages[messageId];
        }

        if (removedDuplicateFallbackMessages) {
          packageChanges.push(`Removed ${removedDuplicateFallbackMessages} translations from ${fullLocaleId} which are also found in ${comparisonTag}. Whale Store gives 400 Error on too many translations.`);
        }
      }

      for (const messageId of Object.keys(specificMessages).reverse()) {
        if (initialMessageCount - removedDuplicateFallbackMessages - removedNonDuplicateMessages < whaleMessageLimit) break;
        removedNonDuplicateMessages += 1;
        delete specificMessages[messageId];
      }

      if (removedNonDuplicateMessages) {
        packageChanges.push(`Removed ${removedNonDuplicateMessages} translations from ${fullLocaleId}. Whale Store gives 400 Error on too many translations.`);
      }

      const finalText = stringifyInOriginalFormat(text, specificMessages);
      await zip.file(file.name, finalText);
    });
  }
}

function handleManifestIcons ({ manifestJson, store, packageChanges }) {
  if (store !== 'chrome') return;
  if (!manifestJson.icons) return;

  const unsupportedSizes = ['16', '32', '48', '128'];
  const regex = /^\//;
  for (const size of unsupportedSizes) {
    const sizeDeclaration = manifestJson.icons[size];
    const absolutePath = regex.test(sizeDeclaration);
    if (!absolutePath) continue;

    manifestJson.icons[size] = sizeDeclaration.replace('/', '');
    packageChanges.push(`Replaced "icons[${size}]" absolute path to relative path.`);
  }
}

function handleBackgroundScripting ({ manifestJson, store, packageChanges }) {
  if (store !== 'edge') return;

  const manifestVersion = manifestJson.manifest_version;
  if (manifestVersion < 3) return;

  const backgroundData = manifestJson.background;
  if (!backgroundData) return;

  const scripts = backgroundData.scripts;
  if (!scripts) return;

  const serviceWorker = backgroundData.service_worker;
  if (!serviceWorker) return;

  packageChanges.push('Removed background.scripts property as it is not accepted in manifest v3 by the Edge store. See: https://github.com/microsoft/MicrosoftEdge-Extensions/issues/136');
  delete backgroundData.scripts;
}

function truncatePropertyValue (manifestJson, key, limit, packageChanges) {
  const originalValue = manifestJson[key];
  if (typeof originalValue !== 'string') return;

  // do not truncate if it includes i18n replacements
  if (originalValue.includes('__MSG_')) return;

  // only truncate if exceeding the limits
  if (originalValue.length <= limit) return;

  let newValue = originalValue.replace(/\s/g, '');
  if (newValue.length > limit) {
    newValue = originalValue.slice(0, limit);
  }

  manifestJson[key] = newValue;
  packageChanges.push(`Truncated "${key}": "${originalValue}" to "${newValue}".`);
}

function handleManifestName ({ manifestJson, store, packageChanges }) {
  // name
  const nameLimit = store === 'chrome' || store === 'firefox' ? 45 : Infinity;
  truncatePropertyValue(manifestJson, 'name', nameLimit, packageChanges);

  // short_name
  let shortNameLimit = Infinity;
  if (store === 'opera') {
    shortNameLimit = 12;
  } else if (store === 'firefox' || store === 'chrome') {
    shortNameLimit = 45;
  }
  truncatePropertyValue(manifestJson, 'short_name', shortNameLimit, packageChanges);

  // description
  let descriptionLimit = Infinity;
  if (store === 'chrome') {
    descriptionLimit = 132;
  } else if (store === 'safari') {
    descriptionLimit = 112;
  }
  truncatePropertyValue(manifestJson, 'description', descriptionLimit, packageChanges);
}

function handleManifestDefaultLocale ({ manifestJson, store, packageChanges }) {
  if (store !== 'whale') return;

  const defaultLocale = manifestJson.default_locale;
  if (!defaultLocale) return;

  const canonicalDefaultLocale = defaultLocale.replaceAll('_', '-');

  const fallbackLocale = whaleLocaleFallbacks[canonicalDefaultLocale];
  if (!fallbackLocale) return;

  manifestJson.default_locale = fallbackLocale;
  packageChanges.push(`Replaced default_locale ${defaultLocale} with ${fallbackLocale}.`);
}

async function handleSinglePackage ({
  zipOrigin, store, params, manifestPolyfill,
}) {
  const packageChanges = [];

  const zip = new JSZip();

  await zip.loadAsync(zipOrigin);

  const manifestString = await zip.file('manifest.json').async('string');

  const manifestJson = JSON.parse(manifestString);

  if (manifestPolyfill) {
    const manifestPolyfillImport = await manifestPolyfill;
    const result = manifestPolyfillImport.default({
      manifestJson: manifestJson,
      targetVersion: manifestJson.manifest_version || 3,
      backgroundStrategy: params.backgroundStrategy || null,
      stores: [store],
      suppressLogging: true,
    });
    packageChanges.push(...result.changes);
  } else {
    handleManifestDefaultLocale({ manifestJson, store, packageChanges });
    handleManifestName({ manifestJson, store, packageChanges });
    handleManifestIcons({ manifestJson, store, packageChanges });
    handleBackgroundScripting({ manifestJson, store, packageChanges });
  }

  if (packageChanges.length > 0) {
    const manifestFinal = stringifyInOriginalFormat(manifestString, manifestJson);
    zip.file('manifest.json', manifestFinal);
  }

  if (store === 'opera') {
    removeIncompatibleWeblocFiles({ zip, packageChanges });
  }

  if (store === 'whale') {
    const { whaleMessageLimit } = params;
    await handleWhaleLocaleIssues({
      zip,
      packageChanges,
      manifestString,
      manifestJson,
      whaleMessageLimit,
    });
  }

  // removes translations from messages used in the manifest file
  // this allows the inclusion of translation without the need
  // to enter all store assets for each language

  if (store === 'edge' && params.edgeLocaleInclusions) {
    const madeLocaleChanges = await handleEdgeLocaleExclusions({
      zip, manifestString, manifestJson, params,
    });

    if (madeLocaleChanges) {
      packageChanges.push('Removed translations for messages in manifest.json.');
    }
  }

  return { zip, packageChanges };
}

function cleanStoreInput (inputStores) {
  const supportedStores = [
    'chrome',
    'edge',
    'opera',
    'samsung',
    'whale',
    'firefox',
    'safari',
  ];

  if (!Array.isArray(inputStores) || inputStores.length === 0) return supportedStores;

  return supportedStores.filter((store) => inputStores.includes(store));
}

function writeZipToDisk ({ zip, outputPath }) {
  const outputStream = fs.createWriteStream(outputPath);

  zip.forEach((relativePath) => {
    if (relativePath.includes('.DS_Store') || relativePath.includes('__MACOSX')) {
      zip.remove(relativePath);
    }
  });

  const inputStream = zip.generateNodeStream({
    type: 'nodebuffer',
    compression: 'DEFLATE',
    compressionOptions: {
      level: 6, // Standard balance of speed/size
    },
    platform: 'DOS',
    streamFiles: false,
  });

  return new Promise((resolve) => {
    inputStream.pipe(outputStream).on('finish', () => {
      resolve();
    });
  });
}

function readJsonFile (filePath) {
  try {
    const fileText = fs.readFileSync(filePath);
    return JSON.parse(fileText) || {};
  } catch {
    return {};
  }
}

function getVersion () {
  return process.env.npm_package_version
  || readJsonFile('./package.json').version
  || readJsonFile('./manifest.json').version
  || 'unknown';
}

function transformSingleStore ({
  zipOrigin, store, params, inputPath, manifestPolyfill,
}) {
  const messageLog = [];
  return handleSinglePackage({
    zipOrigin, store, params, manifestPolyfill,
  }).then(({ zip, packageChanges }) => {
    const fancyStoreName = store.toUpperCase();
    const canonicalStoreName = store.toLowerCase().split('-')[0];
    const colorReset = '\u001B[0m';
    messageLog.push('\u001B[1m\n' + storeTheming[canonicalStoreName] + fancyStoreName + colorReset);

    if (packageChanges.length === 0) {
      messageLog.push('\u001B[2m- No adaptions needed\u001B[0m.');
      return null;
    }

    const outputPath = inputPath.replace('.zip', '__adapted_for_' + store + '.zip');
    packageChanges.forEach((changeMessage) => {
      messageLog.push('- ' + changeMessage);
    });
    return writeZipToDisk({ zip, outputPath });
  }).then(() => messageLog);
}

export default function generate (params) {
  const stores = cleanStoreInput(params.stores);
  const version = getVersion();
  const inputPath = params.inputPath.replace('{version}', version);
  const zipOrigin = fs.promises.readFile(inputPath);
  const manifestPolyfill = params.manifestPolyfill && import('webext-manifest-browser-polyfill');
  const storeTransformations = stores.map((store) => transformSingleStore({
    zipOrigin, store, params, inputPath, manifestPolyfill,
  }));

  return Promise.all(storeTransformations).then((messageLogMap) => {
    const messageLog = messageLogMap.flat();
    messageLog.push('\nHandled store incompatibilities');
    return { messageLog };
  });
}