@lipsurf/cli/src/transform.ts

Build LipSurf plugins to add custom voice commands to any website.

/*;
 * Split the plugin into a frontend/backend and a matching/non-matching URL for
 * frontend. Frontend needs to be loaded on every page, so it should be lightweight.
 *
 * order of parts:
 *   backend
 *   matching frontend
 *   non-matching frontend
 *
 * We operate on the eval'd plugin js (with the node VM module)
 * because manipulating the js object is much more straightforward, less tedious
 * and less error prone.
 *
 * Using a parser like SWC is tedious and error prone because depending on the format
 * of the output, many cases would need to be handled. E.g. consider the various ways
 * that the various syntax that can be used to export the default module:
 *
 * export default Plugin;
 * export {
 *   Plugin as default
 * }
 * export default { ... }
 *
 * Before we eval the plugin, we replace all fn calls with:
 *   (specialVarName = () => {/* ...fn call... * /})
 *
 * Tried making it a string, but ran into unicode escaping issues.
 * A comment in a function can still be extracted with .toString()
 *
 * For frontend:
 *    * remove homophones, contexts, settings and other props only needed on the backend
 *    * remove command properties such as match, description, fn, test, delay etc.
 *    * replace non-default exports
 *    * TODO: remove commands that have no pageFn or dynamic match
 * Backend:
 *    * remove init, destroy
 *    * remove command properties: dynamic match fn, test, pageFn
 */
/// <reference types="@lipsurf/types/extension"/>
import { build } from "esbuild";
import { PluginPartType } from "./util";
import { evalPlugin } from "./evaluator";
import keyBy from "lodash/keyBy";
import mapValues from "lodash/mapValues";
import omit from "lodash/omit";
import { join } from "path";
import { promises as fs } from "fs";
import {
  CallExpression,
  parse,
  ModuleItem,
  Span,
  Identifier,
  Expression,
  MemberExpression,
} from "@swc/core";
import Visitor from "@swc/core/Visitor";
import clone from "clone";

type PluginSpreadParts = [
  backend: string,
  freeCS: string,
  freeNonMatchingCS: string,
  plusCS: string,
  plusNonMatchingCS: string,
  premiumCS: string,
  premiumNonMatchingCS: string
];

// --- hack until we have public + private common
// import {
//   PLANS,
//   PLUGIN_SPLIT_SEQ,
//   EXT_ID,
//   FREE_PLAN,
//   PLUS_PLAN,
//   PREMIUM_PLAN,
// } from "@lipsurf/common/constants.cjs";
const FREE_PLAN = 0;
const PLUS_PLAN = 10;
const PREMIUM_PLAN = 20;
const EXT_ID =
  process.env.EXT_ID || process.env.BROWSER === "safari"
    ? "11C90F6F-8158-43E7-B45D-61FBD96C7288"
    : "lnnmjmalakahagblkkcnjkoaihlfglon";
const EXT_RESOURCES_PREFIX =
  process.env.BROWSER === "safari"
    ? `safari-web-extension://${EXT_ID}`
    : `chrome-extension://${EXT_ID}`;
const BEAUTIFY = process.env.BEAUTIFY === "true";
const PLANS: [plan, plan, plan] = [FREE_PLAN, PLUS_PLAN, PREMIUM_PLAN];
const PLUGIN_SPLIT_SEQ = "\vLS-SPLIT";
// import { escapeRegex } from "@lipsurf/common/util.cjs";
const escaper = /[.*+?^${}()|[\]\\]/g;
export function escapeRegex(s) {
  return s.replace(escaper, "\\$&");
}
const PURE_FUNCS =
  (process.env["STRIP_LOGS"] || "").toLowerCase() === "false"
    ? []
    : [
        "console.log",
        "console.dir",
        "console.trace",
        "console.debug",
        "console.time",
        "console.timeEnd",
      ];
// -- end hack
const PLUGIN_PROPS_TO_REMOVE_FROM_CS = [
  "description",
  "homophones",
  "version",
  "authors",
  "icon",
  "match",
  "plan",
  "apiVersion",
  "contexts",
  "niceName",
  "replacements",
  "settings",
];

const COMMAND_PROPS_TO_REMOVE_FROM_CS = [
  "fn",
  "delay",
  "description",
  "test",
  "global",
  "normal",
  "context",
  "onlyFinal",
  "minConfidence",
  "enterContext",
  "activeDocument",
];

const PLUGIN_BASE_PROPS = [
  "annotations",
  "util",
  "help",
  "constants",
  "languages",
];

const importPluginBase = `import PluginBase from '${EXT_RESOURCES_PREFIX}/dist/modules/plugin-base.js';`;
const importExtensionUtil = `import ExtensionUtil from '${EXT_RESOURCES_PREFIX}/dist/modules/extension-util.js';`;
const FN_ESCAPE_TAG = `LIPSURF_FN_ESC`;
const COMMENT_ENDER_PLACEHOLDER = "LIPSURF_CMT_ENDER";
const COMMENT_ENDER_PLACEHOLDER_REGX = new RegExp(
  COMMENT_ENDER_PLACEHOLDER,
  "g"
);
const COMMENT_ENDER_REGEX = new RegExp("\\*/", "g");
const FN_ESCAPE_PREFIX = `(()=>{/*${FN_ESCAPE_TAG}`;
const FN_ESCAPE_SUFFIX = `*/})`;

class BlankPartError extends Error {}

function replaceCmdsAbovePlan(plugin: IPlugin, buildForPlan: plan): IPlugin {
  let cmdsOnThisPlan: boolean = false;
  const pluginPlan = plugin.plan || 0;
  // if nothing is replaced, and the highestLevel is lower than build for plan, then return false so we
  // don't build for this level (the highest level might have been 10 or 0, and already built)
  plugin.commands = plugin.commands.map((cmd) => {
    const cmdPlan = cmd.plan;
    const minNeededPlan = cmdPlan ? cmdPlan : pluginPlan;
    let replace = false;
    if (!cmdPlan) {
      if (pluginPlan === buildForPlan) cmdsOnThisPlan = true;
      else if (pluginPlan > buildForPlan) replace = true;
    } else {
      if (buildForPlan === cmdPlan) cmdsOnThisPlan = true;
      if (minNeededPlan > buildForPlan) replace = true;
    }
    if (replace) {
      // @ts-ignore
      // `top.` since this can be called on frame-beacon
      cmd.pageFn = new Function(
        `top.showNeedsUpgrade({plan: ${minNeededPlan}})`
      );
    }
    return cmd;
  });
  if (!cmdsOnThisPlan && buildForPlan !== 0) throw new BlankPartError();
  return plugin;
}

/**
 * Slim down backend code
 * TODO: remove languages.commands.match.fn
 * @param plugin
 */
function makeBackend(plugin: IPlugin, prod: boolean = false) {
  delete plugin.init;
  delete plugin.extraClickables;
  delete plugin.destroy;
  // @ts-ignore
  delete plugin.annotations;
  // @ts-ignore
  delete plugin.util;
  // @ts-ignore
  delete plugin.help;
  // delete plugin.languages;
  for (let i = plugin.commands.length - 1; i >= 0; i--) {
    const cmd = plugin.commands[i];
    if (prod) delete cmd.test;
    delete cmd.pageFn;
    // @ts-ignore
    if (cmd.match.fn) {
      // @ts-ignore
      cmd.match.fn = () => {};
    }
  }
  return plugin;
}

function makeCS(plugin: IPlugin, plan: plan, type: PluginPartType) {
  // must happen before we remove COMMAND_PROPS_TO_REMOVE_FROM_CS and PLUGIN_PROPS_TO_REMOVE_FROM_CS
  // since both have plan property
  plugin = replaceCmdsAbovePlan(plugin, plan);

  for (const prop of PLUGIN_PROPS_TO_REMOVE_FROM_CS) {
    delete plugin[prop];
  }

  for (let i = plugin.commands.length - 1; i >= 0; i--) {
    const cmd = plugin.commands[i];
    if (type === PluginPartType.nonmatching && !cmd.global) {
      plugin.commands.splice(i, 1);
      continue;
    }
    for (const prop of COMMAND_PROPS_TO_REMOVE_FROM_CS) {
      delete cmd[prop];
    }
    if (Array.isArray(cmd.match) || typeof cmd.match === "string") {
      // @ts-ignore
      delete cmd.match;
    } else {
      // merge localized match fns into the plugin.commands.match object
      const oldMatch = cmd.match;
      // @ts-ignore
      cmd.match = Object.keys(plugin.languages || []).reduce(
        (memo, lang) => {
          const localizedFn =
            plugin.languages![lang]?.commands[cmd.name]?.match.fn;
          if (localizedFn) memo[lang] = localizedFn;
          return memo;
        },
        { en: oldMatch.fn }
      );
    }
    if (typeof cmd.nice !== "function") {
      delete cmd.nice;
    }
    // only the name key
    if (Object.keys(cmd).length === 1) plugin.commands.splice(i, 1);
  }
  if (!plugin.commands.length && !plugin.init && !plugin.destroy)
    throw new BlankPartError();

  // array to obj
  // @ts-ignore
  plugin.commands = mapValues(keyBy(plugin.commands, "name"), (v) =>
    omit(v, "name")
  );

  if (type === PluginPartType.nonmatching) {
    delete plugin.extraClickables;
  }

  // remove plugin-base props
  for (const prop of PLUGIN_BASE_PROPS) {
    delete plugin[prop];
  }
  return plugin;
}

function replaceSpans(
  code: string,
  spans: Span[],
  replacer?: (s: string) => string
) {
  const codeParts: string[] = [];
  let prevEnd = 0;
  let i = 0;
  for (; i < spans.length; i++) {
    const curSpan = spans[i];
    // console.log("replacing span", code.substring(spans[i].start, spans[i].end));
    codeParts.push(code.substring(prevEnd, curSpan.start));
    if (replacer) {
      codeParts.push(replacer(code.substring(curSpan.start, curSpan.end)));
    }
    prevEnd = curSpan.end;
  }
  codeParts.push(code.substring(prevEnd));
  return codeParts.join("");
}

function getLanguageDefSpans(pluginId: string, body: ModuleItem[]): Span[] {
  return body
    .filter(
      (x) =>
        x.type === "ExpressionStatement" &&
        x.expression &&
        x.expression.type === "AssignmentExpression" &&
        x.expression.left &&
        x.expression.left.type === "MemberExpression" &&
        x.expression.left.object.type === "MemberExpression" &&
        x.expression.left.object.object.type === "Identifier" &&
        x.expression.left.object.object.value === `${pluginId}_default` &&
        x.expression.left.property.type === "Identifier" &&
        x.expression.left.property.value.length === 2
    )
    .map((x) => x.span);
}

/**
 * replace all called fns with strings, so that they
 * aren't resolved in the wrong context (when plugin
 * is building)
 * Important so that closures are preserved (eg. when
 * pageFn is a called fn with closure vars)
 */
class FnReplacer extends Visitor {
  public callExpressionSpans: Span[] = [];

  // hack to get around span not starting at 0 in swc (last tested in v1.2.62)
  constructor(private offsetHack) {
    super();
  }

  visitCallExpression(c: CallExpression) {
    // console.log(c);
    // return c;
    // hack to get around span not starting at 0 in swc (last tested in v1.2.62)
    const start = c.span.start - this.offsetHack;
    const end = c.span.end - this.offsetHack;
    this.callExpressionSpans.push({ start, end, ctxt: c.span.ctxt });
    return c;
  }

  /**
   * Need to rescue PluginBase, ExtensionUtil... calls from being optimized out
   * @param n
   * @returns
   */
  visitMemberExpression(n: MemberExpression): Expression {
    if (
      n.object.type === "MemberExpression" &&
      n.object.object.type == "Identifier" &&
      (n.object.object.value === "PluginBase" ||
        n.object.object.value === "ExtensionUtil")
    ) {
      // // @ts-ignore
      // console.log("pushing member expression", n.property.value);
      this.callExpressionSpans.push({
        start: n.span.start - this.offsetHack,
        end: n.span.end - this.offsetHack,
        ctxt: n.span.ctxt,
      });
    }
    return n;
  }
}

function getPluginSpan(pluginId: string, body: ModuleItem[]): Span {
  return body.find(
    (x) =>
      x.type === "VariableDeclaration" &&
      x.declarations[0].type === "VariableDeclarator" &&
      x.declarations[0].id.type === "Identifier" &&
      x.declarations[0].id.value === `${pluginId}_default` &&
      x.declarations[0].init &&
      x.declarations[0].init.type == "ObjectExpression"
    // @ts-ignore
  )!.declarations[0].init.span;
}

function versionConvertDots(v) {
  return v.replace(/\./g, "-");
}

export function transformJSToPlugin(
  pluginId: string,
  globbedTs: string[],
  outdir: string,
  prod: boolean,
  baseImports: boolean,
  define: {}
) {
  const pluginWLanguageFiles = globbedTs
    .map((f) => f.replace(/^src\//, "dist/tmp/").replace(/.ts$/, ".js"))
    .filter((x) => x.substring(x.lastIndexOf("/")).includes(`/${pluginId}.`))
    .sort((a, b) => a.length - b.length);
  const resolveDir = pluginWLanguageFiles[0].substring(
    0,
    pluginWLanguageFiles[0].lastIndexOf("/")
  );
  return makePlugin(
    pluginId,
    pluginWLanguageFiles,
    resolveDir,
    prod,
    baseImports,
    define
  )
    .then((res) => {
      const version = versionConvertDots(res[1]);
      return Promise.all(
        res[0].map((c, i) =>
          c
            ? fs.writeFile(
                `${join(outdir, pluginId)}.${version}.${PLANS[i]}.ls`,
                c,
                "utf8"
              )
            : undefined
        )
      );
    })
    .catch((e: any) => {
      console.error(e, `Error building ${pluginId}`);
      throw e;
    });
}

/**
 * * The enclosing parenthesis are something stripped in evaluation,
 *   so we don't look for them in this regex.
 * * "s" flag is so dot can match newlines as well
 */
const ARTIFACT_REGX = new RegExp(
  `${escapeRegex(FN_ESCAPE_PREFIX.substring(1))}(.*?)${escapeRegex(
    FN_ESCAPE_SUFFIX.substring(0, FN_ESCAPE_SUFFIX.length - 1)
  )}`,
  "gs"
);
function removeReplacedCallArtifacts(s: string): string {
  return s
    .replace(ARTIFACT_REGX, "$1")
    .replace(COMMENT_ENDER_PLACEHOLDER_REGX, "*/");
}

async function makePlugin(
  pluginId: string,
  pluginWLanguageFiles: string[],
  resolveDir: string,
  prod = false,
  baseImports = true,
  define = {}
): Promise<[pluginForSub: PluginSub, version: string]> {
  let buildRes;
  try {
    const importPluginDepsCode = [
      ...pluginWLanguageFiles.map((x, i) => {
        const name = x.substring(x.lastIndexOf("/") + 1, x.length - 3);
        return i === 0
          ? `import plugin from "./${name}";`
          : `import "./${name}";`;
      }),
      `export default plugin;`,
    ].join("\n");
    const dumbySrcName = `dumby`;

    // bundle the deps
    buildRes = await build({
      stdin: {
        contents: importPluginDepsCode,
        sourcefile: `${dumbySrcName}.js`,
        resolveDir,
        loader: "js",
      },
      // charset: "utf8",
      format: "esm",
      write: false,
      bundle: true,
      incremental: true,
      define,
      minifyWhitespace: true,
      // does not minify syntax because missing comments cause offset issues with swc
    });

    const resolvedPluginCode = <string>buildRes.outputFiles[0].text
      .replace("...PluginBase", "...PluginBase, ...{languages: {}}")
      // needed because ES build does not escape unicode characters in regex literals
      .replace(/[^\x00-\x7F]/g, (x) => {
        const escaped = escape(x);
        if (escaped.startsWith("%u")) return `\\${escaped.substring(1)}`;
        return `\\x${escaped.substring(1)}`;
      });

    const byPlanAndMatching = {
      [FREE_PLAN]: {},
      [PLUS_PLAN]: {},
      [PREMIUM_PLAN]: {},
    };

    const ast = await parse(resolvedPluginCode, {
      syntax: "ecmascript",
      dynamicImport: true,
    });

    let { start: pluginSrcReplacementStartI, end: pluginSrcReplacementEndI } =
      getPluginSpan(pluginId, ast.body);

    // hack to get around span not starting at 0 in swc (last tested in v1.2.60)
    pluginSrcReplacementEndI = pluginSrcReplacementEndI - ast.span.start;
    pluginSrcReplacementStartI = pluginSrcReplacementStartI - ast.span.start;

    // assume languages come after plugin definition (otherwise pluginSrcReplacementStartI would need to be adjusted)
    const languageObjsRemovedCode = replaceSpans(
      resolvedPluginCode,
      getLanguageDefSpans(pluginId, ast.body)
    );

    // console.time("escape fn calls");
    const fnReplacer = new FnReplacer(ast.span.start);
    fnReplacer.visitProgram(ast);
    const noFnCallsCode = replaceSpans(
      resolvedPluginCode,
      fnReplacer.callExpressionSpans,
      (code) =>
        `${FN_ESCAPE_PREFIX}${code.replace(
          COMMENT_ENDER_REGEX,
          COMMENT_ENDER_PLACEHOLDER
        )}${FN_ESCAPE_SUFFIX}`
    );
    // console.log(noFnCallsCode);
    // console.timeEnd("escape fn calls");

    let parsedPluginObj: IPlugin;
    try {
      parsedPluginObj = await evalPlugin(
        `var ${FN_ESCAPE_TAG};${noFnCallsCode}`
      );
    } catch (e) {
      throw new Error(`Error evaluating ${e}\n\ncode: ${noFnCallsCode}`);
    }
    const version = parsedPluginObj.version || "1.0.0";
    const exportRegx = new RegExp(
      `var\\s*${dumbySrcName}_default\\s*=\\s*${pluginId}_default;\\s*export\\s*{\\s*${dumbySrcName}_default\\s+as\\s+default\\s*};?`
    );

    for (const plan of PLANS) {
      let type: PluginPartType;
      for (type of Object.values<PluginPartType>(<any>PluginPartType).filter(
        (x) => !isNaN(Number(x))
      )) {
        const cloned: IPlugin = clone(parsedPluginObj, false);
        let code: string;
        try {
          code = `${languageObjsRemovedCode.substring(
            0,
            pluginSrcReplacementStartI
          )}{...PluginBase, ...${removeReplacedCallArtifacts(
            uneval(makeCS(cloned, plan, type))
          )}}${languageObjsRemovedCode.substring(pluginSrcReplacementEndI)}`;
        } catch (e) {
          if (e instanceof BlankPartError) code = "";
          else throw new Error(`Error transforming ${pluginId}.${plan} ${e}`);
        }
        /**
         * Might look like this for production build:
         * var t = { ... },
         * l = t;
         * var randomStuff;
         * function randomFn() { .. }
         * export {
         *  l as
         *  default
         */
        if (code && !exportRegx.test(code)) {
          throw new Error(`Could not find the export regex in code.`);
        }
        byPlanAndMatching[plan][type] = code
          ? `window.allPlugins.${pluginId} = (() => { ${code.replace(
              exportRegx,
              ""
            )}
              return ${pluginId}_default; })();`
          : "";
      }
    }

    // make backend
    const backend = `${resolvedPluginCode.substring(
      0,
      pluginSrcReplacementStartI
    )}${removeReplacedCallArtifacts(
      uneval(makeBackend(parsedPluginObj, prod))
    )}`.replace(new RegExp(`var ${pluginId}_default\s*=`), "export default ");

    const transformedPluginsTuple = [
      backend,
      ...PLANS.reduce(
        (memo, p) =>
          memo.concat([
            byPlanAndMatching[p][PluginPartType.matching],
            byPlanAndMatching[p][PluginPartType.nonmatching],
          ]),
        <string[]>[]
      ),
    ] as PluginSpreadParts;

    // Only for minifying and treeshaking
    const builtParts = await Promise.all(
      transformedPluginsTuple.map((code, i) =>
        // it would put in the allPlugins.${pluginId} = ... code if we build with code=""
        code
          ? build({
              // entryPoints: ,
              // outdir: options.outDir,
              stdin: {
                contents: code,
                sourcefile: `${pluginId}.js`,
                resolveDir,
                loader: "js",
              },
              charset: "utf8",
              write: false,
              bundle: true,
              // for iife
              // format: "iife",
              // globalName: `allPlugins.${pluginId}`,
              format: "esm",
              treeShaking: true,
              // handles minifyWhitespace, minifyIdentifiers, and minifySyntax
              minify: !BEAUTIFY && prod,
              pure: prod ? PURE_FUNCS : [],
              // incremental: true,
              // defaults to esNext (we build to the target with tsc)
              target: "es2019",
            }).catch((e) => {
              console.log(`Error building ${i}\n${e}`);
              throw e;
            })
          : Promise.resolve({ outputFiles: [{ text: "" }] })
      )
    );

    const splitPluginTuple = builtParts.map((f) =>
      f ? f.outputFiles[0].text : f
    );

    let baseImportsStr = "";
    if (baseImports) {
      baseImportsStr = importPluginBase + importExtensionUtil;
    }

    const finalPluginsTuple: Partial<PluginSub> = [];
    // combine the files into .ls file
    for (let i = 0; i < PLANS.length; i++) {
      const matchingNonMatching = splitPluginTuple.slice(
        i * 2 + 1,
        (1 + i) * 2 + 1
      );
      if (
        PLANS[i] !== FREE_PLAN &&
        // @ts-ignore
        matchingNonMatching.reduce((memo, x) => memo + x.length, 0) === 0
      )
        // no plugin for this level
        finalPluginsTuple.push("");
      else
        finalPluginsTuple.push(
          [baseImportsStr + splitPluginTuple[0], ...matchingNonMatching].join(
            PLUGIN_SPLIT_SEQ
          )
        );
    }
    return [<PluginSub>finalPluginsTuple, version];
  } catch (e) {
    console.error(e);
    throw e;
  } finally {
    // cleanup
    buildRes.rebuild?.dispose();
  }
}

function uneval(l: any): string {
  switch (true) {
    // case l instanceof Function: // (does not work for async functions)
    case typeof l === "function":
      /**
       * Handle named functions.
       * e.g.
       *   {
       *    init() { ... }
       *   }
       */
      const stringified: string = l.toString();
      const name = l.name;
      return name && new RegExp(`^(async )?${name}`).test(stringified)
        ? stringified.replace(name, "function")
        : stringified.toString();
    /**
     * instanceof RegExp breaks when coming from different window
     */
    case Object.prototype.toString.call(l) == "[object RegExp]":
      // hmm, this might break if there's a new RegExp line with slashes or newlines?
      return `/${l.source}/${l.flags}`;
    case Array.isArray(l):
      return `[${l.map((item) => uneval(item)).join(",")}]`;
    case l === Object(l):
      return `{${Object.keys(l)
        // JSON.stringify properly escapes the quotes
        .map((k) => `${JSON.stringify(k)}: ${uneval(l[k])}`)
        .join(",")}}`;
    // instanceof String doesn't work
    case typeof l === "string":
      // JSON.stringify properly escapes the quotes
      return JSON.stringify(l);
    default:
      return l;
  }
}