@thi.ng/sexpr/tokenize.js

Extensible S-Expression parser & runtime infrastructure

import { DEFAULT_SYNTAX } from "./api.js";
function* tokenize(src, opts) {
  const {
    scopes: rawScopes,
    whiteSpace,
    string,
    comment
  } = {
    ...DEFAULT_SYNTAX,
    ...opts
  };
  const scopes = rawScopes.reduce((acc, x) => acc.concat(x), []).join("");
  let token = "";
  let isString = false;
  let isComment = false;
  let tokenLine = 0;
  let tokenCol = 0;
  let line = 0;
  let col = -1;
  const $ = (value) => ({
    value,
    line: tokenLine,
    col: tokenCol
  });
  for (let c of src) {
    if (c === "\n") {
      line++;
      col = -1;
    } else {
      col++;
    }
    if (isComment) {
      if (c === "\n") isComment = false;
    } else if (!isString) {
      if (whiteSpace.test(c)) {
        token && (yield $(token));
        token = "";
      } else if (scopes.indexOf(c) !== -1) {
        token && (yield $(token));
        tokenLine = line;
        tokenCol = col;
        yield $(c);
        token = "";
        tokenCol++;
      } else if (c === string) {
        token && (yield $(token));
        tokenLine = line;
        tokenCol = col;
        token = '"';
        isString = true;
      } else if (c === comment) {
        isComment = true;
      } else {
        if (!token) {
          tokenLine = line;
          tokenCol = col;
        }
        token += c;
      }
    } else if (c === string && token[token.length - 1] !== "\\") {
      token += '"';
      yield $(token);
      token = "";
      isString = false;
    } else {
      token += c;
    }
  }
  token && (yield $(token));
}
export {
  tokenize
};