ciallorize/tokenize.js

Ciallorize your bundled js into ascii art.

const parser = require('@babel/parser');
const compareAST = require('./compare_ast.js');
const traverse = require('@babel/traverse').default;
const errorMsg = require('./error_msg.js');

function splitCodeIntoTokens(code) {
  // 解析 AST 获取敏感区域
  const ast = parser.parse(code, {
    sourceType: 'script',
    tokens: true,
  });
  
  // 收集敏感区域（模板字符串、字符串、正则等）
  const sensitiveRanges = [];
  traverse(ast, {
    // 模板字符串
    TemplateLiteral(path) {
      sensitiveRanges.push({
        start: path.node.start,
        end: path.node.end,
        type: 'TemplateLiteral'
      });
    },
    // 字符串字面量
    StringLiteral(path) {
      sensitiveRanges.push({
        start: path.node.start,
        end: path.node.end,
        type: 'StringLiteral'
      });
    },
    // 正则表达式
    RegExpLiteral(path) {
      sensitiveRanges.push({
        start: path.node.start,
        end: path.node.end,
        type: 'RegExpLiteral'
      });
    }
  });

  // 提取所有 token 的位置
  const tokenPositions = new Set([0, code.length]);
  ast.tokens.forEach(token => {
    tokenPositions.add(token.start);
  });

  // 创建最终分割点集合
  const splitPositions = new Set();
  
  // 添加非敏感区域的 token 分割点
  tokenPositions.forEach(pos => {
    const isInSensitiveRange = sensitiveRanges.some(range => 
      pos > range.start && pos < range.end
    );
    
    if (!isInSensitiveRange) {
      splitPositions.add(pos);
    }
  });
  
  // 添加敏感区域的边界作为分割点
  sensitiveRanges.forEach(range => {
    splitPositions.add(range.start);
    splitPositions.add(range.end);
  });

  // 分割代码
  const sortedPositions = [...splitPositions].sort((a, b) => a - b);
  const elements = [];
  
  for (let i = 0; i < sortedPositions.length - 1; i++) {
    const start = sortedPositions[i];
    const end = sortedPositions[i + 1];
    elements.push(code.slice(start, end));
  }
  
  return elements;
}

// // 示例用法
// const originalCode = fs.readFileSync("D:/nodert/lib/util.corelib.js", 'utf8');
// const sequence = splitCodeIntoTokens(originalCode);
// console.log(sequence);

// // 插入空格后重组（任意数量空格 ≥1）
// const rebuiltCode = sequence.join(' '.repeat(Math.floor(Math.random()*42+1))); 

// // 验证 AST 相同（需自行用 Babel 解析并比较，此处省略）
// console.log(compareAST(originalCode, rebuiltCode))

module.exports = (code)=>{
    const sequence = splitCodeIntoTokens(code);
    try{
        const rebuiltCode = sequence.join(' '.repeat(Math.floor(Math.random()*3)));
        if(!(compareAST(code, rebuiltCode))){
            throw new Error()
        }
    }catch(e){
        errorMsg('Tokenize')
        throw new Error('Failed to tokenize the code')
    }
    return sequence
}