unity-find-fault/dist/core/JsonStriper.js

A tool to find fault in unity project.

import fg from "fast-glob";
import fs from "fs-extra";
import path from "path";
import { Project } from "ts-morph";
import xml2js from "xml-js";
import { readGB2312, writeGB2312 } from "./vendor.js";
import moment from "moment";
import { toolchain } from "../toolchain.js";
export class JsonStriper {
    async removeUseless() {
        const project = new Project({
            tsConfigFilePath: path.join(toolchain.opts.projectRoot, 'TsScripts/tsconfig.json')
        });
        const outputFile = `tmp/${toolchain.opts.projectName}.jsonNotUsed.txt`;
        await this.findNotUsed(project, outputFile);
        const outContent = await fs.readFile(outputFile, 'utf-8');
        const lines = outContent.split(/\r?\n/);
        const notUsedItfs = [];
        const notUsedPpts = [];
        for (const line of lines) {
            if (!line || line.includes('---'))
                continue;
            const pair = line.split(':');
            if (pair.length == 1) {
                notUsedItfs.push(line);
            }
            else {
                notUsedPpts.push(line);
            }
        }
        const book = await this.loadConvertXml();
        await this.removeBin(book, notUsedItfs);
        await this.removeItfDefinitions(book, notUsedItfs, notUsedPpts);
        await this.removeJsonLoading();
    }
    async findNotUsed(project, outFile) {
        if (!toolchain.opts.designerRoot) {
            console.error('no designer root');
            process.exit(1);
        }
        // GameConfig.d.ts
        const usedItfs = [];
        const notUsedItfs = [];
        const notUsedPpts = [];
        const gameConfigFile = path.join(toolchain.opts.projectRoot, 'TsScripts/System/data/GameConfig.d.ts');
        const gameConfigSrc = project.getSourceFile(gameConfigFile);
        const module = gameConfigSrc.getModuleOrThrow('GameConfig');
        const interfaces = module.getInterfaces();
        for (const itf of interfaces) {
            const properties = itf.getProperties();
            const nups = [];
            for (const property of properties) {
                // if (property.getName() == 'm_iSkillPriority' && itf.getName() == 'SkillConfigM') {
                //     console.log('this comes!');
                // }
                if (!this.isNodeReferedOutside(property, [gameConfigFile])) {
                    nups.push(itf.getName() + ':' + property.getName());
                    // console.log('property not used:', itf.getName() + ':' + property.getName());
                }
            }
            if (nups.length == properties.length) {
                notUsedItfs.push(itf.getName());
            }
            else {
                usedItfs.push(itf.getName());
                notUsedPpts.push(...nups);
            }
        }
        // console.log('used itfs:', usedItfs.join(' | '));
        await fs.writeFile(outFile, notUsedItfs.join('\n') + '\n\n-------------\n\n' + notUsedPpts.join('\n'), 'utf-8');
        // 删除json文件
        for (const itf of notUsedItfs) {
            const jsonFile = path.join(toolchain.opts.projectRoot, 'Assets/AssetSources/data', itf + '.json');
            if (fs.existsSync(jsonFile)) {
                await fs.unlink(jsonFile);
            }
            const metaFile = jsonFile + '.meta';
            if (fs.existsSync(metaFile)) {
                await fs.unlink(metaFile);
            }
        }
    }
    isNodeReferedOutside(node, excludes) {
        const refSymbols = node.findReferences();
        let used = false;
        for (const refSymbol of refSymbols) {
            for (const ref of refSymbol.getReferences()) {
                if (ref.isDefinition())
                    continue;
                // console.log(`${refSymbol.getDefinition().getName()} refered in ${ref.getSourceFile().getFilePath()}`);
                if (!excludes.includes(path.resolve(ref.getSourceFile().getFilePath()))) {
                    used = true;
                    break;
                }
            }
            if (used)
                break;
        }
        return used;
    }
    async loadConvertXml() {
        const book = {};
        const convertXml = path.join(toolchain.opts.designerRoot, 'xml/1_ConvList.xml');
        const xmlContent = await readGB2312(convertXml);
        const xml = xml2js.xml2json(xmlContent, { compact: true, spaces: 0 });
        const xmlObj = JSON.parse(xml);
        for (const node of xmlObj.ConvList.ConvTree.CommNode) {
            if (node.ResNode == null)
                continue;
            if (node.ResNode instanceof Array) {
                for (const n of node.ResNode) {
                    book[n._attributes.Meta.replace('_Flash', 'M')] = n._attributes;
                }
            }
            else {
                book[node.ResNode._attributes.Meta.replace('_Flash', 'M')] = node.ResNode._attributes;
            }
        }
        return book;
    }
    /**删除bin文件 */
    async removeBin(book, notUsedItfs) {
        for (const itf of notUsedItfs) {
            if (book[itf] == null)
                continue;
            const binFile = path.join(toolchain.opts.designerRoot, 'bin', book[itf].BinFile);
            if (fs.existsSync(binFile)) {
                await fs.unlink(binFile);
            }
            const xmlFile = binFile.replace('.bin', '.xml');
            if (fs.existsSync(xmlFile)) {
                await fs.unlink(xmlFile);
            }
        }
    }
    async removeJsonLoading() {
        const cmf = path.join(toolchain.opts.projectRoot, 'TsScripts/System/data/configmanager.ts');
        const cmfContent = await fs.readFile(cmf, 'utf-8');
        const lines = cmfContent.split(/\r?\n/);
        for (let i = lines.length - 1; i >= 0; i--) {
            const mch = lines[i].match(/^\s*(['|"])(\w+M)\1,?\s*$/);
            if (mch) {
                const jf = path.join(toolchain.opts.projectRoot, 'Assets/AssetSources/data', mch[2] + '.json');
                if (!fs.existsSync(jf)) {
                    lines.splice(i, 1);
                }
            }
        }
        const newContent = lines.join('\n');
        if (newContent != cmfContent) {
            await fs.writeFile(cmf, newContent, 'utf-8');
        }
    }
    async removeItfDefinitions(book, notUsedItfs, notUsedPpts) {
        const xmlRoot = path.join(toolchain.opts.designerRoot, 'xml');
        const xmlFiles = await fg('*.xml', { ignore: ['1_ConvList.xml'], cwd: xmlRoot });
        const svrStructs = [];
        for (const n in book) {
            if (book[n].BinFile.startsWith('Server\\')) {
                svrStructs.push(n);
            }
        }
        const notUsedPptMap = {}, proteceds = [];
        for (const ppt of notUsedPpts) {
            const [itf, property] = ppt.split(':');
            let arr = notUsedPptMap[itf];
            if (arr == null)
                notUsedPptMap[itf] = arr = [];
            arr.push(property);
        }
        for (const x of xmlFiles) {
            const xmlFile = path.join(xmlRoot, x);
            const xmlContent = await readGB2312(xmlFile);
            const lines = xmlContent.split(/\r?\n/);
            // 先获取被服务器结构引用的子结构
            this.getProtectedStructures(lines, svrStructs, proteceds);
        }
        for (const x of xmlFiles) {
            const xmlFile = path.join(xmlRoot, x);
            const xmlContent = await readGB2312(xmlFile);
            const lines = xmlContent.split(/\r?\n/);
            // 获取被refer的字段
            const referedMap = this.getReferedEntries(lines);
            let theItf = '', sortkey = '', notUsed = false, newLines = [], modified = false, lastIsComment = false;
            for (let i = 0, len = lines.length; i < len; i++) {
                let line = lines[i];
                const originalLine = line;
                let removed = false;
                const mch = line.match(/^\s*<struct\s+name\s*=\s*"(\w+)"/);
                if (mch != null) {
                    // struct定义开始
                    theItf = mch[1].replace('_Flash', 'M');
                    const skmch = line.match(/sortkey\s*=\s*"(\w+)"/);
                    if (skmch) {
                        sortkey = skmch[1];
                    }
                    else {
                        sortkey = '';
                    }
                    if (notUsedItfs.includes(theItf) && !proteceds.includes(theItf)) {
                        // 整个结构不要
                        notUsed = true;
                        removed = true;
                        if (lastIsComment) {
                            // 把注释也删掉
                            newLines.pop();
                        }
                    }
                }
                else {
                    if (notUsed) {
                        removed = true;
                    }
                    if (line.match(/^\s*<\/struct>/)) {
                        // struct定义结束
                        theItf = '';
                        sortkey = '';
                        notUsed = false;
                    }
                    else if (theItf && !notUsed) {
                        const pmch = line.match(/^\s*<entry\s+name="(\w+)"/);
                        if (pmch != null) {
                            // entry定义开始
                            const entry = pmch[1];
                            const arr = notUsedPptMap[theItf];
                            if (arr != null && arr.includes(entry)) {
                                if (entry == sortkey || referedMap[theItf][entry] && !arr.includes(referedMap[theItf][entry]) || proteceds.includes(theItf)) {
                                    // sortkey、被refer的字段、和后台共享的子结构的字段均需保留，但添加convertOption noJson
                                    const coMch = line.match(/convertOption="(\S+)"/);
                                    if (coMch != null) {
                                        if (!coMch[1].includes('noJson')) {
                                            line = line.replace('convertOption="', 'convertOption="noJson,').replace('?', '');
                                        }
                                    }
                                    else {
                                        line = line.replace('/>', ' convertOption="noJson"/>');
                                    }
                                }
                                else {
                                    // 字段不要
                                    removed = true;
                                }
                            }
                        }
                    }
                }
                if (removed) {
                    modified = true;
                }
                else {
                    if (line != originalLine)
                        modified = true;
                    newLines.push(line);
                }
                lastIsComment = line.match(/^\s*<!--.+-->\s*$/) != null;
            }
            if (modified)
                await writeGB2312(xmlFile, newLines.join('\n'));
        }
    }
    getProtectedStructures(lines, svrStructs, out) {
        // 获取因被服务器结构引用而需要保护的结构
        let isSvrStruct = false;
        for (let i = 0, len = lines.length; i < len; i++) {
            let line = lines[i];
            const mch = line.match(/^\s*<struct\s+name\s*=\s*"(\w+?)"/);
            if (mch != null) {
                // struct定义开始
                const structName = mch[1];
                isSvrStruct = svrStructs.includes(structName) || structName.endsWith('_Server');
            }
            else {
                if (line.match(/^\s*<\/struct>/)) {
                    // struct定义结束
                    isSvrStruct = false;
                }
                else if (isSvrStruct) {
                    const pmch = line.match(/^\s*<entry\s+.*\btype\b="(\w+)"/);
                    if (pmch != null) {
                        // entry定义开始
                        const type = pmch[1];
                        if (!out.includes(type))
                            out.push(type);
                    }
                }
            }
        }
    }
    getReferedEntries(lines) {
        // 获取因被refer而需要保护的字段
        const out = {};
        let theItf = '';
        for (let i = 0, len = lines.length; i < len; i++) {
            let line = lines[i];
            const mch = line.match(/^\s*<struct\s+name\s*=\s*"(\w+)".*/);
            if (mch != null) {
                // struct定义开始
                theItf = mch[1].replace('_Flash', 'M');
                out[theItf] = {};
            }
            else {
                if (line.match(/^\s*<\/struct>/)) {
                    // struct定义结束
                    theItf = '';
                }
                else if (theItf) {
                    const pmch = line.match(/^\s*<entry\s+.*\bname\b\s*=\s*"(\w+)".+refer\s*="(\w+)"/);
                    if (pmch != null) {
                        // entry定义开始
                        const entry = pmch[1], refer = pmch[2];
                        out[theItf][refer] = entry;
                    }
                }
            }
        }
        return out;
    }
    async checkLiveness() {
        const cfgs = toolchain.opts.cfg?.jsonCfg?.lives;
        if (!cfgs) {
            console.error('no liveness configuration.');
            process.exit(1);
        }
        const outputFile = `tmp/${toolchain.opts.projectName}.jsonDead.txt`;
        await fs.ensureFile(outputFile);
        await fs.appendFile(outputFile, '-------------------------------\n' + moment().format(moment.HTML5_FMT.DATETIME_LOCAL_SECONDS) + '\n\n', 'utf-8');
        const jsonRoot = path.join(toolchain.opts.projectRoot, 'Assets/AssetSources/data');
        for (const c of cfgs) {
            const file = path.join(jsonRoot, c.file);
            const f = await fs.readJSON(file);
            const content = await fs.readFile(file, 'utf-8');
            let values = f.map((v) => v[c.key]);
            const total = values.length;
            for (const rk of c.reserveKeys) {
                values = values.filter((v) => !content.includes(`"${rk}": ${v}`));
            }
            const otherJsons = await fg('*.json', { cwd: jsonRoot, ignore: [c.file] });
            for (const oj of otherJsons) {
                const otherContent = await fs.readFile(path.join(jsonRoot, oj), 'utf-8');
                values = values.filter((v) => !otherContent.includes(String(v)));
                if (values.length == 0) {
                    break;
                }
            }
            if (values.length > 0) {
                const sumary = `${c.file}, total cnt: ${total}, dead cnt: ${values.length}`;
                console.log(sumary);
                await fs.appendFile(outputFile, sumary + '\n' + values.join('\n') + '\n\n');
            }
        }
    }
}
//# sourceMappingURL=JsonStriper.js.map