fy-convertor/lib/core/JsonOptimizer.js

Convert excel/xml/json/bin/protocl/ts/as ...

import fg from "fast-glob";
import fs from "fs-extra";
import path from "path";
import _ from "lodash";
/**
 * @deprecated 无内存优化效果，弃用
 */
export class JsonOptimizer {
    /**
     * 压缩JSON，合并相同字段
     * @param objs
     * @returns
     */
    static optimize(objs) {
        const om = JsonOptimizer.getMetaObject(objs);
        const data = [];
        for (const obj of objs) {
            data.push(JsonOptimizer.pruneObject(obj, om));
        }
        return { ...om, data };
    }
    /**
     * 还原JSON，通过proxy
     * @param oo
     * @returns
     */
    static restore(oo) {
        return this.restoreObject(oo.data, oo, true);
    }
    static getMetaObject(objs) {
        // 先收集公共key
        const keysArr = [];
        for (const o of objs) {
            const keys = Object.keys(o);
            keysArr.push(keys);
        }
        const commonKeys = _.intersection(...keysArr);
        // 统计所有字段的高频值
        const meta = {}, arrayFields = [], complexFields = {};
        for (const key of commonKeys) {
            let isArrayField = false, isComplex = false;
            if (objs[0][key] instanceof Array) {
                meta[key] = [];
                isArrayField = true;
                arrayFields.push(key);
                // 数组的话需要遍历才能确定是否复杂结构
                for (const o of objs) {
                    const value = o[key];
                    for (const sv of value) {
                        if (typeof (sv) == 'object') {
                            isComplex = true;
                            break;
                        }
                    }
                    if (isComplex)
                        break;
                }
            }
            else {
                isComplex = typeof (objs[0][key]) == 'object';
            }
            if (isComplex) {
                // 复杂结构抽取公共字段
                const values = [];
                for (const o of objs) {
                    const value = o[key];
                    if (value instanceof Array) {
                        values.push(...value);
                    }
                    else {
                        values.push(value);
                    }
                }
                const om = JsonOptimizer.getMetaObject(values);
                complexFields[key] = om;
            }
            const frequencyCnt = {};
            for (const o of objs) {
                const value = o[key];
                const valueKey = isArrayField || isComplex ? JSON.stringify(value) : value;
                let counter = frequencyCnt[valueKey];
                if (counter != null) {
                    counter.count += 1;
                }
                else {
                    frequencyCnt[valueKey] = counter = { value, count: 1 };
                }
            }
            let defaultCounter = null;
            for (const value in frequencyCnt) {
                const counter = frequencyCnt[value];
                if (defaultCounter == null || defaultCounter.count < counter.count) {
                    defaultCounter = counter;
                }
            }
            if (Object.keys(frequencyCnt).length < objs.length) {
                meta[key] = defaultCounter.value;
            }
            else {
                console.log('field not shared in common:', key);
            }
        }
        return { meta, arrayFields, complexFields };
    }
    static pruneObject(obj, om) {
        const o = {};
        for (const key in obj) {
            const value = obj[key];
            if (key in om.meta) {
                // 有默认值
                if (!_.isEqual(value, om.meta[key])) {
                    // 与默认值不同
                    const subMeta = om.complexFields?.[key];
                    if (subMeta != null) {
                        // 复杂结构进一步剥离字段
                        if (value instanceof Array) {
                            const newValue = [];
                            for (const sv of value) {
                                newValue.push(JsonOptimizer.pruneObject(sv, subMeta));
                            }
                            o[key] = newValue;
                        }
                        else {
                            o[key] = JsonOptimizer.pruneObject(value, subMeta);
                        }
                    }
                    else {
                        o[key] = value;
                    }
                }
            }
            else {
                // 没默认值
                o[key] = value;
            }
        }
        return o;
    }
    static restoreObject(o, patch, isArray) {
        if (isArray) {
            for (let i = o.length - 1; i >= 0; i--) {
                o[i] = JsonOptimizer.restoreObject(o[i], patch, false);
            }
            return o;
        }
        else {
            const proxy = new Proxy(o, {
                get(target, p, receiver) {
                    if (p in target) {
                        return target[p];
                    }
                    return patch.meta[p];
                }
            });
            if (patch.complexFields != null) {
                for (const k in patch.complexFields) {
                    o[k] = JsonOptimizer.restoreObject(proxy[k], patch.complexFields[k], Boolean(patch.arrayFields?.includes(k)));
                }
            }
            return proxy;
        }
    }
}
export async function test() {
    const root = 'D:/works/fy-convertor/temp/mu_mobile/json';
    const out = path.join(root, '../optimizedJson');
    await fs.ensureDir(out);
    const jsons = await fg('*.json', { cwd: root });
    // console.log(jsons.map((v)=>`'${v}'`).join(', '));
    for (const json of jsons) {
        const objs = await fs.readJSON(path.join(root, json));
        const om = JsonOptimizer.optimize(objs);
        await fs.writeJSON(path.join(out, json), om, { spaces: 2 });
    }
}
//# sourceMappingURL=JsonOptimizer.js.map