@visactor/vmind/esm/atom/dataQuery/utils.js

<div align="center"> <a href="https://github.com/VisActor#gh-light-mode-only" target="_blank"> <img alt="VisActor Logo" width="200" src="https://github.com/VisActor/.github/blob/main/profile/logo_500_200_light.svg"/> </a> <a href="https://githu

import JSON5 from "json5";

import { isArray, isString, uniqArray } from "@visactor/vutils";

import { capitalize, replaceAll } from "../../utils/text";

import { alasqlKeywordList } from "./const";

import { ROLE } from "../../types";

import alasql from "alasql";

import { detectFieldType } from "../../utils/field";

export function generateRandomString(len) {
    const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
    let result = "";
    for (let i = 0; i < len; i++) result += chars.charAt(Math.floor(52 * Math.random()));
    return result;
}

const operatorList = [ [ "+", `_${generateRandomString(3)}_PLUS_${generateRandomString(3)}_` ], [ "-", `_${generateRandomString(3)}_DASH_${generateRandomString(3)}_` ], [ "*", `_${generateRandomString(3)}_ASTERISK_${generateRandomString(3)}_` ], [ "/", `_${generateRandomString(3)}_SLASH_${generateRandomString(3)}_` ] ], operators = operatorList.map((op => op[0])), RESERVE_REPLACE_MAP = new Map([ ...operatorList, ...alasqlKeywordList.map((keyword => [ keyword, generateRandomString(10) ])) ]);

export const parseSQLResponse = response => {
    const sql = response.match(/sql:\n?```(.*?)```/s)[1], fieldInfoStr = response.match(/fieldInfo:\n?```(.*?)```/s)[1];
    let fieldInfo = [];
    try {
        const tempFieldInfo = JSON5.parse(fieldInfoStr);
        fieldInfo = isArray(tempFieldInfo) ? tempFieldInfo : tempFieldInfo.fieldInfo;
    } catch (e) {
        fieldInfo = JSON5.parse(`[${fieldInfoStr}]`);
    }
    return {
        sql: sql,
        llmFieldInfo: fieldInfo,
        thoughts: ""
    };
};

export function removeEmptyLines(str) {
    return str.replace(/\n\s*\n/g, "\n");
}

export const swapMap = map => {
    const swappedMap = new Map;
    return map.forEach(((value, key) => {
        swappedMap.set(value, key);
    })), swappedMap;
};

export const replaceByMap = (str, replaceMap) => [ ...replaceMap.keys() ].reduce(((prev, cur) => {
    const originColumnName = cur, validColumnName = replaceMap.get(cur);
    return replaceAll(prev, originColumnName, validColumnName);
}), str);

const replaceNonASCIICharacters = str => {
    const nonAsciiCharMap = new Map;
    return {
        validStr: str.replace(/([^\x00-\x7F]+)/g, (m => {
            let replacement;
            return nonAsciiCharMap.has(m) ? replacement = nonAsciiCharMap.get(m) : (replacement = generateRandomString(10), 
            nonAsciiCharMap.set(m, replacement)), replacement;
        })),
        replaceMap: nonAsciiCharMap
    };
};

export const replaceString = (str, replaceMap) => {
    if (!isString(str)) return str;
    if (replaceMap.has(str)) return replaceMap.get(str);
    return [ ...replaceMap.keys() ].sort(((a, b) => b.length - a.length)).reduce(((prev, cur) => replaceAll(prev, cur, replaceMap.get(cur))), str);
};

export const replaceDataset = (dataset, replaceMap, keysOnly) => dataset.map((d => Object.keys(d).reduce(((prev, cur) => {
    const replacedKey = replaceString(cur, replaceMap), replacedValue = replaceString(d[cur], replaceMap);
    return prev[replacedKey] = keysOnly ? d[cur] : replacedValue, prev;
}), {})));

export const getValueByAttributeName = (obj, outterKey) => {
    const values = [];
    for (const key in obj) if (key === outterKey && "string" == typeof obj[key]) values.push(obj[key]); else if ("object" == typeof obj[key]) {
        const childValues = getValueByAttributeName(obj[key], outterKey);
        values.push(...childValues);
    }
    return uniqArray(values);
};

export const replaceInvalidWords = (sql, columns) => {
    const operatorReplaceMap = new Map, validColumnNames = columns.map((column => [ ...RESERVE_REPLACE_MAP.keys() ].reduce(((prev, cur) => {
        const replaceStr = [ cur.toUpperCase(), cur.toLowerCase(), capitalize(cur) ].find((str => operators.includes(cur) ? prev.includes(str) : prev === str));
        return replaceStr ? (operatorReplaceMap.has(replaceStr) || operatorReplaceMap.set(replaceStr, RESERVE_REPLACE_MAP.get(cur)), 
        replaceAll(prev, replaceStr, RESERVE_REPLACE_MAP.get(cur))) : prev;
    }), column))), columnReplaceMap = new Map(columns.map(((column, index) => {
        const validStr = validColumnNames[index];
        if (column !== validStr) return [ column, validStr ];
    })).filter(Boolean)), sqlWithoutOperator = (str = sql, [ ...(replaceMap = columnReplaceMap).keys() ].reduce(((prev, cur) => {
        const originColumnName = cur, validColumnName = replaceMap.get(cur);
        return replaceAll(prev, originColumnName, validColumnName);
    }), str));
    var str, replaceMap;
    const {validStr: sqlWithoutAscii, replaceMap: asciiReplaceMap} = replaceNonASCIICharacters(sqlWithoutOperator);
    return {
        validStr: sqlWithoutAscii,
        columnReplaceMap: operatorReplaceMap,
        sqlReplaceMap: asciiReplaceMap
    };
};

export const mergeMap = (map1, map2) => (map2.forEach(((value, key) => {
    map1.set(key, value);
})), map1);

export const matchColumnName = (columnName, fieldName) => {
    const fieldWithoutSpace = fieldName.replace(/\s/g, "");
    return columnName.replace(/\s/g, "") === fieldWithoutSpace;
};

export const replaceBlankSpace = (sql, fieldNames) => {
    const ast = alasql.parse(sql), columnsInSql = getValueByAttributeName(ast.statements[0], "columnid"), validColumnNames = columnsInSql.map((column => {
        const matchedFieldName = fieldNames.find((field => matchColumnName(column, field)));
        return null != matchedFieldName ? matchedFieldName : column;
    }));
    return columnsInSql.reduce(((prev, _cur, index) => {
        const originColumnName = columnsInSql[index], validColumnName = validColumnNames[index];
        return validColumnName !== originColumnName ? replaceAll(prev, originColumnName, validColumnName) : prev;
    }), sql);
};

export const sumAllMeasureFields = (sql, fieldInfo, columnReplaceMap, sqlReplaceMap) => {
    var _a;
    const measureFieldsInSql = fieldInfo.filter((field => field.role === ROLE.MEASURE)).map((field => {
        const {fieldName: fieldName} = field, replacedName1 = replaceString(fieldName, columnReplaceMap);
        return replaceString(replacedName1, sqlReplaceMap);
    })), ast = alasql.parse(sql), selectedColumns = ast.statements[0].columns, nonAggregatedColumns = selectedColumns.filter((column => !column.aggregatorid)).map((column => column.columnid)), groupByColumns = (null !== (_a = ast.statements[0].group) && void 0 !== _a ? _a : []).map((column => column.columnid));
    let needAggregateColumns = [];
    groupByColumns.length > 0 && nonAggregatedColumns.length !== selectedColumns.length && (needAggregateColumns = nonAggregatedColumns.filter((column => measureFieldsInSql.includes(column))).filter((column => !groupByColumns.includes(column))));
    const patchedFields = needAggregateColumns.map((column => `SUM(\`${column}\`) as ${column}`));
    return needAggregateColumns.reduce(((prev, cur, index) => {
        const regex = new RegExp(`\`?${cur}\`?`, "g");
        return prev.replace(regex, patchedFields[index]);
    }), sql);
};

export const convertGroupByToString = (sql, dataset) => {
    const groupByColumns = alasql.parse(sql).statements[0].group.map((column => column.columnid));
    dataset.forEach((item => {
        groupByColumns.forEach((column => {
            item[column] = item[column].toString();
        }));
    }));
};

export const parseRespondField = (responseFieldInfo, dataset) => responseFieldInfo.map((field => Object.assign(Object.assign({}, field), detectFieldType(dataset, field.fieldName))));
//# sourceMappingURL=utils.js.map