app-lib-mock-server-stream/src/scan.js

流媒体文件-【app-lib-mock-server】

/*************************************************************************************
 * 
 *       扫描磁盘操作                        
 * 
 *  1.  
 * 
 * 
 *
 **************************************************************************************/

const Progress = require("app-lib-progress");
const { STREAM_NAMESPACE_NAME, STREAM_COMPLETE_PATH } = require('./constant');
const fs = require("fs");
const path = require("path");





const innerScan = ({ paths, parentId, scanId, process, config, level = 0 }) => {
    let relativePath = paths.slice(1);
    let relativePathStr = path.join(...relativePath);
    let realPath = path.join(...(paths.map((p, index) => index ? `/${p}` : p))); // 兼容 .xx 是文件的情况
    let stats = fs.statSync(realPath);
    let id = Frame.option.utils.autoId();
    let { name, ext } = path.parse(realPath);
    let { exclude } = config;

    // 排除不需要扫描的内容
    let isExclude = (exclude || []).filter(exp => {
        let res = false;
        try {
            res = new RegExp(exp).test(realPath);
        } catch (e) {
            res = realPath.includes(exp)
            process.log(Frame.log.mwn, 'stream exclude config is not reg string', exp);
        }
        if (res) {
            process.log(Frame.log.md, "exclude", relativePathStr);
        }
        return res;
    }).length;

    if (isExclude) return []; // 排除的内容不做处理


    const record = {
        id,
        parentId: parentId || Frame.option.utils.autoId(), // 根目录的时候  直接使用服务中的math地址
        name,
        displayName: name + ext,
        isFile: stats.isFile(),
        size: stats.size,
        ext: ext,
        level: level,
        // accesstime: + stats.atime, // The timestamp indicating the last time this file was accessed.
        updateTime: + stats.mtime,// The timestamp indicating the last time this file was modified.
        // createTime: + stats.birthtime, // The timestamp indicating the creation time of this file.
        changeTime: + stats.ctime, // The timestamp indicating the last time the file status was changed.
        path: relativePath.slice(0,-1), // 相对路径 避免路径太长 数组便于快速检索和计算文件夹
        // path: relativePath.pop() && relativePath.join('/'),
        // paths: relativePath,
        scanId,
    }

    let records = [record];

    process.count(1, { ...records[0], path: relativePathStr })

    if (stats.isDirectory()) {
        let children = fs.readdirSync(realPath);
        let childIds = [];
        children.forEach(childPath => {
            let childrenInstances = innerScan({
                paths: paths.concat(`${childPath}`),
                parentId: id,
                scanId,
                process,
                config,
                level: level + 1,
            });
            records = records.concat(childrenInstances);
            childIds.push(childrenInstances[0]?.id);
        })
        records[0].children = childIds;
    }

    return records;
}


/**
 * 扫描单个配置
 *    1. 所有的文件ID为路径
 *    2. 所有的名称为文件或者文件夹名称
 *    
 * 
 * @param {object} item  扫描的配置对象
 * @returns {array} 扫描结果
 */
const scanItem = (item) => {
    let { node: { getRootPath } } = Frame.option;
    let { diskPath, storageDatabaseName, displayName, storageTableName, id } = item;

    if (!diskPath) {
        log.mwn(`STREAM`, `need config diskPath in [${displayName}]`)
        return [];
    }

    let path = getRootPath(diskPath);


    if (!storageDatabaseName) {
        log.mwn(`STREAM`, `need config storageDatabaseName in [${displayName}]`)
        return [];
    }
    if (!storageTableName) {
        log.mwn(`STREAM`, `need config storageTableName in [${displayName}] `)
        return [];
    }
    log.mi('begain scan stream:', path);
    let beginTime = + new Date();

    let process = new Progress({
        output: ` :flag  [:current] :path`,
        cusField: ['path']
    })
    // 跟节点的id为0
    let scanData = innerScan({ paths: [path], parentId: '0', scanId: id, process, config: item });

    process.complate();

    let endTime = + new Date();

    // Frame.log.T(scanData);

    // 建立索引表 TODO 后面在处理

    // 比较数据--保留原来的数据 数据抄作直接操作对应的文件

    let resourcePath = `${STREAM_NAMESPACE_NAME}.${storageDatabaseName}.${storageTableName}`;

    // 记录到库 路径和文件
    Frame.createData(resourcePath, scanData, false, true)
    log.ms('finish scan stream:', `total:[${scanData.length}]`, path);

    // 设置流配置isInit为true-- 避免下一次再次扫描
    Frame.upadteData(STREAM_COMPLETE_PATH, { ...item, isInit: true });

    return { useTime: endTime - beginTime, scanData, item };
}



/**
 *  扫描所有配置
 * @param {array} streamData 配置数据
 * @returns {array} 扫描后的数据
 */
const scanDisk = (streamData) => streamData.map(scanItem);


getFilePath = (basePath, dataItem) => {
    // let paths = dataItem.path.slice();
    // let fileName = paths.pop();
    let relativePath = dataItem.path.join('/')
    return {
        root: path.join(basePath,relativePath),
        fileName: dataItem.displayName,
        complatePath: path.join(basePath, relativePath, dataItem.displayName)
    }
}



module.exports = {
    scanDisk,
    getFilePath
}