@4dsas/doc_preprocessing/lib/preprocessor.js

Preprocess 4D doc to be built by docusaurus

var fs = require('fs');
var path = require('path')
const { Instruction, TYPE } = require("./preprocessing/instruction.js")
const { REFManager } = require("./preprocessing/ref.js")
const { show, MESSAGE } = require('./log.js')
const settings = require("./settings.js")
const simpleGit = require("simple-git")

var chokidar = require('chokidar');

function trimFirstLineBreak(inText) {
    let text = ""
    let firstIndex = 0
    const lastIndex = inText.length
    if (inText.length > 0) {
        if (inText[0] === '\r') {
            firstIndex++
            if (inText[1] === '\n') {
                firstIndex++
            }
        }
        else if (inText[0] === '\n') {
            firstIndex++
        }
    }

    if (firstIndex < lastIndex) {
        text = inText.substr(firstIndex, lastIndex - firstIndex)
    }

    return text
}


function getFileList(dir, inExclusionList, callback, parent) {
    if (inExclusionList.includes(dir))
        return;

    const files = fs.readdirSync(dir);
    parent = parent || dir;
    files.forEach(function (file) {
        if (inExclusionList.includes(dir + file))
            return;

        if (fs.statSync(dir + file).isDirectory()) {
            callback((dir + file + '/').replace(parent, ""), false)
            getFileList(dir + file + '/', inExclusionList, callback, parent);
        }
        else {
            callback((dir + file).replace(parent, ""), true)
        }
    });
}

class Error {
    constructor(inType, inMessage) {
        this.message = inMessage
        this.type = inType
    }
}

class Preprocessing {
    constructor(inSettings, inRoot = "") {
        this._refManager = new REFManager(this.log.bind(this), 
                        inSettings.getArray(settings.SETTINGS_KEY.SYNTAX_ESCAPE_LIST))

        this._path = path.join(inRoot, inSettings.getString(settings.SETTINGS_KEY.PATH))
        this._configFile = path.join(inRoot, this._path + inSettings.getString(settings.SETTINGS_KEY.CONFIG));
        this._watch = inSettings.getBoolean(settings.SETTINGS_KEY.WATCH)
        this._destination = path.join(inRoot, inSettings.getString(settings.SETTINGS_KEY.OUTPUT))
        this._settings = inSettings;
        this._syntax_only = inSettings.getBoolean(settings.SETTINGS_KEY.SYNTAX_ONLY)
        this._dependencies = new Array()
        let excludeList = inSettings.getArray(settings.SETTINGS_KEY.EXCLUDE_LIST)
        if (excludeList) {
            this._excludeList = excludeList.map(x => (this._path + x).replaceAll("/", path.sep))
        }
        else {
            this._excludeList = []
        }
        this._include_escape_list = inSettings.getArray(settings.SETTINGS_KEY.INCLUDE_ESCAPE_LIST);
        this._errors = []
    }

    log(inType, inMessage) {
        if (this._settings.getBoolean(settings.SETTINGS_KEY.VERBOSE)) {
            show(inType, inMessage);
        }
        this._errors.push(new Error(inMessage, inType))
    }

    getErrors() {
        return this._errors
    }

    _collectFile(inCurrentPath, inUpdate) {
        return this._refManager.collectFile(inCurrentPath, inUpdate)
    }

    _IsMD(inExtension) {
        return inExtension === ".md" || inExtension === ".mdx";
    }

    collectDependencies() {
        return new Promise((r, reject) => {
            let dependencies = this._settings.get(settings.SETTINGS_KEY.DEPENDENCIES)

            let listPromises = []

            if (dependencies != null && dependencies.length > 0) {
                let buildingPath = this._settings.get(settings.SETTINGS_KEY.BUILD_DEPENDENCIES)

                dependencies.forEach(dependency => {

                    function processDependency(buildingPath, inSettings) {
                        return new Promise((resolve, reject) => {
                            const s = new settings.Settings()
                            s.load(inSettings.get(settings.SETTINGS_KEY.ARGS_DEPENDENCIES), buildingPath, inSettings)
                            let preprocessor = new Preprocessing(s, buildingPath)
                            preprocessor.collect().then(() => {
                                resolve(preprocessor)
                            })
                        })
                    }

                    //clone
                    async function fetch(dependency, inSettings) {

                        let url = dependency["url"]
                        let urls = url.split(':')
                        let name = dependency["name"]

                        if (urls[0] === "git") {
                            const gitPath = buildingPath + name
                            let gitUrl = urls.slice(1, urls.length).join(':')
                            let branch = dependency["branch"]
                            buildingPath = gitPath + path.posix.sep

                            if (!fs.existsSync(gitPath)) {
                                fs.mkdirSync(gitPath)
                                let git = simpleGit(gitPath)
                                await git.clone(gitUrl, "./", ["-b", branch, "--single-branch"])
                            }
                            else {
                                let git = simpleGit(gitPath)
                                const statusSummary = await git.status()
                                let isBehind = statusSummary.behind != 0
                                if (isBehind) {
                                    await git.pull()
                                }
                            }
                        }
                        else if (urls[0] === "file") {
                            buildingPath = urls.slice(1, urls.length).join(':')
                        }
                        return await processDependency(buildingPath, inSettings)
                    }

                    listPromises.push(fetch(dependency, this._settings))
                })

            }


            Promise.all(listPromises).then((values) => {
                values.forEach((v) => {
                    this._dependencies.push(v)
                })
                r()
            })
        })
    }

    collect() {
        return this.collectDependencies().then(() => {

            getFileList(this._path, this._excludeList, (file, isFile) => {
                if (isFile && file.length > 0) {
                    const currentPath = this._path + file
                    const extension = path.extname(file)
                    if (this._IsMD(extension)) {
                        this._collectFile(currentPath);
                    }
                }
            })
            let isFileExists = this._configFile.length > 0 && fs.existsSync(this._configFile) && fs.lstatSync(this._configFile).isFile();
            if (isFileExists) {
                this._collectFile(this._configFile);
            }
        })
    }

    getRef(inKeyworkd) {
        var value = this._refManager.getContentFromID(inKeyworkd)

        //go into dependencies
        if (value.found === false) {
            this._dependencies.some(d => {
                value = d.getRef(inKeyworkd)
                return value.found == true
            })
        }

        return value
    }

    _resolveInclude(inContent, inPath) {
        let re = /(<!--)(.*?)(-->)/g
        let match
        let startIndex = 0
        let currentContent = ""

        while ((match = re.exec(inContent)) != null) {
            currentContent += inContent.substr(startIndex, match.index - startIndex)
            let keywords = match[2].trim().split(" ")
            let isCommand = false;
            switch (keywords[0]) {
                case TYPE.INCLUDE:
                    if (keywords.length > 1) {
                        const keyword = Instruction.convertID2Args(keywords)
                        const value = this.getRef(keyword)
                        if (value.found === true && value.type === TYPE.REF) {
                            const lineEnding = match[4] != undefined ? match[4] : ""
                            let content = value.content;
                            if (this._include_escape_list) {
                                for (let escape of this._include_escape_list) {
                                    if (escape.from && escape.to)
                                        content = content.replaceAll(escape.from, escape.to);
                                }
                            }
                            let c = trimFirstLineBreak(content) + lineEnding
                            currentContent += this._resolveInclude(c, inPath)
                        }
                        else {
                            this.log(MESSAGE.WARNING, "The include \'" + keyword + "\' in path " + inPath + " has an invalid reference")
                        }
                        isCommand = true;
                    }
                    break;
                case TYPE.IREF:
                case TYPE.REF:
                    isCommand = true;
                    break;
                case TYPE.END:
                    if (keywords.length > 1 && keywords[1].trim() === TYPE.REF) {
                        isCommand = true;
                    }
                    break;
            }

            if (isCommand) {
                startIndex = match.index + match[0].length
            }
            else {
                startIndex = match.index
            }
        }
        currentContent += inContent.substr(startIndex, inContent.length - startIndex)

        return currentContent
    }

    resolve(inPath) {
        const content = fs.readFileSync(inPath, 'utf8');
        return this._resolveInclude(content, inPath, [])
    }

    write(inPath, newContent) {
        fs.mkdirSync(path.dirname(inPath), { recursive: true })
        fs.writeFileSync(inPath, newContent)
    }

    copyFile(inPath, toPath) {
        fs.mkdirSync(path.dirname(toPath), { recursive: true })
        fs.copyFileSync(inPath, toPath)
    }

    getSyntaxObject() {
        return this._refManager.formatToJSON('.', '#')
    }

    formatIndex() {
        return this._refManager.formatIndex('.', '#')
    }

    copy(inCompare = false) {

        getFileList(this._path, this._excludeList, (file, isFile) => {
            if (!isFile) return;
            const currentPath = this._path + file
            const destination = this._destination + file
            if (this._IsMD(path.extname(file))) {
                const content = this.resolve(currentPath)
                if (!inCompare || (inCompare && content !== fs.readFileSync(destination, 'utf-8'))) {
                    this.write(destination, content)
                }
            }
            else {
                try {
                    let statCurrentPath = fs.statSync(currentPath)
                    let statDestinationPath = fs.statSync(destination)

                    if (new Date(statDestinationPath.mtime).getTime() !== new Date(statCurrentPath.mtime).getTime()) {
                        this.copyFile(currentPath, destination)
                        fs.utimesSync(destination, statCurrentPath.atime, statCurrentPath.mtime)
                    }
                }
                catch (error) {
                    //Does not exist => Try to copy
                    this.copyFile(currentPath, destination)
                }
            }
        });
    }

    deleteOldFiles() {

        let itemsToDelete = []
        getFileList(this._destination, this._excludeList, (file, isFile) => {
            const currentPath = this._path + file
            const destination = this._destination + file
            if (isFile) {
                if (!fs.existsSync(currentPath)) {
                    fs.unlinkSync(destination)
                }
            }
            else {
                if (!fs.existsSync(currentPath)) {
                    const deleteFolderRecursive = function (inPath) {
                        if (fs.existsSync(inPath)) {
                            fs.readdirSync(inPath).forEach((file, index) => {
                                const curPath = path.join(inPath, file);
                                if (fs.lstatSync(curPath).isDirectory()) { // recurse
                                    deleteFolderRecursive(curPath);
                                } else { // delete file
                                    itemsToDelete.push(curPath)
                                }
                            });
                            itemsToDelete.push(inPath)
                        }
                    };
                    deleteFolderRecursive(destination)
                }
            }
        });
        itemsToDelete.forEach((value) => {
            let isDirectory = value.indexOf(path.sep) !== -1
            if (isDirectory) {
                fs.rmdirSync(value)
            }
            else {
                fs.unlinkSync(value)
            }
        });
    }


    runWatch() {
        function add(inPath, prepro) {

            const isMD = this._IsMD(path.extname(inPath)) //is markdown

            if (isMD) {
                prepro._collectFile(inPath)
                prepro.copy()
            }
            else {
                let newFilePath = inPath.replace(prepro._path, prepro._destination)
                prepro.copyFile(inPath, newFilePath)
            }
        }

        function change(inPath, prepro) {

            const isMD = this._IsMD(path.extname(inPath)) //is markdown
            if (isMD && prepro._collectFile(inPath, true)) //a REF has changed so an include, better copy all
            {
                prepro.copy(true)
            }
            else {
                let newFilePath = inPath.replace(prepro._path, prepro._destination)
                if (isMD) {
                    const content = prepro.resolve(inPath)
                    prepro.write(newFilePath, content)
                }
                else {
                    prepro.copyFile(inPath, newFilePath)
                }
            }
        }

        function unlink(inPath, prepro) {

            const isMD = this._IsMD(path.extname(inPath)) //is markdown
            let newFilePath = inPath.replace(prepro._path, prepro._destination)

            fs.unlinkSync(newFilePath)

            if (isMD) {
                prepro._collectFile(inPath)
                prepro.copy()
            }
        }


        var watcher = chokidar.watch(this._path, {
            persistent: true,
            ignoreInitial: true
        });
        watcher
            .on('add', inPath => {
                add(inPath, this)
            })
            .on('change', inPath => {
                change(inPath, this)
            })
            .on('all', (event, inPath) => {
                if (event === 'unlink')
                    unlink(inPath, this)
            })
        const pause = () => new Promise(res => setTimeout(res, 1000000));
        process.on('SIGINT', () => {
            watcher.close().then(() => console.log('Done!'));
            process.exit(1);
        });

        (async function () {

            while (true) {
                await pause();
            }
        })();

    }

    run() {
        return this.collect().then(() => {
            if (!this._syntax_only) {
                if (this._destination) {
                    this.copy()
                    this.deleteOldFiles()
                    if (this._watch) {
                        this.runWatch()
                    }
                }
            }
        })
    }
}

module.exports = { Preprocessing, Error }