@summarisation/fileutils/dist/src/files.js

fileutils for summarisation

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.calculateSHA256 = exports.transformFilesIfShaChanged = exports.transformFiles = exports.markerIsShaInOldFile = exports.markerIsFieldInOldFile = exports.markerIsFromJsonInOldFile = exports.markerIsFromOldFile = exports.inputToOutputFileName = exports.addMetrics = exports.changeExtensionAddIndex = exports.changeExtension = exports.mapAsyncG = exports.getFilesRecursively = exports.getDirectoriesRecursively = void 0;
const crypto_1 = require("crypto");
const fs_1 = require("fs");
const path = __importStar(require("path"));
async function* getChildDirectories(dir, filterFn) {
    const dirEntries = await fs_1.promises.readdir(dir, { withFileTypes: true });
    for (const entry of dirEntries.sort()) {
        const fullPath = path.join(dir, entry.name);
        if (entry.isDirectory()) {
            if (!filterFn || filterFn(entry.name)) {
                yield fullPath;
            }
            yield* getChildDirectories(fullPath, filterFn);
        }
    }
}
async function* getDirectoriesRecursively(dir, filterFn) {
    yield dir;
    for await (const child of getChildDirectories(dir, filterFn))
        yield child;
}
exports.getDirectoriesRecursively = getDirectoriesRecursively;
async function* getFilesRecursively(dir, filterFn) {
    const dirEntries = await fs_1.promises.readdir(dir, { withFileTypes: true });
    for (const entry of dirEntries.sort()) {
        const fullPath = path.join(dir, entry.name);
        if (entry.isDirectory()) {
            yield* getFilesRecursively(fullPath, filterFn);
        }
        else if (entry.isFile()) {
            if (!filterFn || filterFn(entry.name)) {
                yield fullPath;
            }
        }
    }
}
exports.getFilesRecursively = getFilesRecursively;
async function mapAsyncG(gen, fn) {
    const res = [];
    for await (const t of gen)
        res.push(fn(t));
    return Promise.all(res);
}
exports.mapAsyncG = mapAsyncG;
const changeExtension = (newExt) => (s) => s.replace(/\.[^/.]+$/, newExt);
exports.changeExtension = changeExtension;
const changeExtensionAddIndex = (newExt) => (s, index) => s.replace(/\.[^/.]+$/, '.' + index + newExt);
exports.changeExtensionAddIndex = changeExtensionAddIndex;
function addMetrics(metrics) {
    return metrics.reduce((acc, m) => ({
        readCount: acc.readCount + m.readCount,
        writeCount: acc.writeCount + m.writeCount,
        failed: [...acc.failed, ...m.failed],
        markerErrors: [...acc.markerErrors, ...m.markerErrors]
    }), { readCount: 0, writeCount: 0, failed: [], markerErrors: [] });
}
exports.addMetrics = addMetrics;
const inputToOutputFileName = (inputDir, outputDir, config) => (file, index) => {
    const { newFileNameFn = (f, index) => f } = config;
    if (index === undefined)
        index = 0;
    let to = newFileNameFn(file, index);
    const relativePath = path.relative(inputDir, to);
    const outputFilePath = path.join(outputDir, relativePath);
    return outputFilePath;
};
exports.inputToOutputFileName = inputToOutputFileName;
const markerIsFromOldFile = (findMarker) => async (config, file) => {
    const { inputDir, outputDir } = config;
    const outputFilePath = (0, exports.inputToOutputFileName)(inputDir, outputDir, config)(file, 0);
    const oldOutput = await config.readFile(outputFilePath).catch(() => undefined);
    const marker = oldOutput === undefined ? undefined : findMarker(file, oldOutput);
    return marker;
};
exports.markerIsFromOldFile = markerIsFromOldFile;
const markerIsFromJsonInOldFile = (findMarker) => (0, exports.markerIsFromOldFile)((filename, content) => findMarker(JSON.parse(content)));
exports.markerIsFromJsonInOldFile = markerIsFromJsonInOldFile;
const markerIsFieldInOldFile = (field) => (0, exports.markerIsFromJsonInOldFile)((content) => content?.[field]);
exports.markerIsFieldInOldFile = markerIsFieldInOldFile;
exports.markerIsShaInOldFile = (0, exports.markerIsFieldInOldFile)('sha');
async function findMarker(config, metrics, file) {
    const { markerFn, debug } = config;
    try {
        return markerFn === undefined ? undefined : await markerFn(config, file);
    }
    catch (e) {
        if (debug)
            console.error(e);
        metrics.markerErrors.push(file);
    }
}
const transformOneFile = (config) => {
    const { fn, inputDir, readFile, outputDir, debug, dryRun, markerFn, newFileNameFn = (f => f) } = config;
    return async (file) => {
        const metrics = { readCount: 0, writeCount: 0, failed: [], markerErrors: [] };
        try {
            const content = await readFile(file);
            const marker = await findMarker(config, metrics, file);
            if (debug || dryRun)
                console.log('file', file, 'marker', marker);
            const newContent = await fn(content, marker, index => (0, exports.inputToOutputFileName)(inputDir, outputDir, config)(file, index));
            for (const { file, content } of newContent) {
                const newDir = path.dirname(file);
                await fs_1.promises.mkdir(newDir, { recursive: true });
                await fs_1.promises.writeFile(file, content);
                metrics.writeCount++;
            }
            metrics.readCount++;
        }
        catch (e) {
            if (e.name === 'AxiosError')
                console.error(`Failed`, file);
            else
                console.error(`Failed`, file, e);
            metrics.failed.push(file);
        }
        return metrics;
    };
};
const transformFiles = async (config) => addMetrics(await mapAsyncG(getFilesRecursively(config.inputDir, config.filter), transformOneFile(config)));
exports.transformFiles = transformFiles;
const transformFilesIfShaChanged = (config) => {
    const { fn, digest, getShaFromOutput, ...rest } = config;
    const newFn = async (content, marker, filenameFn) => {
        const thisSha = await digest?.(content);
        if (marker === thisSha)
            return [];
        return fn(content, thisSha, filenameFn);
    };
    return (0, exports.transformFiles)({ ...rest, fn: newFn });
};
exports.transformFilesIfShaChanged = transformFilesIfShaChanged;
const calculateSHA256 = async (s) => {
    const hashSum = (0, crypto_1.createHash)('sha256');
    hashSum.update(s);
    return hashSum.digest('base64url');
};
exports.calculateSHA256 = calculateSHA256;