ayakashi/lib/coreScripts/saveToJSON.js

The next generation web scraping framework

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const opLog_1 = require("../opLog/opLog");
const path_1 = require("path");
const fs_1 = require("fs");
const mkdirp_1 = __importDefault(require("mkdirp"));
const util_1 = require("util");
const exists = util_1.promisify(fs_1.exists);
const writeFile = util_1.promisify(fs_1.writeFile);
const open = util_1.promisify(fs_1.open);
const stat = util_1.promisify(fs_1.stat);
const write = util_1.promisify(fs_1.write);
const truncate = util_1.promisify(fs_1.truncate);
const close = util_1.promisify(fs_1.close);
const mkdirp = util_1.promisify(mkdirp_1.default);
function default_1(input, params, system) {
    return __awaiter(this, void 0, void 0, function* () {
        const opLog = opLog_1.getOpLog();
        let extraction;
        if (Array.isArray(input)) {
            extraction = input.filter(inp => !!inp);
        }
        else {
            extraction = [input].filter(inp => !!inp);
        }
        if (!extraction || extraction.length === 0) {
            opLog.warn("saveToJSON: nothing to print");
            opLog.warn("Learn more here: https://ayakashi-io.github.io/docs/guide/builtin-saving-scripts.html");
            return;
        }
        if (extraction.some(ext => typeof ext !== "object")) {
            opLog.warn("saveToJSON: invalid extraction format. Extracted data must be wrapped in an object");
            opLog.warn("Learn more here: https://ayakashi-io.github.io/docs/guide/builtin-saving-scripts.html");
            return;
        }
        const dataFolder = path_1.join(system.projectFolder, "data", system.startDate);
        yield mkdirp(dataFolder);
        const file = path_1.join(dataFolder, params.file || "data.json");
        let firstEntry = false;
        if (!(yield exists(file))) {
            yield writeFile(file, "[]");
            firstEntry = true;
        }
        const st = yield stat(file);
        yield truncate(file, st.size - 1);
        const fd = yield open(file, "a");
        let data = JSON.stringify(extraction, null, 4);
        data = data.substr(1, data.length);
        const ap = firstEntry ? Buffer.from(`\n${data}`) : Buffer.from(`,\n${data}`);
        yield write(fd, ap, 0, ap.length);
        yield close(fd);
        return input;
    });
}
exports.default = default_1;