ayakashi/lib/coreScripts/saveToCSV.js

The next generation web scraping framework

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const opLog_1 = require("../opLog/opLog");
const path_1 = require("path");
const os_1 = require("os");
const json2csv_1 = require("json2csv");
const fs_1 = require("fs");
const mkdirp_1 = __importDefault(require("mkdirp"));
const util_1 = require("util");
const exists = util_1.promisify(fs_1.exists);
const writeFile = util_1.promisify(fs_1.writeFile);
const open = util_1.promisify(fs_1.open);
const write = util_1.promisify(fs_1.write);
const close = util_1.promisify(fs_1.close);
const mkdirp = util_1.promisify(mkdirp_1.default);
function default_1(input, params, system) {
    return __awaiter(this, void 0, void 0, function* () {
        const opLog = opLog_1.getOpLog();
        let extraction;
        if (Array.isArray(input)) {
            extraction = input.filter(inp => !!inp);
        }
        else {
            extraction = [input].filter(inp => !!inp);
        }
        if (!extraction || extraction.length === 0) {
            opLog.warn("saveToCSV: nothing to print");
            opLog.warn("Learn more here: https://ayakashi-io.github.io/docs/guide/builtin-saving-scripts.html");
            return;
        }
        if (extraction.some(ext => typeof ext !== "object")) {
            opLog.warn("saveToCSV: invalid extraction format. Extracted data must be wrapped in an object");
            opLog.warn("Learn more here: https://ayakashi-io.github.io/docs/guide/builtin-saving-scripts.html");
            return;
        }
        const dataFolder = path_1.join(system.projectFolder, "data", system.startDate);
        yield mkdirp(dataFolder);
        const file = path_1.join(dataFolder, params.file || "data.csv");
        const csvConfig = {
            header: true,
            flatten: true
        };
        if (!(yield exists(file))) {
            yield writeFile(file, "");
        }
        else {
            csvConfig.header = false;
        }
        const parser = new json2csv_1.Parser(csvConfig);
        const csv = parser.parse(extraction);
        const fd = yield open(file, "a");
        const ap = Buffer.from(`${csv}${os_1.EOL}`);
        yield write(fd, ap, 0, ap.length);
        yield close(fd);
        return input;
    });
}
exports.default = default_1;