ayakashi/lib/store/project.js

The next generation web scraping framework

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.configChanged = exports.getLastConfig = exports.saveLastConfig = exports.getPipeprocFolder = exports.clearPreviousRun = exports.hasPreviousRun = exports.getOrCreateStoreProjectFolder = void 0;
const mkdirp_1 = __importDefault(require("mkdirp"));
const crypto_1 = require("crypto");
const path_1 = require("path");
const store_1 = require("./store");
const fs_1 = require("fs");
const rimraf_1 = __importDefault(require("rimraf"));
const lodash_1 = require("lodash");
function getOrCreateStoreProjectFolder(projectFolderOrScraperName, sessionKey) {
    return __awaiter(this, void 0, void 0, function* () {
        const storeDir = yield store_1.getStoreDir();
        const folderName = crypto_1.createHash("md5").update(projectFolderOrScraperName).digest("hex");
        const sessionFolder = crypto_1.createHash("md5").update(sessionKey).digest("hex");
        const fullFolder = path_1.resolve(storeDir, "projects", folderName, sessionFolder);
        return new Promise(function (resolve, reject) {
            mkdirp_1.default(fullFolder, function (err) {
                if (err) {
                    reject(err);
                }
                else {
                    resolve(fullFolder);
                }
            });
        });
    });
}
exports.getOrCreateStoreProjectFolder = getOrCreateStoreProjectFolder;
function hasPreviousRun(projectFolderOrScraperName) {
    return new Promise(function (resolve) {
        fs_1.exists(getPipeprocFolder(projectFolderOrScraperName), function (ex) {
            resolve(ex);
        });
    });
}
exports.hasPreviousRun = hasPreviousRun;
function clearPreviousRun(projectFolderOrScraperName) {
    return new Promise(function (resolve) {
        rimraf_1.default(projectFolderOrScraperName, function (_err) {
            resolve();
        });
    });
}
exports.clearPreviousRun = clearPreviousRun;
function getPipeprocFolder(projectFolderOrScraperName) {
    return path_1.join(projectFolderOrScraperName, "pipeproc");
}
exports.getPipeprocFolder = getPipeprocFolder;
function saveLastConfig(config, storeProjectFolder) {
    return __awaiter(this, void 0, void 0, function* () {
        return new Promise(function (resolve, reject) {
            fs_1.writeFile(path_1.resolve(storeProjectFolder, "lastConfig.json"), JSON.stringify(config), "utf8", function (err) {
                if (err) {
                    reject(err);
                }
                else {
                    resolve();
                }
            });
        });
    });
}
exports.saveLastConfig = saveLastConfig;
function getLastConfig(storeProjectFolder) {
    return __awaiter(this, void 0, void 0, function* () {
        return new Promise(function (resolve, reject) {
            fs_1.readFile(path_1.resolve(storeProjectFolder, "lastConfig.json"), "utf8", function (err, content) {
                if (err) {
                    reject(err);
                }
                else {
                    resolve(content);
                }
            });
        });
    });
}
exports.getLastConfig = getLastConfig;
function configChanged(config, storeProjectFolder) {
    return __awaiter(this, void 0, void 0, function* () {
        const lastConfig = JSON.parse(yield getLastConfig(storeProjectFolder));
        return [
            !lodash_1.isEqual(lastConfig, JSON.parse(JSON.stringify(config))),
            lastConfig
        ];
    });
}
exports.configChanged = configChanged;