ayakashi
Version:
The next generation web scraping framework
65 lines (64 loc) • 2.11 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.prepareSimple = void 0;
const opLog_1 = require("../opLog/opLog");
const getDirectory_1 = require("./getDirectory");
const path_1 = require("path");
function prepareSimple(file, out, scraperType) {
const opLog = opLog_1.getOpLog();
if (file === ".") {
opLog.error("Simple mode requires a scraper file as input");
return process.exit(1);
}
const splittedDir = file.split(path_1.sep);
const scraper = splittedDir.pop();
const directory = getDirectory_1.getDirectory(splittedDir.join("/"));
process.chdir(directory);
opLog.info("running scraper in simple mode");
opLog.info("directory:", directory);
opLog.info("scraper:", scraper);
let saveFile;
let saveScript;
if (out === "sqlite") {
saveFile = "data.sqlite";
saveScript = "saveToSQL";
opLog.info("data will be saved to a sqlite database");
}
else if (out === "csv") {
saveFile = "data.csv";
saveScript = "saveToCSV";
opLog.info("data will be saved to a csv file");
}
else if (out === "json") {
saveFile = "data.json";
saveScript = "saveToJSON";
opLog.info("data will be saved to a json file");
}
else {
saveFile = "";
saveScript = "printToConsole";
opLog.info("data will be printed to console, specify --out=sqlite|json|csv to change the format");
}
return {
config: {
waterfall: [{
type: scraperType,
module: scraper,
config: {
localAutoLoad: false,
simple: true
}
}, {
type: "script",
module: saveScript,
params: {
file: saveFile,
database: saveFile
}
}]
},
directory: directory,
scraper: scraper
};
}
exports.prepareSimple = prepareSimple;