ayakashi
Version:
The next generation web scraping framework
148 lines (147 loc) • 6.21 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const sequelize_1 = __importDefault(require("sequelize"));
const crypto_1 = require("crypto");
const path_1 = require("path");
const opLog_1 = require("../opLog/opLog");
const mkdirp_1 = __importDefault(require("mkdirp"));
const util_1 = require("util");
const mkdirp = util_1.promisify(mkdirp_1.default);
function default_1(input, params, system) {
return __awaiter(this, void 0, void 0, function* () {
const opLog = opLog_1.getOpLog();
let extraction;
if (Array.isArray(input)) {
extraction = input.filter(inp => !!inp);
}
else {
extraction = [input].filter(inp => !!inp);
}
if (!extraction || extraction.length === 0) {
opLog.warn("saveToSQL: nothing to print");
opLog.warn("Learn more here: https://ayakashi-io.github.io/docs/guide/builtin-saving-scripts.html");
return;
}
if (extraction.some(ext => typeof ext !== "object")) {
opLog.warn("saveToSQL: invalid extraction format. Extracted data must be wrapped in an object");
opLog.warn("Learn more here: https://ayakashi-io.github.io/docs/guide/builtin-saving-scripts.html");
return;
}
let useDialect;
let useStorage = "";
if (!params.dialect) {
useDialect = "sqlite";
}
else {
if (["mysql", "mariadb", "postgres", "mssql", "sqlite"].indexOf(params.dialect) === -1) {
throw new Error(`Invalid sql dialect: ${params.dialect}`);
}
else {
useDialect = params.dialect;
}
}
if (useDialect === "sqlite") {
const dataFolder = path_1.join(system.projectFolder, "data", system.startDate);
yield mkdirp(dataFolder);
useStorage = path_1.join(dataFolder, params.database || "database.sqlite");
}
let sequelize;
const hash = crypto_1.createHash("md5");
hash.update(JSON.stringify(params));
const hashedParams = hash.digest("hex");
//@ts-ignore
if (global[`dbconnection-${system.operationId}-${hashedParams}`]) {
//@ts-ignore
sequelize = global[`dbconnection-${system.operationId}-${hashedParams}`];
}
else {
if (useStorage) {
//@ts-ignore
sequelize = new sequelize_1.default({
dialect: "sqlite",
storage: useStorage,
logging: false
});
//@ts-ignore
global[`dbconnection-${system.operationId}-${hashedParams}`] = sequelize;
}
else if (params.connectionURI) {
//@ts-ignore
sequelize = new sequelize_1.default(params.connectionURI, {
pool: {
max: 1,
min: 1
},
logging: false
});
//@ts-ignore
global[`dbconnection-${system.operationId}-${hashedParams}`] = sequelize;
}
else {
//@ts-ignore
sequelize = new sequelize_1.default(params.database, params.username, params.password, {
dialect: params.dialect,
host: params.host || "localhost",
port: params.port,
pool: {
max: 1,
min: 1
},
logging: false
});
//@ts-ignore
global[`dbconnection-${system.operationId}-${hashedParams}`] = sequelize;
}
}
const tableSchema = {};
Object.entries(extraction[0]).forEach(function ([col, row]) {
if (typeof row === "string") {
tableSchema[col] = sequelize_1.default.TEXT;
}
else if (typeof row === "boolean") {
tableSchema[col] = sequelize_1.default.BOOLEAN;
}
else if (typeof row === "number" && Number.isInteger(row)) {
tableSchema[col] = sequelize_1.default.INTEGER;
}
else if (typeof row === "number" && !Number.isInteger(row)) {
tableSchema[col] = sequelize_1.default.FLOAT;
}
else if (typeof row === "object" && row !== null) {
if (useDialect === "sqlite" || useDialect === "postgres") {
tableSchema[col] = sequelize_1.default.JSON;
}
else {
extraction.forEach(function (e) {
e[col] = JSON.stringify(e[col]);
});
tableSchema[col] = sequelize_1.default.TEXT;
}
}
else {
tableSchema[col] = sequelize_1.default.STRING;
}
});
const table = sequelize.define(params.table || system.operationId, tableSchema, {
timestamps: true,
updatedAt: false,
freezeTableName: true
});
yield table.sync();
yield table.bulkCreate(extraction);
return input;
});
}
exports.default = default_1;