ayakashi
Version:
The next generation web scraping framework
194 lines (193 loc) • 9.25 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createQuery = void 0;
const opLog_1 = require("../../opLog/opLog");
const uuid_1 = require("uuid");
const debug_1 = __importDefault(require("debug"));
const d = debug_1.default("ayakashi:prelude:query");
function createQuery(ayakashiInstance, opts) {
const opLog = opLog_1.getOpLog();
const query = {};
let triggered = false;
const window = opts.window;
return {
$$prop: Symbol("AyakashiProp"),
id: (opts && opts.propId) || `prop_${uuid_1.v4()}`,
parent: [],
__trackMissingChildren: false,
where: function (q) {
query.where = q;
return this;
},
limit: function (n) {
query.limit = n;
return this;
},
skip: function (n) {
query.skip = n;
return this;
},
order: function (ord) {
if (ord === "asc" || ord === "desc") {
query.order = ord;
}
return this;
},
from: function (fromPropId) {
if (Array.isArray(fromPropId)) {
fromPropId.forEach((id) => {
const p = ayakashiInstance.prop(id);
if (!p) {
throw new Error(`Uknown parent prop : ${id}`);
}
this.parent.push(p);
});
}
else {
const p = ayakashiInstance.prop(fromPropId);
if (!p) {
throw new Error(`Uknown parent prop : ${fromPropId}`);
}
this.parent.push(p);
}
return this;
},
trigger: function (triggerOptions) {
return __awaiter(this, void 0, void 0, function* () {
//if it's not a force-trigger and prop not already triggered, just return the match count
if (!triggerOptions || triggerOptions.force !== true) {
if (triggered) {
const propMatches = yield ayakashiInstance.evaluate(function (scopedPropId) {
if (this.propTable[scopedPropId] &&
this.propTable[scopedPropId].matches) {
return this.propTable[scopedPropId].matches.length;
}
else {
return 0;
}
}, this.id);
if (propMatches === 0 &&
(!triggerOptions || (triggerOptions && triggerOptions.showNoMatchesWarning))) {
opLog.warn(`prop: ${this.id} has no matches`);
}
return propMatches;
}
}
triggered = true;
//trigger the parents first
if (this.parent && this.parent.length > 0) {
const parentsWithMatch = (yield Promise.all(this.parent.map(p => p.trigger()))).filter(c => c > 0);
if (parentsWithMatch.length === 0) {
opLog.warn(`prop: ${this.id} has no parent matches`);
return 0;
}
}
//cascade trackMissingChildren on child props
if (this.parent && this.parent.length > 0 && this.parent.find(p => p.__trackMissingChildren === true)) {
this.trackMissingChildren();
}
d(`triggering prop: ${this.id}`);
if (opts && opts.triggerFn) {
const propMatches = yield opts.triggerFn();
if (propMatches === 0 && (!triggerOptions || (triggerOptions && triggerOptions.showNoMatchesWarning))) {
opLog.warn(`prop: ${this.id} has no matches`);
}
return propMatches;
}
else {
const propMatches = yield ayakashiInstance.evaluate(function (scopedQuery, scopedId, scopedParentIds) {
let matches = [];
if (scopedParentIds.length > 0) {
scopedParentIds.forEach(({ parentId, trackMissingChildren }) => {
const parentMatches = this.propTable[parentId].matches;
if (parentMatches && parentMatches.length > 0) {
parentMatches.forEach((parentEl) => {
let childrenMatches = Array.from(this.preloaders.domQL.domQuery(scopedQuery, {
scope: parentEl,
env: window
}));
if (childrenMatches.length === 0 && trackMissingChildren) {
childrenMatches = [
window ?
window.document.createElement("void") :
document.createElement("void")
];
}
matches = matches.concat(childrenMatches);
});
}
});
}
else {
matches = Array.from(this.preloaders.domQL.domQuery(scopedQuery, { env: window }));
}
this.propTable[scopedId] = {
matches: matches
};
return matches.length;
}, query, this.id, this.parent.map(p => ({ parentId: p.id, trackMissingChildren: p.__trackMissingChildren })));
if (propMatches === 0 && (!triggerOptions || (triggerOptions && triggerOptions.showNoMatchesWarning))) {
opLog.warn(`prop: ${this.id} has no matches`);
}
return propMatches;
}
});
},
hasMatches: function () {
return __awaiter(this, void 0, void 0, function* () {
yield this.trigger({ force: true, showNoMatchesWarning: false });
const matches = yield ayakashiInstance.evaluate(function (scopedPropId) {
return (this.propTable[scopedPropId] &&
this.propTable[scopedPropId].matches &&
this.propTable[scopedPropId].matches.length) || 0;
}, this.id);
return matches > 0;
});
},
update: function () {
triggered = false;
return this;
},
selectChildren: function (childPropId) {
const childQuery = createQuery(ayakashiInstance, { propId: childPropId, window: window }).from(this);
ayakashiInstance.propRefs[childQuery.id] = childQuery;
d(`registering prop: ${childQuery.id}`);
return childQuery;
},
selectChild: function (childPropId) {
const childQuery = createQuery(ayakashiInstance, { propId: childPropId, window: window }).from(this);
childQuery.limit(1);
ayakashiInstance.propRefs[childQuery.id] = childQuery;
d(`registering prop: ${childQuery.id}`);
return childQuery;
},
selectFirstChild: function (childPropId) {
return this.selectChild(childPropId);
},
selectLastChild: function (childPropId) {
const childQuery = createQuery(ayakashiInstance, { propId: childPropId, window: window }).from(this);
childQuery.limit(1);
childQuery.order("desc");
ayakashiInstance.propRefs[childQuery.id] = childQuery;
d(`registering prop: ${childQuery.id}`);
return childQuery;
},
trackMissingChildren: function () {
this.__trackMissingChildren = true;
return this;
}
};
}
exports.createQuery = createQuery;