syphonx-core
Version:
SyphonX is a template-driven solution for extracting data from HTML in a highly efficient way. It combines the power of jQuery, Regular Expressions, and Javascript into a declarative template-driven format that extracts and reshapes HTML data into JSON.
1,241 lines (1,215 loc) • 114 kB
JavaScript
var syphonx = (function (exports) {
'use strict';
const errorCodeMessageMap = {
"app-error": "An application defined error occured.",
"click-timeout": "Timeout waiting for click result.",
"click-required": "Could not find click target on the page.",
"error-limit": "Error limit exceeded.",
"eval-error": "Error evaluating formula.",
"fatal-error": "An unexpected error occurred.",
"invalid-select": "Invalid selector query found.",
"invalid-operator": "Invalid jQuery formula.",
"invalid-operand": "Invalid argument to a jQuery formula.",
"select-required": "Could not find target of a required selector on the page.",
"waitfor-timeout": "Timeout waiting for selector."
};
function parseBoolean(value) {
if (typeof value === "boolean")
return value;
else if (typeof value === "string")
return value !== "" && value.trim().toLowerCase() !== "false" && value.trim() !== "0";
else
return undefined;
}
function parseNumber(value) {
if (typeof value === "number") {
return !isNaN(value) ? value : undefined;
}
if (typeof value === "string") {
let [, text] = /([0-9.,]+)/.exec(value) || [];
if (/\.\d+$/.test(text))
text = text.replace(/,/g, "");
if (/,\d+$/.test(text))
text = text.replace(/\./g, "");
const result = parseFloat(text);
return !isNaN(result) ? result : undefined;
}
return undefined;
}
function parseUrl(url) {
if (typeof url === "string" && /^https?:\/\//.test(url)) {
const [protocol, , host] = url.split("/");
const a = host.split(":")[0].split(".").reverse();
return {
domain: a.length >= 3 && a[0].length === 2 && a[1].length === 2 ? `${a[2]}.${a[1]}.${a[0]}` : a.length >= 2 ? `${a[1]}.${a[0]}` : undefined,
origin: protocol && host ? `${protocol}//${host}` : undefined
};
}
return {};
}
function coerce(value, type) {
if (type === "number")
return parseNumber(value);
else if (type === "boolean")
return parseBoolean(value);
else
return value;
}
function coerceSelectValue(value, type, repeated) {
if (repeated) {
return value instanceof Array ? value.map(v => coerceSelectValue(v, type, false)) : [coerceSelectValue(value, type, false)];
}
else if (type === "string") {
return typeof value === "string" ? value : typeof value === "number" || typeof value === "boolean" ? value.toString() : null;
}
else if (type === "number") {
return typeof value === "number" ? value : typeof value === "string" ? parseNumber(value) : null;
}
else if (type === "boolean") {
return typeof value === "boolean" ? value : typeof value === "string" ? value.trim().length > 0 : typeof value === "number" && !isNaN(value) ? value !== 0 : null;
}
else {
return value;
}
}
function isCoercibleTo(value, type) {
if (type === "number" && typeof value === "string" && parseNumber(value) !== undefined)
return true;
else if (type === "boolean" && typeof value === "string" && parseBoolean(value) !== undefined)
return true;
else
return false;
}
function collapseWhitespace(text, newlines = true) {
if (typeof text === "string" && text.trim().length === 0) {
return null;
}
else if (typeof text === "string" && newlines) {
return text
.replace(/\s*\n\s*/gm, "\n")
.replace(/\n{2,}/gm, "\n")
.replace(/\s{2,}/gm, " ")
.trim();
}
else if (typeof text === "string" && !newlines) {
return text
.replace(/\n/gm, " ")
.replace(/\s{2,}/gm, " ")
.trim();
}
else {
return text;
}
}
function combineUrl(base, relative) {
if (!relative)
return base;
if (!base)
return relative;
return new URL(relative, base).toString();
}
function cut(text, splitter, n, limit) {
if (typeof text === "string") {
const a = text
.split(splitter, limit)
.map(value => value.trim())
.filter(value => value.length > 0);
const i = n >= 0 ? n : a.length + n;
return i >= 0 && i < a.length ? a[i] : null;
}
else {
return null;
}
}
function filterQueryResult($, result, predicate) {
if (result.value instanceof Array) {
const input = {
elements: result.nodes.toArray(),
values: result.value
};
const output = {
elements: [],
values: []
};
const n = input.elements.length;
for (let i = 0; i < n; ++i) {
const hit = predicate(input.values[i], i, input.values);
if (hit) {
output.elements.push(input.elements[i]);
output.values.push(input.values[i]);
}
}
result.nodes = $(output.elements);
result.value = output.values;
}
}
function formatHTML(value) {
if (typeof value === "string") {
return value
.replace(/(<[a-z0-9:._-]+>)[ ]*/gi, "$1")
.replace(/[ ]*<\//g, "</")
.trim();
}
else if (value instanceof Array && value.every(obj => typeof obj === "string")) {
return value.map(obj => formatHTML(obj));
}
else {
return value;
}
}
function isAbsoluteUrl(url) {
return url.startsWith("http://") || url.startsWith("https://");
}
function isEmpty(obj) {
if (obj === undefined || obj === null) {
return true;
}
else if (obj instanceof Array) {
return obj.length === 0;
}
else if (typeof obj === "string") {
return obj.length === 0;
}
else {
return false;
}
}
function isFormula(value) {
return typeof value === "string" && value.startsWith("{") && value.endsWith("}");
}
function isRegexp(value) {
return typeof value === "string" && (value.startsWith("/") || value.startsWith("!/"));
}
function isInvocableFrom(obj, method) {
return obj !== null && typeof obj === "object" && typeof obj[method] === "function";
}
function isJQueryObject(obj) {
return typeof obj === "object" && obj !== null && (!!obj.jquery || !!obj.cheerio);
}
function isObject$1(obj) {
return typeof obj === "object" && obj !== null && !(obj instanceof Array) && !(obj instanceof Date);
}
function isNullOrUndefined(obj) {
return obj === null || obj === undefined;
}
function formatStringValue(value, format, origin) {
if (format === "href" && typeof value === "string" && origin && !isAbsoluteUrl(value)) {
return combineUrl(origin, value);
}
else if (format === "multiline") {
return collapseWhitespace(value, true);
}
else if (format === "singleline") {
return collapseWhitespace(value, false);
}
else if (format === "none") {
return value;
}
else {
return value;
}
}
function evaluateFormula(formula, scope = {}) {
const keys = Object.keys(scope);
const values = keys.map(key => scope[key]);
const fn = new Function(...keys, `return ${formula}`);
const result = fn(...values);
return result;
}
function tryParseJson(value) {
try {
return JSON.parse(value);
}
catch (err) {
return undefined;
}
}
function merge(source, target) {
if (source instanceof Array && target instanceof Array) {
return [...source, ...target];
}
else if (isObject$1(source) && isObject$1(target)) {
const obj = {};
const keys = Array.from(new Set([...Object.keys(source), ...Object.keys(target)]));
for (const key of keys) {
obj[key] = merge(source[key], target[key]);
}
return obj;
}
else if (target) {
return target;
}
else {
return source;
}
}
function mergeElements(source, target) {
for (const targetAttr of Array.from(target[0].attributes)) {
const sourceAttr = Array.from(source[0].attributes).find(attr => attr.name === targetAttr.name);
if (sourceAttr && targetAttr.name === "class") {
const value = Array.from(new Set([
...sourceAttr.value.split(" "),
...targetAttr.value.split(" ")
])).join(" ");
source.attr("class", value);
}
else if (!sourceAttr) {
source.attr(targetAttr.name, targetAttr.value);
}
}
}
function createRegExp(value) {
if (typeof value === "string" && value.startsWith("/")) {
const i = value.lastIndexOf("/");
const pattern = value.substring(1, i);
const flags = value.length > i ? value.substring(i + 1) : "m";
return new RegExp(pattern, flags);
}
}
function regexpExtract(text, regexp) {
if (typeof text !== "string")
return null;
if (typeof regexp === "string") {
regexp = createRegExp(regexp);
if (!regexp)
return null;
}
const match = regexp.exec(text);
if (!match || !match[1])
return null;
return match[1];
}
function regexpExtractAll(text, regexp) {
if (typeof text !== "string")
return null;
if (typeof regexp === "string") {
if (regexp.endsWith("/"))
regexp += "g";
regexp = createRegExp(regexp);
if (!regexp)
return null;
}
if (!regexp.global)
regexp = new RegExp(regexp.source, regexp.flags + "g");
const matches = Array.from(text.matchAll(regexp) || []);
const result = [];
for (const match of matches)
if (match[1])
result.push(match[1]);
return result.length > 0 ? result : null;
}
function regexpReplace(text, regexp, replace) {
if (typeof text === "string")
return text.replace(regexp, replace);
else
return text;
}
function regexpTest(text, pattern) {
const negate = pattern.startsWith("!");
if (negate)
pattern = pattern.slice(1);
const regexp = createRegExp(pattern);
if (!regexp)
return null;
let result = regexp?.test(text);
if (result === undefined)
return null;
else if (negate)
return !result;
else
return result;
}
function waitForScrollEnd() {
return new Promise(resolve => {
let timer = setTimeout(() => resolve(), 3000);
function onScroll() {
clearTimeout(timer);
timer = setTimeout(() => {
removeEventListener("scroll", onScroll);
resolve();
}, 200);
}
addEventListener("scroll", onScroll);
});
}
function selectorStatement(query) {
const valid = query instanceof Array && query.length > 0 && typeof query[0] === "string" && query.slice(1).every(op => op instanceof Array);
if (!valid) {
return `INVALID: ${JSON.stringify(query)}`;
}
const selector = query[0];
const ops = query.slice(1);
return [`$("${selector}")`, ...ops.map(op => `${op[0]}(${op.slice(1).map(param => JSON.stringify(param)).join(", ")})`)].join(".");
}
function selectorStatements(query) {
if (query && query.length > 0)
return `${selectorStatement(query[0])}${query.length > 1 ? ` (+${query.length - 1} more))` : ""}`;
else
return "(none)";
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
class Timer {
t0;
constructor() {
this.t0 = Date.now();
}
elapsed() {
return Date.now() - this.t0;
}
}
function trim(text, pattern = " ") {
return ltrim(rtrim(text, pattern));
}
function ltrim(text, pattern = " ") {
if (typeof text === "string") {
if (typeof pattern === "string") {
while (text.startsWith(pattern)) {
text = text.slice(pattern.length);
}
}
else {
const hits = pattern.exec(text) || [];
let hit = hits.find(hit => text.startsWith(hit));
while (hit) {
text = text.slice(hit.length);
hit = hits.find(hit => text.startsWith(hit));
}
}
}
return text;
}
function rtrim(text, pattern = " ") {
if (typeof text === "string") {
if (typeof pattern === "string") {
while (text.endsWith(pattern)) {
text = text.slice(0, -1 * pattern.length);
}
}
else {
const hits = pattern.exec(text) || [];
let hit = hits.find(hit => text.endsWith(hit));
while (hit) {
text = text.slice(0, -1 * hit.length);
hit = hits.find(hit => text.endsWith(hit));
}
}
}
return text;
}
function trunc(obj, max = 300) {
if (obj) {
const text = JSON.stringify(obj);
if (typeof text === "string")
return text.length <= max ? text : `${text[0]}${text.slice(1, max)}…${text[text.length - 1]}`;
}
return "(empty)";
}
function typeName(obj) {
if (obj === null)
return "null";
else if (obj === undefined)
return "undefined";
else if (typeof obj === "string")
return "string";
else if (typeof obj === "number")
return "number";
else if (obj instanceof Array)
return obj.length > 0 ? `Array<${Array.from(new Set(obj.map(value => typeName(value)))).join("|")}>` : "[]";
else if (obj instanceof Date)
return "date";
else if (typeof obj === "object")
return "object";
else
return "unknown";
}
function unpatch(keys) {
const iframe = document.createElement("iframe");
iframe.style.display = "none";
document.body.appendChild(iframe);
if (iframe.contentWindow) {
const contentWindow = iframe.contentWindow;
for (const key of keys) {
if (!key.includes(".")) {
const obj = contentWindow[key];
if (obj) {
const descriptor = Object.getOwnPropertyDescriptor(contentWindow, key);
if (descriptor)
Object.defineProperty(window, key, descriptor);
}
}
else {
const parts = key.split(".");
const objectType = parts.slice(0, -1).join(".");
const method = parts[parts.length - 1];
let obj = contentWindow;
for (const part of objectType.split('.')) {
obj = obj[part];
}
if (obj) {
const descriptor = Object.getOwnPropertyDescriptor(obj, method);
if (descriptor) {
let targetObj = window;
for (const part of objectType.split('.')) {
targetObj = targetObj[part];
}
Object.defineProperty(targetObj, method, descriptor);
}
}
}
}
}
document.body.removeChild(iframe);
}
function unwrap(obj) {
if (isUnwrappable(obj)) {
return unwrap(obj.value);
}
else if (isObject(obj)) {
const source = obj;
const target = {};
const keys = Object.keys(obj);
for (const key of keys)
target[key] = unwrap(source[key]);
return target;
}
else if (obj instanceof Array) {
return obj.map(item => unwrap(item));
}
else {
return obj;
}
}
function isObject(obj) {
return typeof obj === "object" && obj !== null && !(obj instanceof Array) && !(obj instanceof Date);
}
function isUnwrappable(obj) {
return isObject(obj) && obj.hasOwnProperty("value") && obj.hasOwnProperty("nodes");
}
function evaluateXPath(xpath, nodes) {
const result = {
nodes: [],
value: []
};
for (const node of nodes) {
const eval_result = document.evaluate(xpath, node, null, XPathResult.ANY_TYPE, null);
if (eval_result.resultType === XPathResult.STRING_TYPE) {
result.nodes.push(node);
result.value.push(eval_result.stringValue);
}
else if (eval_result.resultType === XPathResult.NUMBER_TYPE) {
result.nodes.push(node);
result.value.push(eval_result.numberValue);
}
else if (eval_result.resultType === XPathResult.BOOLEAN_TYPE) {
result.nodes.push(node);
result.value.push(eval_result.booleanValue);
}
else if (eval_result.resultType === XPathResult.UNORDERED_NODE_ITERATOR_TYPE) {
let node = eval_result.iterateNext();
while (node) {
result.nodes.push(node);
result.value.push(node.textContent);
node = eval_result.iterateNext();
}
return result;
}
else if (eval_result.resultType === XPathResult.ORDERED_NODE_ITERATOR_TYPE) {
const result = {
nodes: [],
value: []
};
let node = eval_result.iterateNext();
while (node) {
result.nodes.push(node);
result.value.push(node.textContent || null);
node = eval_result.iterateNext();
}
return result;
}
}
return result;
}
const defaultTimeoutSeconds = 30;
class Controller {
jquery;
state;
online;
lastLogLine = "";
lastLogLength = 0;
lastLogTimestamp = 0;
lastStep = [];
step = [];
constructor(state) {
this.jquery = state.root || $;
this.online = typeof this.jquery.noConflict === "function";
if (this.online)
state.url = window.location.href;
const { domain, origin } = parseUrl(state.url);
if (state.__locator) {
state.data = merge(state.data, state.__locator);
state.__locator = undefined;
}
const version = "1.2.78";
this.state = {
params: {},
errors: [],
log: "",
...state,
yield: undefined,
vars: {
__instance: 0,
__context: [],
__metrics: {},
__repeat: {},
__t0: Date.now(),
__timeout: state.timeout || defaultTimeoutSeconds,
...state.vars,
__step: [],
__yield: state.yield?.step
},
domain,
origin,
version
};
this.state.vars.__metrics = {
actions: 0,
clicks: 0,
elapsed: 0,
errors: 0,
navigate: 0,
queries: 0,
renavigations: 0,
retries: 0,
skipped: 0,
snooze: 0,
steps: 0,
timeouts: 0,
waitfor: 0,
yields: 0,
...state.vars?.__metrics
};
if (this.state.context) {
const $ = this.jquery;
const nodes = $(this.state.context);
const value = this.text(nodes);
this.state.vars.__context = [{ name: "context", nodes, value }];
}
}
appendError(code, message, level, stack) {
const key = this.contextKey();
this.state.errors.push({ code, message, key, level, stack });
const text = `ERROR ${key ? `${key}: ` : ""}${message} code=${code} level=${level}${stack ? `\n${stack}` : ""}`;
this.log(text);
}
break({ name = "", query, on = "any", pattern, when }) {
if (name)
name = " " + name;
if (this.online) {
if (this.when(when, "BREAK")) {
this.state.vars.__metrics.steps += 1;
if (query) {
this.log(`BREAK${name} WAITFOR QUERY ${trunc($)} on=${on}, pattern=${pattern}`);
const [pass, result] = this.queryCheck(query, on, pattern);
this.log(`BREAK${name} QUERY ${selectorStatements(query)} -> ${trunc(result?.value)}${pattern ? ` (valid=${result?.valid})` : ""} -> on=${on} -> ${pass}`);
if (pass) {
this.log(`BREAK${name} ${when || ""}`);
return true;
}
}
else {
this.log(`BREAK${name} ${when || ""}`);
return true;
}
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`BREAK${name} SKIPPED ${when}`);
}
}
else {
this.log(`BREAK${name ? ` ${name}` : ""} BYPASSED ${when}`);
}
return false;
}
async click({ name, query, waitfor, snooze, required, retry, when, ...options }) {
if (this.online) {
if (this.when(when, `CLICK${name ? ` ${name}` : ""}`)) {
const mode = snooze ? snooze[2] || "before" : undefined;
if (snooze && (mode === "before" || mode === "before-and-after")) {
const duration = this.maxTimeout(snooze[0]);
this.log(`CLICK${name ? ` ${name}` : ""} SNOOZE BEFORE (${duration.toFixed(1)}s) ${selectorStatements(query)}`);
await sleep(duration);
this.state.vars.__metrics.snooze += duration;
}
const result = this.query({ query });
if (result && result.nodes.length > 0) {
if (this.clickElement(result.nodes[0], selectorStatements(query))) {
if (waitfor) {
const code = await this.waitfor(waitfor, "CLICK");
if (!code) {
if (snooze && (mode === "after" || mode === "before-and-after")) {
const duration = this.maxTimeout(snooze[0]);
this.log(`CLICK${name ? ` ${name}` : ""} SNOOZE AFTER (${duration.toFixed(1)}s) ${selectorStatements(query)}`);
await sleep(duration);
this.state.vars.__metrics.snooze += duration;
}
}
else if (code === "timeout") {
this.appendError("click-timeout", `Timeout waiting for CLICK${name ? ` ${name}` : ""} result. ${trunc(waitfor.query)}${waitfor.pattern ? `, pattern=${waitfor.pattern}` : ""}`, 1);
return "timeout";
}
else ;
}
else if (options.yield) {
this.yield({
name: `CLICK ${name ? ` ${name}` : ""}`,
params: {
click: {},
waitUntil: options.waitUntil
}
});
}
this.state.vars.__metrics.clicks += 1;
this.state.vars.__metrics.steps += 1;
}
}
else {
if (required) {
this.appendError("click-required", `Required click target not found. ${trunc(query)}`, 1);
}
return "not-found";
}
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`CLICK${name ? ` ${name}` : ""} SKIPPED ${selectorStatements(query)}`);
}
}
else {
this.log(`CLICK${name ? ` ${name}` : ""} IGNORED ${selectorStatements(query)}`);
}
return null;
}
clickElement(element, context) {
if (element instanceof HTMLElement) {
if (element instanceof HTMLOptionElement && element.parentElement instanceof HTMLSelectElement) {
this.log(`CLICK ${context} <select> "${element.parentElement.value}" -> "${element.value}"`);
element.parentElement.value = element.value;
element.parentElement.dispatchEvent(new Event("change", { bubbles: true, cancelable: false }));
element.parentElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: false }));
}
else {
this.log(`CLICK ${context}`);
element.click();
}
return true;
}
else {
this.log(`CLICK ${context} not insanceof HTMLElement`);
return false;
}
}
context() {
const stack = this.state.vars.__context;
let j = stack.length - 1;
const context = { ...stack[j] };
let subcontext = context;
while (--j >= 0) {
subcontext.parent = { ...stack[j] };
subcontext = subcontext.parent;
}
return context;
}
contextKey() {
const stack = this.state.vars.__context;
let key = "";
for (const { name, index } of stack) {
if (name) {
if (key) {
key += ".";
}
key += name;
if (index !== undefined) {
key += `[${index}]`;
}
}
}
return key;
}
contextKeyInfo() {
const key = this.contextKey();
const stack = this.state.vars.__context;
let info = "";
if (stack.length > 0) {
const [top] = stack.slice(-1);
if (top.pivot !== undefined) {
info = `PIVOT(${top.pivot})`;
}
else if (top.union !== undefined) {
info = `UNION(${top.union})`;
}
else if (top.action !== undefined) {
info = `${top.action.toUpperCase()}`;
}
}
return info ? `${key} ${info}` : key;
}
async dispatch(action) {
if (action.hasOwnProperty("select")) {
const select = action.select;
const code = await this.selectWaitfor(select);
if (code === "timeout")
return "timeout";
const data = this.select(select);
this.state.data = merge(this.state.data, data);
}
else if (action.hasOwnProperty("break")) {
if (this.break(action.break)) {
return "break";
}
}
else if (action.hasOwnProperty("click")) {
const required = action.click.required;
const code = await this.click(action.click);
if (code === "timeout" && required) {
return "timeout";
}
else if (code === "not-found" && required) {
return "required";
}
}
else if (action.hasOwnProperty("each")) {
await this.each(action.each);
}
else if (action.hasOwnProperty("error")) {
this.error(action.error);
}
else if (action.hasOwnProperty("goback")) {
await this.goback(action.goback);
}
else if (action.hasOwnProperty("keypress")) {
this.keypress(action.keypress);
}
else if (action.hasOwnProperty("locator")) {
await this.locator(action.locator);
}
else if (action.hasOwnProperty("navigate")) {
await this.navigate(action.navigate);
}
else if (action.hasOwnProperty("reload")) {
await this.reload(action.reload);
}
else if (action.hasOwnProperty("repeat")) {
await this.repeat(action.repeat);
}
else if (action.hasOwnProperty("screenshot")) {
await this.screenshot(action.screenshot);
}
else if (action.hasOwnProperty("scroll")) {
await this.scroll(action.scroll);
}
else if (action.hasOwnProperty("snooze")) {
await this.snooze(action.snooze);
}
else if (action.hasOwnProperty("switch")) {
await this.switch(action.switch);
}
else if (action.hasOwnProperty("transform")) {
this.transform(action.transform);
}
else if (action.hasOwnProperty("waitfor")) {
const required = action.waitfor.required;
const code = await this.waitfor(action.waitfor);
if (code === "timeout" && required) {
return "timeout";
}
}
else if (action.hasOwnProperty("yield")) {
this.yield(action.yield || {});
}
return null;
}
async each({ name, query, actions, context, when }) {
const $ = this.jquery;
const label = `EACH${name ? ` ${name}` : ""}`;
if (this.when(when, label)) {
const result = this.query({ query, repeated: true });
if (result && result.nodes.length > 0) {
const elements = result.nodes.toArray();
for (const element of elements) {
const nodes = $(element);
this.pushContext({
nodes,
value: this.text(nodes),
action: "each",
index: elements.indexOf(element)
}, context);
const code = await this.run(actions, label, true);
this.popContext();
if (code === "break") {
break;
}
}
}
this.state.vars.__metrics.steps += 1;
}
else {
this.state.vars.__metrics.skipped += 1;
}
}
eachNode({ nodes, value }, callback) {
const $ = this.jquery;
if (nodes) {
const elements = nodes.toArray();
for (let i = 0; i < elements.length; ++i) {
const node = $(elements[i]);
const subvalue = value instanceof Array ? value[i] : value;
callback(node, subvalue);
}
}
}
elapsed() {
return Date.now() - this.state.vars.__t0;
}
error({ query, code = "app-error", message, level = 1, negate, stop, when }) {
if (!message)
message = errorCodeMessageMap[code] || "Unknown error.";
if (query) {
const result = this.query({ query, type: "boolean", repeated: false });
const hit = !negate ? result?.value === false : result?.value === true;
if (hit) {
this.appendError(code, String(this.evaluate(message)), level);
this.state.vars.__metrics.steps += 1;
if (stop === true || (stop === undefined && level === 0))
throw "STOP";
}
}
else if (this.when(when, "ERROR")) {
this.appendError(code, String(this.evaluate(message)), level);
this.state.vars.__metrics.steps += 1;
if (stop === true || (stop === undefined && level === 0))
throw "STOP";
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`ERROR ${code} SKIPPED ${when}`);
}
}
evaluate(input, params = {}) {
if (isFormula(input)) {
const { data, ...extra } = params;
const context = this.context();
const args = {
...this.state.vars,
...this.state,
...context,
data: unwrap(merge(this.state.data, data)),
...extra
};
try {
const result = evaluateFormula(input.slice(1, -1).trim(), args);
return result;
}
catch (err) {
this.appendError("eval-error", `Error evaluating formula "${input}": ${err instanceof Error ? err.message : JSON.stringify(err)}`, 0);
}
}
else if (isRegexp(input)) {
const result = regexpExtract(params.value, input);
return result;
}
else {
return input;
}
}
evaluateBoolean(input, params = {}) {
if (input !== undefined && input !== null) {
if (isRegexp(input)) {
const result = regexpTest(params.value, input);
return result !== null ? result : undefined;
}
else {
const result = this.evaluate(input, params);
if (typeof result === "boolean")
return result;
}
}
return undefined;
}
evaluateNumber(input, params = {}) {
if (input !== undefined && input !== null) {
const result = this.evaluate(input, params);
if (typeof result === "number")
return result;
}
return undefined;
}
evaluateString(input, params = {}) {
if (input !== undefined && input !== null) {
const result = this.evaluate(input, params);
if (typeof result === "string")
return result;
}
return undefined;
}
formatResult(result, type, all, limit, format = "multiline", pattern, distinct, negate, removeNulls) {
const $ = this.jquery;
const regexp = createRegExp(pattern);
if (result.raw)
return result;
if (!type) {
const defaultType = result.value instanceof Array ? typeof result.value[0] : typeof result.value;
type = ["string", "number", "boolean"].includes(defaultType) ? defaultType : "string";
}
if (limit !== undefined && limit !== null && result.value instanceof Array) {
result.nodes = $(result.nodes.toArray().slice(0, limit));
result.value = result.value.slice(0, limit);
}
if (type === "string" && result.value instanceof Array) {
if (!result.formatted) {
result.value = result.value.map(value => formatStringValue(coerceSelectValue(value, "string"), format, this.state.origin));
}
if (regexp && !isEmpty(result.value)) {
result.valid = result.value.every(value => regexp.test(value));
}
}
else if (type === "string") {
if (!result.formatted) {
result.value = formatStringValue(coerceSelectValue(result.value, "string"), format, this.state.origin);
}
if (regexp && !isEmpty(result.value)) {
result.valid = regexp.test(result.value);
}
}
else if (type === "boolean" && result.value instanceof Array && result.value.length === 0) {
result.value = false;
}
else if (type === "boolean" && result.value instanceof Array && all) {
result.value = result.value.every(value => coerceSelectValue(value, "boolean") === true);
}
else if (type === "boolean" && result.value instanceof Array && !all) {
result.value = result.value.some(value => coerceSelectValue(value, "boolean") === true);
}
else if (type === "boolean") {
result.value = coerceSelectValue(result.value, "boolean");
}
else if (type === "number" && result.value instanceof Array) {
result.value = result.value.map(value => coerceSelectValue(value, "number"));
}
else if (type === "number") {
result.value = coerceSelectValue(result.value, "number");
}
if (negate) {
if (typeof result.value === "boolean")
result.value = !result.value;
else if (result.value instanceof Array && result.value.every(value => typeof value === "boolean"))
result.value = result.value.map(value => !value);
}
if (removeNulls && result.value instanceof Array) {
filterQueryResult($, result, value => value !== null);
}
if (distinct && result.value instanceof Array) {
filterQueryResult($, result, (value, index, array) => array.indexOf(value) === index);
}
return result;
}
goback({ name, when }) {
this.yield({
name: `GOBACK ${name ? ` ${name}` : ""}`,
params: {
goback: {},
action: "goBack"
},
when
});
}
keypress({ name = "", key, shift, control, alt, when }) {
if (name)
name = " " + name;
if (this.online) {
if (this.when(when, "KEYPRESS")) {
const event = new KeyboardEvent("keydown", {
key,
code: 'Key' + key.toUpperCase(),
keyCode: key.charCodeAt(0),
which: key.charCodeAt(0),
shiftKey: shift,
ctrlKey: control,
altKey: alt
});
document.dispatchEvent(event);
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`KEYPRESS${name} SKIPPED ${when}`);
}
}
else {
this.log(`KEYPRESS${name ? ` ${name}` : ""} BYPASSED ${when}`);
}
}
log(text) {
if (this.state.debug) {
if (this.lastLogLine === text) {
const elapsed = (Date.now() - this.lastLogTimestamp) / 1000;
this.state.log = `${this.state.log.slice(0, this.lastLogLength)}${text} (${elapsed.toFixed(1)}s)\n`;
}
else {
this.lastLogLine = text;
this.lastLogLength = this.state.log.length;
this.lastLogTimestamp = Date.now();
this.state.log += `${String(this.elapsed()).padStart(8, "0")} ${text}\n`;
}
}
}
maxTimeout(seconds = defaultTimeoutSeconds) {
const elapsed = this.elapsed() / 1000;
const remaining = Math.max(this.state.vars.__timeout - elapsed, 0);
return Math.min(seconds, remaining);
}
mergeQueryResult(source, target) {
const $ = this.jquery;
if (source && target) {
const nodes = source.nodes && target.nodes ? $([...source.nodes.toArray(), ...target.nodes.toArray()]) : target.nodes || source.nodes;
let value = undefined;
if (source.value instanceof Array && target.value instanceof Array) {
value = [...source.value, ...target.value];
}
else if (source.value instanceof Array && !isNullOrUndefined(target.value)) {
value = [...source.value, target.value];
}
else if (target.value instanceof Array && !isNullOrUndefined(source.value)) {
value = [source.value, ...target.value];
}
else if (!isNullOrUndefined(source.value) && !isNullOrUndefined(target.value)) {
value = [source.value, target.value];
}
else {
value = target.value ?? source.value;
}
return { nodes, key: this.contextKey(), value, valid: target.valid ?? source.valid };
}
else if (target) {
return target;
}
else {
return source;
}
}
nodeKey(node) {
const $ = this.jquery;
const path = [];
const elements = $(node && node.length > 1 ? node[0] : node).parents().addBack().not("html").toArray().reverse();
for (const element of elements) {
const $parent = $(element).parent();
const tag = element.tagName?.toLowerCase();
const id = $(element).attr('id') || "";
const className = $(element).attr('class')?.split(' ')[0] || "";
const n = $(element).index() + 1;
const uniqueId = /^[A-Za-z0-9_-]+$/.test(id) ? $(`#${id}`).length === 1 : false;
const uniqueClassName = tag && /^[A-Za-z0-9_-]+$/.test(className) ? $(`${tag}.${className}`).length === 1 : false;
const onlyTag = $parent.children(tag).length === 1;
const onlyClassName = tag && /^[A-Za-z0-9_-]+$/.test(className) ? $parent.children(`${tag}.${className}`).length === 1 : false;
if (uniqueId)
path.push(`#${id}`);
else if (uniqueClassName)
path.push(`${tag}.${className}`);
else if (onlyTag)
path.push(tag);
else if (onlyClassName)
path.push(`${tag}.${className}`);
else
path.push(`${tag}:nth-child(${n})`);
if (uniqueId || uniqueClassName)
break;
}
return path.reverse().join(" > ");
}
nodeKeys(nodes) {
if (typeof nodes === "object" && typeof nodes.toArray === "function") {
return nodes.toArray().map(node => this.nodeKey(node));
}
else {
return [];
}
}
pokeContext(context) {
const stack = this.state.vars.__context;
const i = stack.length - 1;
if (i >= 0) {
stack[i] = { ...stack[i], ...context };
}
if (context.nodes) {
this.log(`--> ${this.contextKeyInfo()} [${this.nodeKey(stack[stack.length - 1].nodes)}] ${trunc(stack[stack.length - 1].value)}`);
}
}
popContext() {
const stack = this.state.vars.__context;
const [top] = stack.slice(-1);
this.log(`<<< ${this.contextKeyInfo()} [${this.nodeKey(top?.nodes)}] ${stack.length - 1}`);
stack.pop();
}
pushContext(context, inherit) {
const stack = this.state.vars.__context;
if (inherit === undefined) {
const [parent] = stack.slice(-1);
stack.push({
nodes: parent?.nodes,
value: parent?.value,
...context
});
}
else if (inherit === null) {
stack.push({ ...context, nodes: undefined });
}
else if (inherit > 0 && inherit <= stack.length) {
const [parent] = stack.slice(-1 * inherit);
stack.push({
...context,
nodes: parent?.nodes,
value: parent?.value
});
}
else {
stack.push(context);
this.appendError("eval-error", `Undefined context #${inherit}`, 1);
}
this.log(`>>> ${this.contextKeyInfo()} [${this.nodeKey(stack[stack.length - 1].nodes)}] ${trunc(stack[stack.length - 1].value)} ${stack.length}`);
}
query({ query, type, repeated = false, all = false, format, pattern, limit, hits, distinct, negate, removeNulls }) {
if (query instanceof Array && query.every(stage => stage instanceof Array) && query[0].length > 0 && !!query[0][0]) {
if (limit === undefined && type === "string" && !repeated && !all) {
limit = 1;
}
if (hits === null || hits === undefined) {
hits = query.length;
}
let hit = 0;
let result = undefined;
for (const stage of query) {
const subresult = this.resolveQuery({ query: stage, type, repeated, all, limit, format, pattern, distinct, negate, removeNulls, result });
if (subresult) {
result = this.mergeQueryResult(result, subresult);
if (subresult.nodes.length > 0) {
if (!all) {
this.log(`[${query.indexOf(stage) + 1}/${query.length}] STOP (first hit)`);
break;
}
if (++hit === hits) {
this.log(`[${query.indexOf(stage) + 1}/${query.length}] STOP (${hits} hits)`);
break;
}
}
}
}
if (result && !result.raw) {
if (repeated && !Array.isArray(result.value))
result.value = [result.value];
else if (!repeated && Array.isArray(result.value) && result.value.every(value => typeof value === "string"))
result.value = result.value.length > 0 ? result.value.join(format === "singleline" ? " " : "\n") : null;
else if (!repeated && type === "boolean" && !negate && Array.isArray(result.value) && result.value.every(value => typeof value === "boolean"))
result.value = result.value.some(value => value === true);
else if (!repeated && type === "boolean" && negate && Array.isArray(result.value) && result.value.every(value => typeof value === "boolean"))
result.val