syphonx-core
Version:
SyphonX is a template-driven solution for extracting data from HTML in a highly efficient way. It combines the power of jQuery, Regular Expressions, and Javascript into a declarative template-driven format that extracts and reshapes HTML data into JSON.
1,161 lines • 83 kB
JavaScript
const defaultTimeoutSeconds = 30;
import { errorCodeMessageMap } from "./errors.js";
import { autoPaginate, coerce, coerceSelectValue, createRegExp, cut, evaluateFormula, evaluateXPath, filterQueryResult, formatHTML, formatStringValue, isCoercibleTo, isEmpty, isFormula, isInvocableFrom, isJQueryObject, isNullOrUndefined, isRegexp, merge, mergeElements, parseBoolean, parseUrl, regexpExtract, regexpExtractAll, regexpReplace, regexpTest, selectorStatement, selectorStatements, sleep, trim, trunc, tryParseJson, typeName, unwrap, waitForScrollEnd, Timer } from "./lib/index.js";
export class Controller {
jquery;
state;
online;
lastLogLine = "";
lastLogLength = 0;
lastLogTimestamp = 0;
lastStep = [];
step = [];
constructor(state) {
this.jquery = state.root || $;
this.online = typeof this.jquery.noConflict === "function";
if (this.online)
state.url = window.location.href;
const { domain, origin } = parseUrl(state.url);
if (state.__locator) {
state.data = merge(state.data, state.__locator);
state.__locator = undefined;
}
const version = "1.2.80";
this.state = {
params: {},
errors: [],
log: "",
...state,
yield: undefined,
vars: {
__instance: 0,
__context: [],
__metrics: {},
__repeat: {},
__t0: Date.now(),
__timeout: state.timeout || defaultTimeoutSeconds,
...state.vars,
__step: [],
__yield: state.yield?.step
},
domain,
origin,
version
};
this.state.vars.__metrics = {
actions: 0,
clicks: 0,
elapsed: 0,
errors: 0,
navigate: 0,
queries: 0,
renavigations: 0,
retries: 0,
skipped: 0,
snooze: 0,
steps: 0,
timeouts: 0,
waitfor: 0,
yields: 0,
...state.vars?.__metrics
};
if (this.state.context) {
const $ = this.jquery;
const nodes = $(this.state.context);
const value = this.text(nodes);
this.state.vars.__context = [{ name: "context", nodes, value }];
}
}
appendError(code, message, level, stack) {
const key = this.contextKey();
this.state.errors.push({ code, message, key, level, stack });
const text = `ERROR ${key ? `${key}: ` : ""}${message} code=${code} level=${level}${stack ? `\n${stack}` : ""}`;
this.log(text);
}
break({ name = "", query, on = "any", pattern, when }) {
if (name)
name = " " + name;
if (this.online) {
if (this.when(when, "BREAK")) {
this.state.vars.__metrics.steps += 1;
if (query) {
this.log(`BREAK${name} WAITFOR QUERY ${trunc($)} on=${on}, pattern=${pattern}`);
const [pass, result] = this.queryCheck(query, on, pattern);
this.log(`BREAK${name} QUERY ${selectorStatements(query)} -> ${trunc(result?.value)}${pattern ? ` (valid=${result?.valid})` : ""} -> on=${on} -> ${pass}`);
if (pass) {
this.log(`BREAK${name} ${when || ""}`);
return true;
}
}
else {
this.log(`BREAK${name} ${when || ""}`);
return true;
}
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`BREAK${name} SKIPPED ${when}`);
}
}
else {
this.log(`BREAK${name ? ` ${name}` : ""} BYPASSED ${when}`);
}
return false;
}
async click({ name, query, waitfor, snooze, required, retry, when, ...options }) {
if (this.online) {
if (this.when(when, `CLICK${name ? ` ${name}` : ""}`)) {
const mode = snooze ? snooze[2] || "before" : undefined;
if (snooze && (mode === "before" || mode === "before-and-after")) {
const duration = this.maxTimeout(snooze[0]);
this.log(`CLICK${name ? ` ${name}` : ""} SNOOZE BEFORE (${duration.toFixed(1)}s) ${selectorStatements(query)}`);
await sleep(duration);
this.state.vars.__metrics.snooze += duration;
}
const result = this.query({ query });
if (result && result.nodes.length > 0) {
if (this.clickElement(result.nodes[0], selectorStatements(query))) {
if (waitfor) {
const code = await this.waitfor(waitfor, "CLICK");
if (!code) {
if (snooze && (mode === "after" || mode === "before-and-after")) {
const duration = this.maxTimeout(snooze[0]);
this.log(`CLICK${name ? ` ${name}` : ""} SNOOZE AFTER (${duration.toFixed(1)}s) ${selectorStatements(query)}`);
await sleep(duration);
this.state.vars.__metrics.snooze += duration;
}
}
else if (code === "timeout") {
this.appendError("click-timeout", `Timeout waiting for CLICK${name ? ` ${name}` : ""} result. ${trunc(waitfor.query)}${waitfor.pattern ? `, pattern=${waitfor.pattern}` : ""}`, 1);
return "timeout";
}
else if (code === "invalid") {
}
}
else if (options.yield) {
this.yield({
name: `CLICK ${name ? ` ${name}` : ""}`,
params: {
click: {},
waitUntil: options.waitUntil
}
});
}
this.state.vars.__metrics.clicks += 1;
this.state.vars.__metrics.steps += 1;
}
}
else {
if (required) {
this.appendError("click-required", `Required click target not found. ${trunc(query)}`, 1);
}
return "not-found";
}
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`CLICK${name ? ` ${name}` : ""} SKIPPED ${selectorStatements(query)}`);
}
}
else {
this.log(`CLICK${name ? ` ${name}` : ""} IGNORED ${selectorStatements(query)}`);
}
return null;
}
clickElement(element, context) {
if (element instanceof HTMLElement) {
if (element instanceof HTMLOptionElement && element.parentElement instanceof HTMLSelectElement) {
this.log(`CLICK ${context} <select> "${element.parentElement.value}" -> "${element.value}"`);
element.parentElement.value = element.value;
element.parentElement.dispatchEvent(new Event("change", { bubbles: true, cancelable: false }));
element.parentElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: false }));
}
else {
this.log(`CLICK ${context}`);
element.click();
}
return true;
}
else {
this.log(`CLICK ${context} not insanceof HTMLElement`);
return false;
}
}
context() {
const stack = this.state.vars.__context;
let j = stack.length - 1;
const context = { ...stack[j] };
let subcontext = context;
while (--j >= 0) {
subcontext.parent = { ...stack[j] };
subcontext = subcontext.parent;
}
return context;
}
contextKey() {
const stack = this.state.vars.__context;
let key = "";
for (const { name, index } of stack) {
if (name) {
if (key) {
key += ".";
}
key += name;
if (index !== undefined) {
key += `[${index}]`;
}
}
}
return key;
}
contextKeyInfo() {
const key = this.contextKey();
const stack = this.state.vars.__context;
let info = "";
if (stack.length > 0) {
const [top] = stack.slice(-1);
if (top.pivot !== undefined) {
info = `PIVOT(${top.pivot})`;
}
else if (top.union !== undefined) {
info = `UNION(${top.union})`;
}
else if (top.action !== undefined) {
info = `${top.action.toUpperCase()}`;
}
}
return info ? `${key} ${info}` : key;
}
async dispatch(action) {
if (action.hasOwnProperty("select")) {
const select = action.select;
const code = await this.selectWaitfor(select);
if (code === "timeout")
return "timeout";
const data = this.select(select);
this.state.data = merge(this.state.data, data);
}
else if (action.hasOwnProperty("break")) {
if (this.break(action.break)) {
return "break";
}
}
else if (action.hasOwnProperty("click")) {
const required = action.click.required;
const code = await this.click(action.click);
if (code === "timeout" && required) {
return "timeout";
}
else if (code === "not-found" && required) {
return "required";
}
}
else if (action.hasOwnProperty("each")) {
await this.each(action.each);
}
else if (action.hasOwnProperty("error")) {
this.error(action.error);
}
else if (action.hasOwnProperty("goback")) {
await this.goback(action.goback);
}
else if (action.hasOwnProperty("keypress")) {
this.keypress(action.keypress);
}
else if (action.hasOwnProperty("locator")) {
await this.locator(action.locator);
}
else if (action.hasOwnProperty("navigate")) {
await this.navigate(action.navigate);
}
else if (action.hasOwnProperty("reload")) {
await this.reload(action.reload);
}
else if (action.hasOwnProperty("repeat")) {
await this.repeat(action.repeat);
}
else if (action.hasOwnProperty("screenshot")) {
await this.screenshot(action.screenshot);
}
else if (action.hasOwnProperty("scroll")) {
await this.scroll(action.scroll);
}
else if (action.hasOwnProperty("snooze")) {
await this.snooze(action.snooze);
}
else if (action.hasOwnProperty("switch")) {
await this.switch(action.switch);
}
else if (action.hasOwnProperty("transform")) {
await this.transform(action.transform);
}
else if (action.hasOwnProperty("waitfor")) {
const required = action.waitfor.required;
const code = await this.waitfor(action.waitfor);
if (code === "timeout" && required) {
return "timeout";
}
}
else if (action.hasOwnProperty("yield")) {
this.yield(action.yield || {});
}
return null;
}
async each({ name, query, actions, context, when }) {
const $ = this.jquery;
const label = `EACH${name ? ` ${name}` : ""}`;
if (this.when(when, label)) {
const result = this.query({ query, repeated: true });
if (result && result.nodes.length > 0) {
const elements = result.nodes.toArray();
for (const element of elements) {
const nodes = $(element);
this.pushContext({
nodes,
value: this.text(nodes),
action: "each",
index: elements.indexOf(element)
}, context);
const code = await this.run(actions, label, true);
this.popContext();
if (code === "break") {
break;
}
}
}
this.state.vars.__metrics.steps += 1;
}
else {
this.state.vars.__metrics.skipped += 1;
}
}
eachNode({ nodes, value }, callback) {
const $ = this.jquery;
if (nodes) {
const elements = nodes.toArray();
for (let i = 0; i < elements.length; ++i) {
const node = $(elements[i]);
const subvalue = value instanceof Array ? value[i] : value;
callback(node, subvalue);
}
}
}
elapsed() {
return Date.now() - this.state.vars.__t0;
}
error({ query, code = "app-error", message, level = 1, negate, stop, when }) {
if (!message)
message = errorCodeMessageMap[code] || "Unknown error.";
if (query) {
const result = this.query({ query, type: "boolean", repeated: false });
const hit = !negate ? result?.value === false : result?.value === true;
if (hit) {
this.appendError(code, String(this.evaluate(message)), level);
this.state.vars.__metrics.steps += 1;
if (stop === true || (stop === undefined && level === 0))
throw "STOP";
}
}
else if (this.when(when, "ERROR")) {
this.appendError(code, String(this.evaluate(message)), level);
this.state.vars.__metrics.steps += 1;
if (stop === true || (stop === undefined && level === 0))
throw "STOP";
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`ERROR ${code} SKIPPED ${when}`);
}
}
evaluate(input, params = {}) {
if (isFormula(input)) {
const { data, ...extra } = params;
const context = this.context();
const args = {
...this.state.vars,
...this.state,
...context,
data: unwrap(merge(this.state.data, data)),
...extra
};
try {
const result = evaluateFormula(input.slice(1, -1).trim(), args);
return result;
}
catch (err) {
this.appendError("eval-error", `Error evaluating formula "${input}": ${err instanceof Error ? err.message : JSON.stringify(err)}`, 0);
}
}
else if (isRegexp(input)) {
const result = regexpExtract(params.value, input);
return result;
}
else {
return input;
}
}
evaluateBoolean(input, params = {}) {
if (input !== undefined && input !== null) {
if (isRegexp(input)) {
const result = regexpTest(params.value, input);
return result !== null ? result : undefined;
}
else {
const result = this.evaluate(input, params);
if (typeof result === "boolean")
return result;
}
}
return undefined;
}
evaluateNumber(input, params = {}) {
if (input !== undefined && input !== null) {
const result = this.evaluate(input, params);
if (typeof result === "number")
return result;
}
return undefined;
}
evaluateString(input, params = {}) {
if (input !== undefined && input !== null) {
const result = this.evaluate(input, params);
if (typeof result === "string")
return result;
}
return undefined;
}
formatResult(result, type, all, limit, format = "multiline", pattern, distinct, negate, removeNulls) {
const $ = this.jquery;
const regexp = createRegExp(pattern);
if (result.raw)
return result;
if (!type) {
const defaultType = result.value instanceof Array ? typeof result.value[0] : typeof result.value;
type = ["string", "number", "boolean"].includes(defaultType) ? defaultType : "string";
}
if (limit !== undefined && limit !== null && result.value instanceof Array) {
result.nodes = $(result.nodes.toArray().slice(0, limit));
result.value = result.value.slice(0, limit);
}
if (type === "string" && result.value instanceof Array) {
if (!result.formatted) {
result.value = result.value.map(value => formatStringValue(coerceSelectValue(value, "string"), format, this.state.origin));
}
if (regexp && !isEmpty(result.value)) {
result.valid = result.value.every(value => regexp.test(value));
}
}
else if (type === "string") {
if (!result.formatted) {
result.value = formatStringValue(coerceSelectValue(result.value, "string"), format, this.state.origin);
}
if (regexp && !isEmpty(result.value)) {
result.valid = regexp.test(result.value);
}
}
else if (type === "boolean" && result.value instanceof Array && result.value.length === 0) {
result.value = false;
}
else if (type === "boolean" && result.value instanceof Array && all) {
result.value = result.value.every(value => coerceSelectValue(value, "boolean") === true);
}
else if (type === "boolean" && result.value instanceof Array && !all) {
result.value = result.value.some(value => coerceSelectValue(value, "boolean") === true);
}
else if (type === "boolean") {
result.value = coerceSelectValue(result.value, "boolean");
}
else if (type === "number" && result.value instanceof Array) {
result.value = result.value.map(value => coerceSelectValue(value, "number"));
}
else if (type === "number") {
result.value = coerceSelectValue(result.value, "number");
}
if (negate) {
if (typeof result.value === "boolean")
result.value = !result.value;
else if (result.value instanceof Array && result.value.every(value => typeof value === "boolean"))
result.value = result.value.map(value => !value);
}
if (removeNulls && result.value instanceof Array) {
filterQueryResult($, result, value => value !== null);
}
if (distinct && result.value instanceof Array) {
filterQueryResult($, result, (value, index, array) => array.indexOf(value) === index);
}
return result;
}
goback({ name, when }) {
this.yield({
name: `GOBACK ${name ? ` ${name}` : ""}`,
params: {
goback: {},
action: "goBack"
},
when
});
}
keypress({ name = "", key, shift, control, alt, when }) {
if (name)
name = " " + name;
if (this.online) {
if (this.when(when, "KEYPRESS")) {
const event = new KeyboardEvent("keydown", {
key,
code: 'Key' + key.toUpperCase(),
keyCode: key.charCodeAt(0),
which: key.charCodeAt(0),
shiftKey: shift,
ctrlKey: control,
altKey: alt
});
document.dispatchEvent(event);
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`KEYPRESS${name} SKIPPED ${when}`);
}
}
else {
this.log(`KEYPRESS${name ? ` ${name}` : ""} BYPASSED ${when}`);
}
}
log(text) {
if (this.state.debug) {
if (this.lastLogLine === text) {
const elapsed = (Date.now() - this.lastLogTimestamp) / 1000;
this.state.log = `${this.state.log.slice(0, this.lastLogLength)}${text} (${elapsed.toFixed(1)}s)\n`;
}
else {
this.lastLogLine = text;
this.lastLogLength = this.state.log.length;
this.lastLogTimestamp = Date.now();
this.state.log += `${String(this.elapsed()).padStart(8, "0")} ${text}\n`;
}
}
}
maxTimeout(seconds = defaultTimeoutSeconds) {
const elapsed = this.elapsed() / 1000;
const remaining = Math.max(this.state.vars.__timeout - elapsed, 0);
return Math.min(seconds, remaining);
}
mergeQueryResult(source, target) {
const $ = this.jquery;
if (source && target) {
const nodes = source.nodes && target.nodes ? $([...source.nodes.toArray(), ...target.nodes.toArray()]) : target.nodes || source.nodes;
let value = undefined;
if (source.value instanceof Array && target.value instanceof Array) {
value = [...source.value, ...target.value];
}
else if (source.value instanceof Array && !isNullOrUndefined(target.value)) {
value = [...source.value, target.value];
}
else if (target.value instanceof Array && !isNullOrUndefined(source.value)) {
value = [source.value, ...target.value];
}
else if (!isNullOrUndefined(source.value) && !isNullOrUndefined(target.value)) {
value = [source.value, target.value];
}
else {
value = target.value ?? source.value;
}
return { nodes, key: this.contextKey(), value, valid: target.valid ?? source.valid };
}
else if (target) {
return target;
}
else {
return source;
}
}
nodeKey(node) {
const $ = this.jquery;
const path = [];
const elements = $(node && node.length > 1 ? node[0] : node).parents().addBack().not("html").toArray().reverse();
for (const element of elements) {
const $parent = $(element).parent();
const tag = element.tagName?.toLowerCase();
const id = $(element).attr('id') || "";
const className = $(element).attr('class')?.split(' ')[0] || "";
const n = $(element).index() + 1;
const uniqueId = /^[A-Za-z0-9_-]+$/.test(id) ? $(`#${id}`).length === 1 : false;
const uniqueClassName = tag && /^[A-Za-z0-9_-]+$/.test(className) ? $(`${tag}.${className}`).length === 1 : false;
const onlyTag = $parent.children(tag).length === 1;
const onlyClassName = tag && /^[A-Za-z0-9_-]+$/.test(className) ? $parent.children(`${tag}.${className}`).length === 1 : false;
if (uniqueId)
path.push(`#${id}`);
else if (uniqueClassName)
path.push(`${tag}.${className}`);
else if (onlyTag)
path.push(tag);
else if (onlyClassName)
path.push(`${tag}.${className}`);
else
path.push(`${tag}:nth-child(${n})`);
if (uniqueId || uniqueClassName)
break;
}
return path.reverse().join(" > ");
}
nodeKeys(nodes) {
if (typeof nodes === "object" && typeof nodes.toArray === "function") {
return nodes.toArray().map(node => this.nodeKey(node));
}
else {
return [];
}
}
pokeContext(context) {
const stack = this.state.vars.__context;
const i = stack.length - 1;
if (i >= 0) {
stack[i] = { ...stack[i], ...context };
}
if (context.nodes) {
this.log(`--> ${this.contextKeyInfo()} [${this.nodeKey(stack[stack.length - 1].nodes)}] ${trunc(stack[stack.length - 1].value)}`);
}
}
popContext() {
const stack = this.state.vars.__context;
const [top] = stack.slice(-1);
this.log(`<<< ${this.contextKeyInfo()} [${this.nodeKey(top?.nodes)}] ${stack.length - 1}`);
stack.pop();
}
pushContext(context, inherit) {
const stack = this.state.vars.__context;
if (inherit === undefined) {
const [parent] = stack.slice(-1);
stack.push({
nodes: parent?.nodes,
value: parent?.value,
...context
});
}
else if (inherit === null) {
stack.push({ ...context, nodes: undefined });
}
else if (inherit > 0 && inherit <= stack.length) {
const [parent] = stack.slice(-1 * inherit);
stack.push({
...context,
nodes: parent?.nodes,
value: parent?.value
});
}
else {
stack.push(context);
this.appendError("eval-error", `Undefined context #${inherit}`, 1);
}
this.log(`>>> ${this.contextKeyInfo()} [${this.nodeKey(stack[stack.length - 1].nodes)}] ${trunc(stack[stack.length - 1].value)} ${stack.length}`);
}
query({ query, type, repeated = false, all = false, format, pattern, limit, hits, distinct, negate, removeNulls }) {
if (query instanceof Array && query.every(stage => stage instanceof Array) && query[0].length > 0 && !!query[0][0]) {
if (limit === undefined && type === "string" && !repeated && !all) {
limit = 1;
}
if (hits === null || hits === undefined) {
hits = query.length;
}
let hit = 0;
let result = undefined;
for (const stage of query) {
const subresult = this.resolveQuery({ query: stage, type, repeated, all, limit, format, pattern, distinct, negate, removeNulls, result });
if (subresult) {
result = this.mergeQueryResult(result, subresult);
if (subresult.nodes.length > 0) {
if (!all) {
this.log(`[${query.indexOf(stage) + 1}/${query.length}] STOP (first hit)`);
break;
}
if (++hit === hits) {
this.log(`[${query.indexOf(stage) + 1}/${query.length}] STOP (${hits} hits)`);
break;
}
}
}
else {
}
}
if (result && !result.raw) {
if (repeated && !Array.isArray(result.value))
result.value = [result.value];
else if (!repeated && Array.isArray(result.value) && result.value.every(value => typeof value === "string"))
result.value = result.value.length > 0 ? result.value.join(format === "singleline" ? " " : "\n") : null;
else if (!repeated && type === "boolean" && !negate && Array.isArray(result.value) && result.value.every(value => typeof value === "boolean"))
result.value = result.value.some(value => value === true);
else if (!repeated && type === "boolean" && negate && Array.isArray(result.value) && result.value.every(value => typeof value === "boolean"))
result.value = !(result.value.some(value => value === false));
else if (!repeated && Array.isArray(result.value))
result.value = result.value[0];
}
return result;
}
}
queryCheck(query, on, pattern) {
const type = pattern ? "string" : "boolean";
const all = on === "all";
const result = this.query({ query, type, pattern, all, repeated: all });
let pass = false;
if (result) {
if (type === "boolean") {
if (on === "any") {
pass = result.value === true;
}
else if (on === "all") {
pass = result.value.every(value => value === true);
}
else if (on === "none") {
pass = result.value === false;
}
}
else if (type === "string") {
if (on === "any") {
pass = !isEmpty(trim(result.value)) && result.valid !== false;
}
else if (on === "all") {
pass = result.value.every(value => !isEmpty(trim(value))) && result.valid !== false;
}
else if (on === "none") {
pass = !(!isEmpty(trim(result.value)) && result.valid !== false);
}
}
}
return [pass, result];
}
reload({ name, waitUntil, when }) {
this.yield({
name: `RELOAD ${name ? ` ${name}` : ""}`,
params: {
reload: {},
action: "reload",
waitUntil
},
when
});
}
async repeat({ name = "", actions, limit, errors = 1, when }) {
if (name)
name = " " + name;
limit = this.evaluateNumber(limit);
if (limit === undefined)
limit = 1;
if (this.when(when, `REPEAT${name}`)) {
const state = this.acquireRepeatState();
let errorOffset = 0;
while (state.index < limit) {
const label = `REPEAT${name} #${++state.index}`;
this.log(`${label} (limit=${limit})`);
const code = await this.run(actions, label, true);
if (!code) {
this.log(`${label} -> ${actions.length} steps completed`);
errorOffset = this.state.errors.length - state.errors;
if (errorOffset >= errors) {
this.appendError("error-limit", `${errorOffset} errors in repeat (error limit of ${errors} exceeded)`, 1);
break;
}
}
else {
break;
}
}
this.clearRepeatState();
this.state.vars.__metrics.steps += 1;
this.log(`REPEAT${name} ${state.index} iterations completed (limit=${limit}, errors=${errorOffset}/${errors})`);
}
else {
this.state.vars.__metrics.skipped += 1;
this.log(`REPEAT${name} SKIPPED ${when}`);
}
}
acquireRepeatState() {
const depth = this.state.vars.__step.length;
if (!this.state.vars.__repeat[depth])
this.state.vars.__repeat[depth] = {
index: 0,
errors: this.state.errors.length
};
return this.state.vars.__repeat[depth];
}
clearRepeatState() {
const depth = this.state.vars.__step.length;
this.state.vars.__repeat[depth] = undefined;
}
locator(locators) {
const activeLocators = locators.filter(locator => this.when(locator.when, `LOCATOR${locator.name ? ` ${locator.name}` : ""}`));
if (activeLocators.length > 0) {
const [locator] = activeLocators;
this.yield({
name: `LOCATOR ${locator.name ? ` ${locator.name}` : ""}${activeLocators.length > 1 ? ` (+${activeLocators.length - 1} more)` : ""}`,
params: {
locators: activeLocators.map(locator => ({
name: this.evaluateString(locator.name) || "_value",
selector: this.evaluateString(locator.selector),
method: this.evaluateString(locator.method),
params: locator.params?.map(param => this.evaluate(param)),
frame: this.evaluateString(locator.frame),
promote: locator.promote,
chain: locator.chain
})),
action: "locator",
name: locator.name,
frame: locator.frame,
selector: locator.selector,
promote: locator.promote,
method: locator.method,
arg0: locator.params ? locator.params[0] : undefined
}
});
}
}
navigate({ name, url, waitUntil, when }) {
this.yield({
name: `NAVIGATE ${name ? ` ${name}` : ""} ${url}`,
params: {
navigate: { url: this.evaluateString(url) },
action: "navigate",
url,
waitUntil
},
when
});
}
resolveOperands(operands, result) {
for (let i = 0; i < operands.length; ++i) {
if (isFormula(operands[i]) || isRegexp(operands[i])) {
this.eachNode(result, (node, value) => {
const resolvedValue = String(this.evaluate(operands[i], { value }));
if (resolvedValue !== operands[i]) {
operands[i] = resolvedValue;
}
});
}
}
}
resolveQuery({ query, type, repeated, all, limit, format, pattern, distinct, negate, removeNulls, result }) {
const obj = this.resolveQuerySelector(query, format, result?.nodes);
if (!obj)
return undefined;
const { ops, nodes, value } = obj;
if (ops.length > 0 && nodes.length > 0) {
try {
return this.resolveQueryOps({ ops, nodes, type, repeated, all, limit, format, pattern, distinct, negate, removeNulls, value });
}
catch (err) {
this.appendError("eval-error", `Failed to resolve operation for "${selectorStatement(query)}": ${err instanceof Error ? err.message : JSON.stringify(err)}`, 0);
return undefined;
}
}
else if (type === "boolean") {
let value = !repeated ? nodes.length > 0 : [nodes.length > 0];
if (negate)
value = !value;
return {
nodes,
key: this.contextKey(),
value
};
}
else if (nodes.length > 0) {
return this.formatResult({ nodes, key: this.contextKey(), value }, type, all, limit, format, pattern, distinct, negate, removeNulls);
}
else {
return undefined;
}
}
resolveQueryNodes(target, result) {
const $ = this.jquery;
if (result) {
const source = result.toArray();
const nodes = target.toArray().filter(node => !source.includes(node));
return $(nodes);
}
else {
return target;
}
}
resolveQueryOps({ ops, nodes, type, repeated, all, limit, format, pattern, distinct, negate, removeNulls, value }) {
const $ = this.jquery;
const result = { nodes, key: this.contextKey(), value };
if (!this.validateOperators(ops))
return result;
const a = ops.slice(0);
while (a.length > 0) {
const [operator, ...operands] = a.shift();
if (operator === "blank") {
result.nodes = $(result.nodes.toArray().filter(element => $(element).text().trim().length === 0));
result.value = this.text(result.nodes, format);
}
else if (operator === "cut") {
if (!this.validateOperands(operator, operands, ["string", "number"], ["number"]))
break;
const splitter = operands[0];
const n = operands[1];
const limit = operands[2];
if (typeof result.value === "string") {
result.value = cut(result.value, splitter, n, limit);
}
else if (result.value instanceof Array && result.value.every(value => typeof value === "string")) {
result.value = result.value.map(value => cut(value, splitter, n, limit));
}
else {
result.value = null;
}
}
else if (operator === "extract") {
if (!this.validateOperands(operator, operands, ["string"]))
break;
const regexp = createRegExp(operands[0]);
if (!regexp) {
this.appendError("invalid-operand", `Invalid regular expression for "extract"`, 0);
}
else if (result.value instanceof Array && result.value.every(value => typeof value === "string")) {
result.value = result.value.map(value => regexpExtract(value.trim(), regexp));
filterQueryResult($, result, value => value !== null);
}
else if (typeof result.value === "string") {
result.value = regexpExtract(result.value.trim(), regexp);
}
else {
result.value = null;
}
}
else if (operator === "extractAll") {
if (!this.validateOperands(operator, operands, ["string"], ["string"]))
break;
const regexp = createRegExp(operands[0]);
const delim = operands[1] ?? "\n";
if (!regexp) {
this.appendError("invalid-operand", `Invalid regular expression for "extractAll"`, 0);
}
else if (result.value instanceof Array && result.value.every(value => typeof value === "string")) {
result.value = result.value.map(value => regexpExtractAll(value.trim(), regexp)?.join(delim) || null);
filterQueryResult($, result, value => value !== null);
}
else if (typeof result.value === "string") {
result.value = regexpExtractAll(result.value.trim(), regexp);
if (!trim)
result.formatted = true;
}
else {
result.value = null;
}
}
else if (operator === "filter" && (isFormula(operands[0]) || isRegexp(operands[0]))) {
if (!this.validateOperands(operator, operands, ["string"]))
break;
if (result.value instanceof Array) {
const input = {
elements: result.nodes.toArray(),
values: result.value
};
const output = {
elements: [],
values: []
};
const n = input.elements.length;
for (let i = 0; i < n; ++i) {
const hit = this.evaluateBoolean(operands[0], { value: input.values[i], index: i, count: n });
if (hit) {
output.elements.push(input.elements[i]);
output.values.push(input.values[i]);
}
}
result.nodes = $(output.elements);
result.value = output.values;
}
else {
const hit = this.evaluateBoolean(operands[0], { value: result.value });
if (!hit) {
result.nodes = $([]);
result.value = null;
}
}
}
else if (operator === "html" && (operands[0] === "outer" || operands[0] === undefined)) {
if (this.online) {
result.value = result.nodes.toArray().map(element => element.outerHTML.trim());
}
else {
result.value = result.nodes.toArray().map(element => $.html(element).toString().trim());
}
if (typeof operands[1] === "boolean" ? operands[1] : true) {
result.value = formatHTML(result.value);
result.formatted = true;
}
}
else if (operator === "html" && operands[0] === "inner") {
result.value = result.nodes.toArray().map(element => $(element).html());
if (typeof operands[1] === "boolean" ? operands[1] : true) {
result.value = formatHTML(result.value);
result.formatted = true;
}
}
else if (operator === "json") {
if (!this.validateOperands(operator, operands, [], ["string"]))
break;
const formula = operands[0];
if (formula && !isFormula(formula)) {
this.appendError("invalid-operand", `Invalid formula for "json"`, 0);
break;
}
const input = {
elements: result.nodes.toArray(),
values: result.value instanceof Array ? result.value : new Array(result.nodes.length).fill(result.value)
};
const output = {
elements: [],
values: []
};
const n = input.elements.length;
for (let i = 0; i < n; ++i) {
const json = tryParseJson(input.values[i]);
if (json !== undefined) {
const value = formula ? this.evaluate(formula, { value: json, index: i, count: n }) : json;
if (value !== null && value !== undefined) {
output.elements.push(input.elements[i]);
output.values.push(value);
}
}
}
if (output.elements.length === 0) {
result.nodes = $([]);
result.value = null;
}
else if (repeated || all) {
result.nodes = $(output.elements);
result.value = output.values;
}
else {
result.nodes = $(output.elements[0]);
result.value = output.values[0];
}
result.raw = true;
}
else if (operator === "map") {
if (!this.validateOperands(operator, operands, ["string"]))
break;
const input = {
elements: result.nodes.toArray(),
values: result.value instanceof Array ? result.value : new Array(result.nodes.length).fill(result.value)
};
const output = {
elements: [],
values: []
};
const n = input.elements.length;
for (let i = 0; i < n; ++i) {
const value = this.evaluate(operands[0], { value: input.values[i], index: i, count: n });
if (value !== null && value !== undefined) {
output.elements.push(input.elements[i]);
output.values.push(value);
}
}
result.nodes = $(output.elements);
result.value = output.values;
}
else if (operator === "nonblank") {
result.nodes = $(result.nodes.toArray().filter(element => $(element).text().trim().length > 0));
result.value = this.text(result.nodes, format);
}
else if (operator === "replace") {
if (!this.validateOperands(operator, operands, ["string", "string"]))
break;
const regexp = createRegExp(operands[0]);
if (!regexp) {
this.appendError("invalid-operand", `Invalid regular expression for "replace"`, 0);
}
else if (result.value instanceof Array && result.value.every(value => typeof value === "string")) {
result.value = result.value.map(value => regexpReplace(value, regexp, operands[1]));
}
else if (typeof result.value === "string") {
result.value = regexpReplace(result.value, regexp, operands[1]);
}
}
else if (operator === "replaceHTML") {
if (!this.validateOperands(operator, operands, ["string"]))
break;
this.eachNode(result, (node, value) => {
const content = String(this.evaluate(operands[0], { value }));
node.html(content);
});
result.value = null;
}
else if (operator === "replaceTag") {
if (!this.validateOperands(operator, operands, ["string"], ["boolean"]))
break;
const newTag = String(this.evaluate(operands[0], { value: operands[0] }));
const keepProps = operands[1] ?? true;
this.eachNode(result, node => {
const newNode = $(newTag);
if (keepProps) {
mergeElements(newNode, node);
}
node.wrapAll(newNode);
node.contents().unwrap();
});
result.value = null;
}
else if (operator === "replaceText") {
if (!this.validateOperands(operator, operands, ["string"]))
break;
this.eachNode(result, (node, value) => {
const content = String(this.evaluate(operands[0], { value }));
node.text(content);
});
result.value = null;
}
else if (operator === "replaceWith") {
if (!this.validateOperands(operator, operands, ["string"]))
break;
this.eachNode(result, (node, value) => {
const content = String(this.evaluate(operands[0], { value }));
node.replaceWith(content);
});
result.value = null;
}
else if (operator === "reverse") {
if (!this.validateOperands(operator, operands, [], []))
break;
result.nodes = $(result.nodes.toArray().reverse());
result.value = this.text(result.nodes, format);
}
else if (operator === "scrollBottom") {
if (this.online) {
const y = result.nodes.scrollTop() + result.nodes.height();
this.log(`scrollBottom ${result.nodes.scrollTop()} ${result.nodes.height()} ${y}`);
window.scrollTo(0, y);
}
}
else if (operator === "size") {
result.value = result.nodes.length;
}
else if (operator === "split") {
if (!this.validateOperands(operator, operands, [], ["string", "number"]))
break;
const text = result.value instanceof Array ? result.value.join("\n") : result.value;
const separator = operands[0] || "\n";
const limit = typeof operands[1] === "number" && operands[1] >= 0 ? operands[1] : undefined;
result.value = text.split(separator, limit);
}
else if (operator === "shadow") {
result.nodes = $(result.nodes.toArray().map(element => element.shadowRoot).filter(obj => !!obj));
}
else if (operator === "slot") {
if (!this.validateOperands(operator, operands, [], ["boolean"]))
break;
const flatten = parseBoolean(operands[0]);
result.nodes = $(result.nodes.toArray().map(element => element.assignedElements({ flatten })).filter(obj => !!obj));
}
else if (operator === "text" && operands[0] === "inline") {
result.value = result.nodes.toArray().map((element) => Array.from(element.childNodes)
.filter(node => node.nodeType === 3)
.map(node => node.textContent ?? node.data)
.join(" ")
.trim()
.replace(/[ ]{2,}/, " "));
result.formatted = false;
}
else if (["appendTo", "each", "prependTo", "insertBefore", "insertAfter", "replaceAll"].includes(operator)) {
this.appendError("invalid-operator", `Operator "${operator}" not supported`, 0);
}
else if (isInvocableFrom(result.nodes