@ruhisfi/rql
Version:
`RQL` (Ruhis Query Language) is a powerful library designed to simplify the process of filtering, sorting, and aggregating large amounts of data. With RQL, you can effortlessly extract valuable insights from complex datasets, making data analysis and mani
1,257 lines (1,239 loc) • 42.4 kB
JavaScript
var __defProp = Object.defineProperty;
var __defProps = Object.defineProperties;
var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
var __getOwnPropSymbols = Object.getOwnPropertySymbols;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __propIsEnum = Object.prototype.propertyIsEnumerable;
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __spreadValues = (a, b) => {
for (var prop in b || (b = {}))
if (__hasOwnProp.call(b, prop))
__defNormalProp(a, prop, b[prop]);
if (__getOwnPropSymbols)
for (var prop of __getOwnPropSymbols(b)) {
if (__propIsEnum.call(b, prop))
__defNormalProp(a, prop, b[prop]);
}
return a;
};
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
var __async = (__this, __arguments, generator) => {
return new Promise((resolve, reject) => {
var fulfilled = (value) => {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
};
var rejected = (value) => {
try {
step(generator.throw(value));
} catch (e) {
reject(e);
}
};
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
step((generator = generator.apply(__this, __arguments)).next());
});
};
// src/dynamicField.ts
var dynamicField_default = (field, data) => {
const fieldPath = field.split(".");
for (const path of fieldPath) {
if (data == null || !Object.prototype.hasOwnProperty.call(data, path)) {
data = null;
break;
}
data = data[path];
}
return data;
};
// src/statement/AlterStatement.ts
import ipRangeCheck from "ip-range-check";
// src/statement/AbstractStatement.ts
var AbstractStatement = class {
};
// src/statement/AlterStatement.ts
var AlterStatement = class extends AbstractStatement {
execute(_query, statement, data) {
if (!statement.alter) {
throw new Error("Alter statement must have alters");
}
const alter = statement.alter;
try {
data.map((row) => {
const fieldValue = dynamicField_default(alter.parameters[0], row);
switch (alter.func) {
case "lowercase":
row[alter.field] = fieldValue.toLowerCase();
break;
case "uppercase":
row[alter.field] = fieldValue.toUpperCase();
break;
case "substring":
row[alter.field] = fieldValue.substring(
isNaN(Number(alter.parameters[1])) ? Number(dynamicField_default(alter.parameters[1], row)) : Number(alter.parameters[1]),
isNaN(Number(alter.parameters[2])) ? Number(dynamicField_default(alter.parameters[2], row)) : Number(alter.parameters[2])
);
break;
case "multiply":
row[alter.field] = fieldValue * (isNaN(Number(alter.parameters[1])) ? Number(dynamicField_default(alter.parameters[1], row)) : Number(alter.parameters[1]));
break;
case "add":
row[alter.field] = fieldValue + (isNaN(Number(alter.parameters[1])) ? Number(dynamicField_default(alter.parameters[1], row)) : Number(alter.parameters[1]));
break;
case "subtract":
row[alter.field] = fieldValue - (isNaN(Number(alter.parameters[1])) ? Number(dynamicField_default(alter.parameters[1], row)) : Number(alter.parameters[1]));
break;
case "coalesce":
for (const param of alter.parameters) {
if (dynamicField_default(param, row) !== null) {
row[alter.field] = dynamicField_default(param, row);
break;
}
}
if (row[alter.field] === void 0) {
row[alter.field] = null;
}
break;
case "incidr":
row[alter.field] = ipRangeCheck(fieldValue, alter.parameters[1]);
break;
case "split":
if (!fieldValue || typeof fieldValue !== "string")
break;
row[alter.field] = fieldValue.split(
alter.parameters[1].replace("\\,", ",")
);
break;
case "to_string":
row[alter.field] = fieldValue.toString();
break;
case "to_date":
row[alter.field] = new Date(fieldValue);
break;
case "to_number":
row[alter.field] = Number(fieldValue);
break;
case "trim": {
if (!fieldValue || typeof fieldValue !== "string" && !Array.isArray(fieldValue)) {
break;
}
if (Array.isArray(fieldValue)) {
row[alter.field] = fieldValue.map((value) => value.trim());
} else {
row[alter.field] = fieldValue.trim();
}
break;
}
case "length": {
if (!fieldValue || typeof fieldValue !== "string" && !Array.isArray(fieldValue)) {
break;
}
if (Array.isArray(fieldValue)) {
row[alter.field] = fieldValue.length;
} else {
row[alter.field] = fieldValue.toString().length;
}
break;
}
case "get": {
if (!fieldValue) {
row[alter.field] = null;
break;
}
row[alter.field] = fieldValue;
break;
}
case "get_array": {
if (!fieldValue || !Array.isArray(fieldValue)) {
row[alter.field] = null;
break;
}
if (alter.parameters[1] === "-1") {
row[alter.field] = fieldValue[fieldValue.length - 1];
break;
}
row[alter.field] = fieldValue[Number(alter.parameters[1])];
break;
}
case "base64_encode":
row[alter.field] = Buffer.from(fieldValue).toString("base64");
break;
case "base64_decode":
row[alter.field] = Buffer.from(fieldValue, "base64").toString(
"utf-8"
);
break;
case "round":
row[alter.field] = Math.round(fieldValue);
break;
case "ceil":
row[alter.field] = Math.ceil(fieldValue);
break;
case "floor":
row[alter.field] = Math.floor(fieldValue);
break;
case "extract_url_host": {
if (!fieldValue || typeof fieldValue !== "string")
break;
try {
const url = new URL(
!fieldValue.match(/^http[s]?:\/\//) ? "http://" + fieldValue : fieldValue
);
row[alter.field] = url.hostname;
} catch (err) {
row[alter.field] = null;
}
break;
}
case "json_parse":
try {
row[alter.field] = JSON.parse(fieldValue);
} catch (err) {
row[alter.field] = null;
}
break;
case "json_stringify":
try {
row[alter.field] = JSON.stringify(fieldValue);
} catch (err) {
row[alter.field] = null;
}
break;
default:
throw new Error(
`Invalid alter statement: '${alter.func}' with parameters '${alter.parameters.join(", ")}'`
);
}
});
} catch (e) {
throw new Error(
`Invalid alter statement: '${alter.func}' with parameters '${alter.parameters.join(", ")}'`
);
}
return data;
}
parse(query, statement) {
const pattern = /^(?:alter)?\s*([a-zA-Z_]\w*)\s*=\s*([a-zA-Z_]\w*)\((.*?)\)$/;
const match = statement.match(pattern);
if (!match) {
throw new Error("Invalid expression format");
}
const [, variableName, functionName, parameters] = match;
query.statements.push({
type: "alter",
alter: {
field: variableName,
func: functionName,
parameters: parameters.split(new RegExp("(?<!\\\\),")).map((param) => param.trim())
}
});
}
};
// src/statement/DedupStatement.ts
var DedupStatement = class extends AbstractStatement {
execute(_query, statement, data) {
if (!statement.dedup) {
throw new Error("Dedup statement must have dedup");
}
const { fields, sortBy, sortDirection } = statement.dedup;
const dedupedResults = /* @__PURE__ */ new Map();
for (const row of data) {
const compositeKeyParts = fields.map((field) => {
const fieldValue = dynamicField_default(field, row);
return fieldValue !== null && fieldValue !== void 0 ? fieldValue.toString() : "";
});
if (!compositeKeyParts.includes("")) {
const compositeKey = compositeKeyParts.join("|");
if (!dedupedResults.has(compositeKey) || sortBy && sortDirection) {
const existingRow = dedupedResults.get(compositeKey);
if (!existingRow || sortDirection === "desc" && dynamicField_default(sortBy || "", row) > dynamicField_default(sortBy || "", existingRow)) {
dedupedResults.set(compositeKey, row);
}
}
}
}
data = Array.from(dedupedResults.values());
return data;
}
parse(query, statement) {
statement = statement.substring(5).trim();
const stmtParts = statement.split(" ");
let fields = [];
let sortBy = void 0;
let sortDirection = void 0;
const byIndex = stmtParts.indexOf("by");
if (byIndex !== -1) {
fields = stmtParts.slice(0, byIndex).map((field) => field.replace(/,$/, ""));
if (stmtParts.length > byIndex + 2) {
sortBy = stmtParts[byIndex + 1];
sortDirection = stmtParts[byIndex + 2].toLowerCase();
if (sortDirection !== "asc" && sortDirection !== "desc") {
throw new Error(`Invalid dedup direction: '${sortDirection}'`);
}
} else {
throw new Error("sortBy and sortDirection expected after 'by'");
}
} else {
fields = stmtParts;
}
if (fields.length === 0) {
throw new Error("No fields specified for dedup");
}
fields = fields.map((f) => f.trim().replace(/,$/, ""));
query.statements.push({
type: "dedup",
dedup: {
fields,
sortBy,
sortDirection
}
});
}
};
// src/statement/FieldsStatement.ts
var FieldsStatement = class extends AbstractStatement {
execute(_query, statement, data) {
data = data.map((row) => {
if (!statement.fields) {
throw new Error("Fields statement must have fields");
}
const result = {};
for (const field of statement.fields) {
const { name, alias } = field;
result[alias || name] = row[name];
}
return result;
});
return data;
}
parse(query, statement) {
statement = statement.substring(6).trim();
const fields = statement.split(",");
const fieldsStat = fields.map((s) => {
const [name, alias] = s.trim().split(/\s+as\s+/i);
if (alias) {
return { name, alias: alias || name };
}
return { name };
});
query.statements.push({ type: "fields", fields: fieldsStat });
}
};
// src/statement/FilterStatement.ts
import ipRangeCheck2 from "ip-range-check";
// src/statement/ConfigStatement.ts
var ConfigStatement = class extends AbstractStatement {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
execute(query, _statement, _data) {
let caseSensitive = true;
let grouping = null;
if (query.config && query.config.length > 0) {
for (const config of query.config) {
if (config.key === "case_sensitive") {
caseSensitive = config.value === "true";
} else if (config.key === "grouping") {
grouping = config.value;
}
}
}
return { caseSensitive, grouping };
}
parse(query, statement) {
const config = statement.substring(6).trim();
const parts = config.match(
/(['"][^'"]+['"]|\S+)\s*(=)\s*(['"][^'"]+['"]|\S+)/i
);
if (!parts) {
throw new Error(`Invalid config statement: '${statement}'`);
}
const key = parts[1].replace(/^['"]|['"]$/g, "");
const value = parts[3].replace(/^['"]|['"]$/g, "");
if (!key || !value) {
throw new Error(`Invalid config statement: '${statement}'`);
}
query.config.push({
key: key.trim(),
value: value.trim()
});
}
};
// src/statement/FilterStatement.ts
var FilterStatement = class extends AbstractStatement {
execute(query, statement, data) {
const { caseSensitive } = new ConfigStatement().execute(
query,
statement,
data
);
data = data.filter((row) => {
let filterResult = false;
if (!statement.filter) {
throw new Error("Filter statement is missing filter object");
}
for (const block of statement.filter.blocks) {
let blockResult = true;
for (const expression of block.expressions) {
const { field, operator } = expression;
let { value } = expression;
let rowValue = dynamicField_default(field, row);
if (rowValue === null && operator !== "notIn" && value.toString().toLowerCase() !== "null" && value.toString().toLowerCase() !== "undefined") {
blockResult = false;
break;
}
const isoRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?$/;
if (typeof rowValue === "object" && rowValue instanceof Date) {
if (typeof value === "string" && new Date(value).toString() !== "Invalid Date") {
value = new Date(value);
}
} else if (isoRegex.test(rowValue)) {
rowValue = new Date(rowValue);
if (typeof value === "string" && new Date(value).toString() !== "Invalid Date") {
value = new Date(value);
}
}
const relativeDateRegex = /^(-)?(\d+)([dhms])/;
const relativeDateMatch = value.toString().trim().match(relativeDateRegex);
if (relativeDateMatch) {
const now = /* @__PURE__ */ new Date();
const relativeDate = new Date(now);
const sign = relativeDateMatch[1] === "-" ? -1 : 1;
const dateVal = parseInt(relativeDateMatch[2], 10);
const unit = relativeDateMatch[3];
switch (unit) {
case "d":
relativeDate.setDate(relativeDate.getDate() + sign * dateVal);
break;
case "h":
relativeDate.setHours(relativeDate.getHours() + sign * dateVal);
break;
case "m":
relativeDate.setMinutes(
relativeDate.getMinutes() + sign * dateVal
);
break;
case "s":
relativeDate.setSeconds(
relativeDate.getSeconds() + sign * dateVal
);
break;
default:
throw new Error(`Invalid relative date unit: '${unit}'`);
}
value = relativeDate;
}
operatorSwitch:
switch (operator) {
case "equals": {
if (value.toString().toLowerCase() === "null" || value.toString().toLowerCase() === "undefined") {
blockResult = rowValue === null || rowValue === void 0;
break operatorSwitch;
}
if (!caseSensitive) {
blockResult = rowValue === value || rowValue.toString().toLowerCase() === value.toString().toLowerCase();
break;
}
blockResult = rowValue === value || rowValue.toString() === value.toString();
break;
}
case "notEquals": {
if (value.toString().toLowerCase() === "null" || value.toString().toLowerCase() === "undefined") {
blockResult = rowValue !== null && rowValue !== void 0;
break operatorSwitch;
}
if (!caseSensitive) {
blockResult = rowValue !== value && rowValue.toString().toLowerCase() !== value.toString().toLowerCase();
break;
}
blockResult = rowValue !== value && rowValue.toString() !== value.toString();
break;
}
case "contains":
if (!caseSensitive) {
blockResult = rowValue.includes(value) || rowValue.toString().toLowerCase().includes(value.toString().toLowerCase());
break;
}
blockResult = rowValue.includes(value) || rowValue.toString().includes(value.toString());
break;
case "notContains":
if (!caseSensitive) {
blockResult = !rowValue.includes(value) && !rowValue.toString().toLowerCase().includes(value.toString().toLowerCase());
break;
}
blockResult = !rowValue.includes(value) || !rowValue.toString().includes(value.toString());
break;
case "lessThan":
blockResult = rowValue < value;
break;
case "greaterThan":
blockResult = rowValue > value;
break;
case "lessThanOrEquals":
blockResult = rowValue <= value;
break;
case "greaterThanOrEquals":
blockResult = rowValue >= value;
break;
case "matches":
try {
const regex = new RegExp(value.toString());
blockResult = regex.test(rowValue.toString());
} catch (e) {
throw new Error(`Invalid regex pattern: '${value}'`);
}
break;
case "incidr":
blockResult = ipRangeCheck2(rowValue, value.toString());
break;
case "notIncidr":
blockResult = !ipRangeCheck2(rowValue, value.toString());
break;
case "in":
blockResult = value.includes(`${rowValue}`);
break;
case "notIn":
blockResult = !value.includes(`${rowValue}`);
break;
default:
throw new Error(`Invalid operator: '${operator}'`);
}
if (!blockResult) {
break;
}
}
if (blockResult) {
filterResult = true;
break;
}
}
return filterResult;
});
return data;
}
parse(query, statement) {
const filter = statement.substring(6).trim();
const parsedFilter = this.parseFilter(filter);
query.statements.push({ type: "filter", filter: parsedFilter });
}
parseFilter(filter) {
const blocks = filter.split(" or ").map((s) => s.trim());
const filters = blocks.map((b) => this.parseFilterBlock(b));
return { blocks: filters };
}
parseFilterBlock(block) {
const expressions = block.split(" and ").map((s) => s.trim());
const filters = expressions.map((e) => this.parseFilterExpression(e));
return { expressions: filters };
}
parseFilterExpression(expression) {
const operators = [
// Make sure that any string operators have a space after them to avoid accidental matching with field names
"matches ",
"not contains ",
"contains ",
"not incidr ",
"incidr ",
"not in ",
"in ",
"~=",
"!=",
"<=",
">=",
"<",
">",
"="
// This should be last to avoid premature matching
];
const operatorsRegexParts = operators.map(
(op) => (
// Escape special regex characters and sort by length to prioritize matching longer operators first
op.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&")
)
).sort((a, b) => b.length - a.length);
const regex = new RegExp(
`(.+?)\\s*(${operatorsRegexParts.join("|")})\\s*(.+)`
);
const match = expression.match(regex);
if (!match) {
throw new Error(`Invalid filter expression: '${expression}'`);
}
const field = match[1].trim();
const operator = match[2].trim();
const value = match[3].trim().replace(/^['"](.*)['"]$/, "$1");
switch (operator) {
case "=":
return {
field,
operator: "equals",
value: this.parseFilterValue(value)
};
case "!=":
return {
field,
operator: "notEquals",
value: this.parseFilterValue(value)
};
case "~=":
case "matches":
return {
field,
operator: "matches",
value: this.parseFilterValue(value)
};
case "contains":
return {
field,
operator: "contains",
value: this.parseFilterValue(value)
};
case "not contains":
return {
field,
operator: "notContains",
value: this.parseFilterValue(value)
};
case "<":
return {
field,
operator: "lessThan",
value: this.parseFilterValue(value)
};
case ">":
return {
field,
operator: "greaterThan",
value: this.parseFilterValue(value)
};
case "<=":
return {
field,
operator: "lessThanOrEquals",
value: this.parseFilterValue(value)
};
case ">=":
return {
field,
operator: "greaterThanOrEquals",
value: this.parseFilterValue(value)
};
case "incidr":
return {
field,
operator: "incidr",
value: this.parseFilterValue(value)
};
case "not incidr":
return {
field,
operator: "notIncidr",
value: this.parseFilterValue(value)
};
case "in":
return {
field,
operator: "in",
value: this.parseListFilterValue(value)
};
case "not in":
return {
field,
operator: "notIn",
value: this.parseListFilterValue(value)
};
default:
throw new Error(`Invalid filter expression: '${expression}'`);
}
}
parseFilterValue(value) {
const v4 = new RegExp(
/^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
);
if (value.toString().match(v4)) {
return value.toString();
} else if (value === "true" || value === "false") {
return value === "true";
} else if (!isNaN(Number(value))) {
return Number(value);
} else if (value === "date()" || value === "now()") {
return /* @__PURE__ */ new Date();
} else {
return value;
}
}
parseListFilterValue(value) {
let parsedValue = this.parseFilterValue(value);
if (typeof parsedValue !== "string") {
throw new Error(`Unsupported filter value: '${value}'`);
}
if (!parsedValue.startsWith("(") || !parsedValue.endsWith(")")) {
throw new Error(`Unsupported filter value: '${value}'`);
}
parsedValue = parsedValue.substring(1, parsedValue.length - 1);
const parsedList = parsedValue.split(",").map((v) => v.trim().replace(/^['"](.*)['"]$/, "$1"));
return parsedList;
}
};
// src/statement/LimitStatement.ts
var LimitStatement = class extends AbstractStatement {
execute(_query, statement, data) {
if (statement.limit === void 0 || statement.limit === null) {
throw new Error("Limit statement must have limit");
}
data = data.slice(0, statement.limit);
return data;
}
parse(query, statement) {
const parts = statement.split(" ");
if (parts.length !== 2) {
throw new Error(`Invalid limit statement: '${statement}'`);
}
const limit = Number(parts[1]);
if (isNaN(limit)) {
throw new Error(`Invalid limit statement: '${statement}'`);
}
query.statements.push({ type: "limit", limit });
}
};
// src/statement/SearchStatement.ts
var SearchStatement = class extends AbstractStatement {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
execute(query, statement, data) {
const searchKey = statement.search;
const { caseSensitive } = new ConfigStatement().execute(
query,
statement,
data
);
if (!searchKey) {
return data;
}
if (caseSensitive) {
return data.filter(
(row) => Object.values(row).some(
(value) => typeof value === "string" ? value.includes(searchKey) : false
)
);
}
return data.filter(
(row) => Object.values(row).some(
(value) => typeof value === "string" ? value.toLowerCase().includes(searchKey.toLowerCase()) : false
)
);
}
parse(query, statement) {
statement = statement.substring(6).trim();
query.statements.push({
type: "search",
search: statement.replace(/^['"](.*)['"]$/, "$1")
});
}
};
// src/dynamicSort.ts
var dynamicSort_default = (properties) => {
return (a, b) => {
for (let i = 0; i < properties.length; i += 1) {
const sortOrder = properties[i][0] === "-" ? -1 : 1;
const property = sortOrder === -1 ? properties[i].slice(1) : properties[i];
const propertyPath = property.split(".");
let propertyValueA = a;
let propertyValueB = b;
let j = 0;
while (propertyValueA && propertyValueB && j < propertyPath.length) {
propertyValueA = propertyValueA[propertyPath[j]];
propertyValueB = propertyValueB[propertyPath[j]];
j += 1;
}
if (propertyValueA < propertyValueB) {
return -1 * sortOrder;
}
if (propertyValueA > propertyValueB) {
return 1 * sortOrder;
}
}
return 0;
};
};
// src/statement/SortStatement.ts
var SortStatement = class extends AbstractStatement {
execute(_query, statement, data) {
if (!statement.sort) {
throw new Error("Sort statement must have sort");
}
const sorts = statement.sort.map((s) => {
if (s.direction === "desc") {
return `-${s.field}`;
}
return s.field;
});
data = data.sort(dynamicSort_default(sorts));
return data;
}
parse(query, statement) {
let sorts = statement.split(",");
sorts[0] = sorts[0].replace("sort", "").trim();
sorts = sorts.map((s) => s.trim());
const stat = { type: "sort", sort: [] };
for (const sort of sorts) {
const stmtParts = sort.trim().split(" ");
if (stmtParts.length === 1 || stmtParts.length === 2) {
const field = stmtParts[0];
const direction = stmtParts.length === 2 ? stmtParts[1].toLowerCase() : "asc";
if (direction === "asc" || direction === "desc") {
stat.sort.push({
field,
direction
});
} else {
throw new Error(`Invalid sort direction: '${direction}'`);
}
} else {
throw new Error(`Invalid sort statement: '${sort}'`);
}
}
query.statements.push(stat);
}
};
// src/statement/CompStatement.ts
var CompStatement = class extends AbstractStatement {
execute(query, statement, data) {
const groupedData = {};
const { grouping } = new ConfigStatement().execute(query, statement, data);
if (grouping) {
for (const row of data) {
const groupKey = dynamicField_default(grouping, row);
if (groupKey !== null && groupKey !== void 0) {
if (!groupedData[groupKey]) {
groupedData[groupKey] = [];
}
groupedData[groupKey].push(row);
}
}
} else {
groupedData["_all"] = data;
}
const results = [];
const compStatement = statement.comp;
if (!compStatement) {
throw new Error("Comp statement must have comp");
}
for (const [groupKey, groupData] of Object.entries(groupedData)) {
const statsRow = {};
if (grouping) {
statsRow[grouping] = groupKey;
}
for (const comp of compStatement) {
if (!comp) {
throw new Error("Comp statement must have comp");
}
const values = groupData.map((row) => dynamicField_default(comp.field, row)).filter((value) => value !== null && value !== void 0);
switch (comp.function) {
case "count":
statsRow[comp.returnField] = values.length;
break;
case "count_distinct":
statsRow[comp.returnField] = new Set(values).size;
break;
case "min":
statsRow[comp.returnField] = Math.min(...values);
break;
case "max":
statsRow[comp.returnField] = Math.max(...values);
break;
case "avg":
statsRow[comp.returnField] = values.reduce((a, b) => a + b, 0) / values.length;
break;
case "sum":
statsRow[comp.returnField] = values.reduce((a, b) => a + b, 0);
break;
case "median":
values.sort((a, b) => a - b);
const middle = Math.floor(values.length / 2);
statsRow[comp.returnField] = values.length % 2 !== 0 ? values[middle] : (values[middle - 1] + values[middle]) / 2;
break;
case "earliest":
statsRow[comp.returnField] = new Date(
Math.min(
...values.map((isoString) => new Date(isoString).getTime())
)
);
break;
case "latest":
statsRow[comp.returnField] = new Date(
Math.max(
...values.map((isoString) => new Date(isoString).getTime())
)
);
break;
case "first":
statsRow[comp.returnField] = groupData.length > 0 ? dynamicField_default(comp.field, groupData[0]) : null;
break;
case "last":
statsRow[comp.returnField] = groupData.length > 0 ? dynamicField_default(
comp.field,
groupData[groupData.length - 1]
) : null;
break;
case "to_array":
statsRow[comp.returnField] = Array.from(new Set(values));
break;
default:
throw new Error(`Invalid comp function: '${comp.function}'`);
}
}
results.push(statsRow);
}
return results;
}
parse(query, statement) {
const individualStatements = statement.split(",").map((s) => s.trim());
individualStatements[0] = individualStatements[0].substring(4).trim();
const baseStatement = {
type: "comp",
comp: []
};
for (const individualStatement of individualStatements) {
const parts = individualStatement.split(" ");
if (parts.length !== 4) {
throw new Error(`Invalid comp statement: '${individualStatement}'`);
}
const func = parts[0].trim();
const field = parts[1].trim();
const asField = parts[2].trim();
const returnField = parts[3].trim();
if (asField !== "as") {
throw new Error(`Invalid comp statement: '${individualStatement}'`);
}
baseStatement.comp.push({
function: func,
field,
returnField
});
}
query.statements.push(baseStatement);
}
};
// src/QueryExecutor.ts
var QueryExecutor = class {
/**
* Executes the provided query on the given data array.
*
* The data array consists of objects with key-value pairs representing the data.
*
* If any field or operator is not found in a row of data, an error will be thrown.
*
* @param {Query} query The query object containing fields, alters, filters, sort, and limit properties.
* @param {Array} data The data to be queried, as an array of objects.
* @returns {Array} The result of the query execution, as an array of objects.
* @throws {Error} If any field in the query is not found in the data, or an invalid operator is used.
* @public
* @static
*/
static executeQuery(query, data) {
let results = data;
for (const statement of query.statements) {
switch (statement.type) {
case "dataset":
case "config":
break;
case "filter":
results = new FilterStatement().execute(query, statement, results);
break;
case "fields":
results = new FieldsStatement().execute(query, statement, results);
break;
case "sort":
results = new SortStatement().execute(query, statement, results);
break;
case "limit":
results = new LimitStatement().execute(query, statement, results);
break;
case "dedup":
results = new DedupStatement().execute(query, statement, results);
break;
case "alter":
results = new AlterStatement().execute(query, statement, results);
break;
case "comp":
results = new CompStatement().execute(query, statement, results);
break;
case "search":
results = new SearchStatement().execute(query, statement, results);
break;
default:
throw new Error(`Invalid statement type: '${statement.type}'`);
}
}
return results;
}
static buildElasticsearchExpression(expr) {
if (expr.field.includes(".")) {
return null;
}
switch (expr.operator) {
case "equals":
return { term: { [expr.field]: expr.value } };
case "notEquals":
return {
bool: { must_not: { term: { [expr.field]: expr.value } } }
};
case "greaterThan":
return { range: { [expr.field]: { gt: expr.value } } };
case "greaterThanOrEquals":
return { range: { [expr.field]: { gte: expr.value } } };
case "lessThan":
return { range: { [expr.field]: { lt: expr.value } } };
case "lessThanOrEquals":
return { range: { [expr.field]: { lte: expr.value } } };
case "contains":
return null;
case "notContains":
return null;
case "matches":
return { regexp: { [expr.field]: expr.value } };
case "incidr":
return null;
case "notIncidr":
return null;
default:
return null;
}
}
static buildElasticsearchFilter(statement) {
if (statement.type !== "filter" || !statement.filter)
return null;
const filter = statement.filter;
const blockFilters = filter.blocks.map((block) => {
const expressionFilters = block.expressions.map(this.buildElasticsearchExpression).filter((f) => f !== null);
if (expressionFilters.length === 0)
return null;
return {
bool: {
must: expressionFilters
}
};
}).filter((f) => f !== null);
if (blockFilters.length === 0)
return null;
return blockFilters.length === 1 ? blockFilters[0] : {
bool: {
should: blockFilters
}
};
}
static buildElasticsearchDedup(statement) {
if (!statement.dedup)
return null;
const { fields, sortBy, sortDirection } = statement.dedup;
if (fields.length === 1) {
const collapse = {
field: fields[0]
};
if (sortBy) {
return {
collapse,
sort: [
{
[sortBy]: {
order: sortDirection || "desc"
}
}
]
};
}
return { collapse };
}
return null;
}
/**
* WIP: Executes the provided query on the Elasticsearch client and index.
*
* The data array consists of objects with key-value pairs representing the data.
*
* If any field or operator is not found in a row of data, an error will be thrown.
*
* @deprecated This function is still under development.
* @param {Query} query The query object containing fields, alters, filters, sort, and limit properties.
* @param {Array} data The data to be queried, as an Elasticsearch response.
* @returns {Array} The result of the query execution, as an array of objects.
* @throws {Error} If any field in the query is not found in the data, or an invalid operator is used.
* @public
* @static
*/
static executeElasticQuery(client, index, query) {
return __async(this, null, function* () {
const body = {
query: {
bool: {
must: []
}
},
size: 1e3
};
const remainingStatements = [];
const elasticFilters = [];
let elasticDedup = null;
let useScroll = true;
for (const statement of query.statements) {
if (statement.type === "filter") {
const elasticFilter = this.buildElasticsearchFilter(statement);
if (elasticFilter) {
elasticFilters.push(elasticFilter);
} else {
remainingStatements.push(statement);
}
} else if (statement.type === "dedup") {
elasticDedup = this.buildElasticsearchDedup(statement);
if (!elasticDedup) {
remainingStatements.push(statement);
} else {
useScroll = false;
}
} else {
remainingStatements.push(statement);
}
}
if (elasticFilters.length > 0) {
body.query.bool.must = elasticFilters;
}
if (elasticDedup) {
if (elasticDedup.collapse) {
body.collapse = elasticDedup.collapse;
}
if (elasticDedup.sort) {
body.sort = elasticDedup.sort;
}
}
const allResults = [];
let scrollId;
try {
if (useScroll) {
let response = yield client.search(__spreadValues({
index,
scroll: "1m"
}, body));
scrollId = response._scroll_id;
allResults.push(...response.hits.hits);
while (response.hits.hits.length) {
response = yield client.scroll({
scroll_id: scrollId,
scroll: "1m"
});
scrollId = response._scroll_id;
allResults.push(...response.hits.hits);
}
} else {
let from = 0;
const size = 1e3;
while (true) {
const response = yield client.search(__spreadProps(__spreadValues({
index
}, body), {
from,
size
}));
allResults.push(...response.hits.hits);
if (response.hits.hits.length < size) {
break;
}
from += size;
}
}
} catch (err) {
throw new Error(`Elasticsearch error: ${err.message}`);
} finally {
if (scrollId) {
yield client.clearScroll({ scroll_id: scrollId });
}
}
const data = allResults.map((hit) => __spreadValues({
_id: hit._id
}, hit._source));
const remainingQuery = __spreadProps(__spreadValues({}, query), {
statements: remainingStatements
});
return this.executeQuery(remainingQuery, data);
});
}
};
// src/statement/DatasetStatement.ts
var DatasetStatement = class extends AbstractStatement {
execute(_query, _statement, _data) {
throw new Error("Method not implemented.");
}
parse(query, statement) {
if (statement.split("=").length !== 2) {
throw new Error(`Invalid dataset statement: '${statement}'`);
}
query.dataset = statement.split("=")[1].trim();
}
};
// src/QueryParser.ts
var QueryParser = class {
/**
* Parses the provided query string into a Query object.
*
* Example query string:
* `dataset = myDataset | filter price > 100 | sort price desc, name asc | fields name, price as cost, description | limit 10`
*
* The returned Query object has properties matching the parsed elements of the query string. If any part of the query string cannot be parsed correctly, an error will be thrown.
*
* @param {string} queryString The query string to be parsed.
* @param {QueryParsingOptions} options Options for parsing the query string.
* @returns {Query} The parsed query as a Query object.
* @throws {Error} If any part of the query string cannot be parsed, or if no dataset is specified.
* @public
* @static
*/
static parseQuery(queryString, options = { strictDataset: true }) {
const query = {
dataset: "",
config: [],
statements: []
};
const queryLines = queryString.split("\n");
let constructedQuery = "";
queryLines.map((l) => {
if (l.startsWith("#") || l.startsWith("//")) {
return;
}
constructedQuery += l;
});
const queryParts = constructedQuery.split("|");
for (const part of queryParts) {
const statement = part.trim();
if (statement.startsWith("dataset")) {
new DatasetStatement().parse(query, statement);
} else if (statement.startsWith("filter")) {
new FilterStatement().parse(query, statement);
} else if (statement.startsWith("fields")) {
new FieldsStatement().parse(query, statement);
} else if (statement.startsWith("sort")) {
new SortStatement().parse(query, statement);
} else if (statement.startsWith("limit")) {
new LimitStatement().parse(query, statement);
} else if (statement.startsWith("dedup")) {
new DedupStatement().parse(query, statement);
} else if (statement.startsWith("alter")) {
new AlterStatement().parse(query, statement);
} else if (statement.startsWith("config")) {
new ConfigStatement().parse(query, statement);
} else if (statement.startsWith("comp")) {
new CompStatement().parse(query, statement);
} else if (statement.startsWith("search")) {
new SearchStatement().parse(query, statement);
} else {
throw new Error(`Invalid statement: '${statement}'`);
}
}
if (!query.dataset && options.strictDataset) {
throw new Error("No dataset specified");
}
return query;
}
};
export {
QueryExecutor,
QueryParser
};