jw-datamaster
Version:
A powerful JavaScript Swiss Army Knife for database-style data manipulation with SQL-like querying, filtering, sorting, and transformation capabilities. Supports recordsets, tables, CSV, and advanced query operations including WHERE clauses, ORDER BY, pag
1,265 lines (1,097 loc) • 143 kB
JavaScript
/**
* ver 5.1.0 25/07/16
* - add slice
* - add splice
* ver 5.0.1 25/07/10
* - remove refreces to table generator
* ver 5.0.0 25/07/10
* - total refactor and major upgrade
**/
(function(global) {
'use strict';
// --- Helper Functions (Stateless, defined outside of class) ---
/**
* Creates a deep copy of data using JSON serialization
* @param {*} data - Data to copy
* @returns {*} Deep copy of the data or error indicator
*/
function deepCopy(data) {
if (data === undefined) {
return undefined;
}
if (data === null) {
return null;
}
try {
return JSON.parse(JSON.stringify(data));
} catch (error) {
return { error: true, message: 'Failed to create deep copy: ' + error.message };
}
}
/**
* Converts a recordset to a table format
* @param {Array<Object>} recordset - Array of objects
* @returns {Object} Object with fields array and table array, or error indicator
*/
function recordsetToTable(recordset) {
if (!Array.isArray(recordset)) {
return { error: true, message: 'Input must be an array' };
}
if (recordset.length === 0) {
return { fields: [], table: [] };
}
if (typeof recordset[0] !== 'object' || recordset[0] === null) {
return { error: true, message: 'Array elements must be objects' };
}
const fields = Object.keys(recordset[0]);
const table = [];
for (let i = 0; i < recordset.length; i++) {
if (typeof recordset[i] !== 'object' || recordset[i] === null) {
return { error: true, message: 'All array elements must be objects' };
}
const row = [];
for (let j = 0; j < fields.length; j++) {
const fieldName = fields[j];
if (recordset[i].hasOwnProperty(fieldName)) {
row.push(recordset[i][fieldName]);
} else {
row.push(null);
}
}
table.push(row);
}
return { fields, table };
}
/**
* Converts a table to a recordset format
* @param {Array<Array>} table - 2D array of data
* @param {Array<string>} fields - Field names
* @returns {Array<Object>} Array of objects or error indicator
*/
function tableToRecordset(table, fields) {
if (!Array.isArray(table)) {
return { error: true, message: 'Table must be an array' };
}
if (!Array.isArray(fields)) {
return { error: true, message: 'Fields must be an array' };
}
if (table.length === 0) {
return [];
}
const recordset = [];
for (let row = 0; row < table.length; row++) {
if (!Array.isArray(table[row])) {
return { error: true, message: 'All table rows must be arrays' };
}
const record = {};
for (let field = 0; field < fields.length; field++) {
if (typeof fields[field] !== 'string') {
return { error: true, message: 'All field names must be strings' };
}
record[fields[field]] = table[row][field] !== undefined ? table[row][field] : null;
}
recordset.push(record);
}
return recordset;
}
/**
* Converts CSV string to table format
* @param {string} csvString - CSV data as string
* @param {Object} options - Parsing options
* @returns {Object} Object with fields array and table array, or error indicator
*/
function csvToTable(csvString, options = {}) {
if (typeof csvString !== 'string') {
return { error: true, message: 'CSV input must be a string' };
}
if (csvString.length === 0) {
return { fields: [], table: [] };
}
const isTSV = options.isTSV || false;
const headersInFirstRow = options.headersInFirstRow || false;
const sep = isTSV ? '\t' : ',';
const cr = options.noCR ? '\n' : '\r\n';
const table = [];
let cell = '';
let row = [];
let started = false;
let protectedMode = false;
let cursor = 0;
function isChar(str) {
let test = '';
const l = str.length;
for (let i = 0; i < l; i++) {
test += csvString[cursor + i];
}
if (str === test) {
cursor += l;
return true;
}
return false;
}
while (cursor < csvString.length) {
if (started) {
if (protectedMode) {
if (isChar('\"' + sep)) {
row.push(cell);
cell = '';
started = false;
protectedMode = false;
} else if (isChar('"' + cr)) {
row.push(cell);
cell = '';
table.push(row);
row = [];
started = false;
protectedMode = false;
} else if (isChar('\"\"')) {
cell += '"';
} else {
cell += csvString[cursor];
cursor++;
}
} else {
if (isChar(sep)) {
row.push(cell);
cell = '';
started = false;
protectedMode = false;
} else if (isChar(cr)) {
row.push(cell);
cell = '';
table.push(row);
row = [];
started = false;
protectedMode = false;
} else if (isChar('""')) {
cell += '"';
} else {
cell += csvString[cursor];
cursor++;
}
}
} else {
if (isChar('"')) {
protectedMode = true;
started = true;
} else if (isChar(sep)) {
row.push(cell);
cell = '';
started = false;
protectedMode = false;
} else if (isChar(cr)) {
table.push(row);
row = [];
cell = '';
started = false;
protectedMode = false;
} else {
cell = csvString[cursor];
started = true;
protectedMode = false;
cursor++;
}
}
}
// Handle any remaining data
if (cell || row.length > 0) {
if (cell) row.push(cell);
if (row.length > 0) table.push(row);
}
let fields = [];
let dataTable = table;
if (headersInFirstRow && table.length > 0) {
fields = table[0];
dataTable = table.slice(1);
// Validate field names
for (let i = 0; i < fields.length; i++) {
if (fields[i] === null || fields[i] === undefined) {
fields[i] = 'Field' + i;
} else {
fields[i] = fields[i].toString();
}
}
} else {
// Generate numeric field names
if (table.length > 0) {
for (let i = 0; i < table[0].length; i++) {
fields.push(i.toString());
}
}
}
return { fields, table: dataTable };
}
/**
* Converts table data to CSV string
* @param {Array<Array>} table - 2D array of data
* @param {Array<string>} fields - Field names
* @param {Object} options - Export options
* @returns {string} CSV formatted string or error indicator
*/
function tableToCsv(table, fields, options = {}) {
if (!Array.isArray(table)) {
return { error: true, message: 'Table must be an array' };
}
if (!Array.isArray(fields)) {
return { error: true, message: 'Fields must be an array' };
}
const newLineString = options.newLineString || '\r\n';
const includeHeaders = options.includeHeaders !== false;
function escape(val) {
if (val === null || val === undefined) {
return '""';
}
try {
val = val.toString();
// Replace quotes with double quotes
val = val.replace(/"/g, '""');
// Wrap in quotes if contains comma, newline, or quotes
if (val.includes(',') || val.includes('\n') || val.includes('\r') || val.includes('"')) {
val = '"' + val + '"';
}
return val;
} catch (error) {
return '""'; // Return empty quoted string on conversion failure
}
}
let csv = '';
if (includeHeaders && fields.length > 0) {
csv += fields.map(escape).join(',') + newLineString;
}
for (let r = 0; r < table.length; r++) {
if (!Array.isArray(table[r])) {
return { error: true, message: 'All table rows must be arrays' };
}
const row = [];
for (let c = 0; c < table[r].length; c++) {
row.push(escape(table[r][c]));
}
csv += row.join(',') + newLineString;
}
return csv;
}
/**
* Checks if an item is in a list using soft equivalence (1 == '1')
* @param {Array} list - The list to check against
* @param {*} item - The item to find
* @returns {boolean} True if item is found
*/
function itemInList(list, item) {
if (!Array.isArray(list)) {
return false;
}
for (let i = 0; i < list.length; i++) {
if (list[i] == item) { // Soft check
return true;
}
}
return false;
}
/**
* Returns a single column from a table
* @param {Array<Array>} table - 2D array of data
* @param {number} column - The index of the column to return
* @param {boolean} [distinct=false] - Whether to return only unique values
* @returns {Array} Array of column values
*/
function getTableColumn(table, column, distinct = false) {
if (!Array.isArray(table)) {
return [];
}
const col = [];
try {
for (let row = 0; row < table.length; row++) {
if (!Array.isArray(table[row]) || column >= table[row].length) {
continue;
}
if (distinct) {
if (!itemInList(col, table[row][column])) {
col.push(table[row][column]);
}
} else {
col.push(table[row][column]);
}
}
return col;
} catch (error) {
return [];
}
}
/**
* Returns a single row from a table
* @param {Array<Array>} table - 2D array of data
* @param {number} rowIndex - The index of the row to return
* @returns {Array|null} Array of row values or null if not found
*/
function getTableRow(table, rowIndex) {
if (!Array.isArray(table)) {
return null;
}
if (rowIndex < 0 || rowIndex >= table.length) {
return null;
}
try {
const row = table[rowIndex];
if (!Array.isArray(row)) {
return null;
}
// Return a copy of the row to prevent external modification
return [...row];
} catch (error) {
return null;
}
}
/**
* Tests if a row is a duplicate based on specified column indexes
* @param {Array} rowData - The row to test
* @param {Array<Array>} existingTable - The table to check against
* @param {Array<number>} columnIndexes - The column indexes to compare
* @returns {boolean} True if the row is a duplicate
*/
function isRowDuplicate(rowData, existingTable, columnIndexes) {
if (!Array.isArray(rowData) || !Array.isArray(existingTable) || !Array.isArray(columnIndexes)) {
return false;
}
for (let existingRow = 0; existingRow < existingTable.length; existingRow++) {
if (!Array.isArray(existingTable[existingRow])) {
continue; // Skip invalid rows
}
let isMatch = true;
// Check all specified columns for a match
for (let col = 0; col < columnIndexes.length; col++) {
const columnIndex = columnIndexes[col];
// Use soft comparison to match original behavior
if (existingTable[existingRow][columnIndex] != rowData[columnIndex]) {
isMatch = false;
break;
}
}
if (isMatch) {
return true; // Found a duplicate
}
}
return false; // No duplicate found
}
/**
* Generates a comparator function for sorting an array of arrays based on
* multiple fields and corresponding sort orders.
* @param {Array<number>} fields - Array of column indices to sort by
* @param {Array<boolean>} directions - Array of booleans where true = descending, false = ascending
* @param {Array<Function>} [primers] - Optional array of functions to transform each field before comparison
* @returns {Function} Comparator function for Array.prototype.sort
*/
function multiFieldSort(fields, directions, primers) {
if (!Array.isArray(fields) || !Array.isArray(directions)) {
return function() { return 0; }; // Return neutral comparator if invalid inputs
}
return function(a, b) {
// Validate that both items are arrays
if (!Array.isArray(a) || !Array.isArray(b)) {
return 0;
}
// Iterate over each field provided
for (let i = 0; i < fields.length; i++) {
const field = fields[i];
const isDescending = directions[i] || false;
const primer = primers && primers[i];
// Determine sort order (1 for ascending, -1 for descending)
const sortOrder = isDescending ? -1 : 1;
// Get values from both arrays
let valueA = a[field];
let valueB = b[field];
// Apply primer function if provided
if (typeof primer === 'function') {
try {
valueA = primer(valueA);
valueB = primer(valueB);
} catch (error) {
// If primer fails, continue with original values
valueA = a[field];
valueB = b[field];
}
}
// Handle null/undefined values - sort them to the end
if (valueA == null && valueB == null) {
continue; // Both null, check next field
}
if (valueA == null) {
return 1; // Null values go to end regardless of sort order
}
if (valueB == null) {
return -1; // Null values go to end regardless of sort order
}
// Perform the comparison for the current field
if (valueA > valueB) {
return sortOrder;
} else if (valueA < valueB) {
return -sortOrder;
}
// If values are equal, continue to next field
}
// All fields are equal
return 0;
};
}
/**
* Validates and converts field references to column indexes
* @param {Array<string>} fieldNames - The field names array
* @param {Array<string|number>} fields - Fields to validate and convert
* @returns {Object} Object with {success: boolean, indexes: Array<number>, error: string}
*/
function validateAndConvertFields(fieldNames, fields) {
if (!Array.isArray(fieldNames) || !Array.isArray(fields)) {
return { success: false, indexes: [], error: 'Invalid field names or fields array' };
}
const validIndexes = [];
for (let i = 0; i < fields.length; i++) {
let columnIndex;
if (typeof fields[i] === 'number') {
if (fields[i] < 0 || fields[i] >= fieldNames.length) {
return {
success: false,
indexes: [],
error: `Field index ${fields[i]} is out of bounds`
};
}
columnIndex = fields[i];
} else if (typeof fields[i] === 'string') {
columnIndex = fieldNames.indexOf(fields[i]);
if (columnIndex === -1) {
return {
success: false,
indexes: [],
error: `Field '${fields[i]}' not found`
};
}
} else {
return {
success: false,
indexes: [],
error: 'Field references must be strings or numbers'
};
}
validIndexes.push(columnIndex);
}
return { success: true, indexes: validIndexes, error: null };
}
/**
* Reorders table data and fields based on the specified field order
* @param {Array<Array>} table - The table data to reorder
* @param {Array<string>} fields - The current field names
* @param {Array<string|number>} order - The fields/indexes to keep and their order
* @returns {Object} Object with {success: boolean, table: Array<Array>, fields: Array<string>, error: string}
*/
function reorderTableData(table, fields, order) {
if (!Array.isArray(table) || !Array.isArray(fields) || !Array.isArray(order)) {
return {
success: false,
table: [],
fields: [],
error: 'Invalid table, fields, or order parameters'
};
}
if (order.length === 0) {
return {
success: false,
table: [],
fields: [],
error: 'Order array cannot be empty'
};
}
try {
// Validate and convert field references to indexes
const validation = validateAndConvertFields(fields, order);
if (!validation.success) {
return {
success: false,
table: [],
fields: [],
error: validation.error
};
}
const fieldIndexes = validation.indexes;
const newTable = [];
const newFields = [];
// Build new field names array
for (let i = 0; i < order.length; i++) {
if (typeof order[i] === 'number') {
// If order item was an index, use the field name at that index
newFields.push(fields[order[i]]);
} else {
// If order item was a field name, use it directly
newFields.push(order[i]);
}
}
// Reorder each row
for (let r = 0; r < table.length; r++) {
if (!Array.isArray(table[r])) {
continue; // Skip invalid rows
}
const newRow = [];
for (let i = 0; i < fieldIndexes.length; i++) {
const columnIndex = fieldIndexes[i];
newRow.push(table[r][columnIndex] !== undefined ? table[r][columnIndex] : null);
}
newTable.push(newRow);
}
return {
success: true,
table: newTable,
fields: newFields,
error: null
};
} catch (error) {
return {
success: false,
table: [],
fields: [],
error: 'Failed to reorder data: ' + error.message
};
}
}
// --- START: Upgraded Query Engine Helpers from DataQuery ---
/**
* Performs case-insensitive string comparison with wildcard support
* @param {*} value - The value to compare
* @param {*} query - The query pattern to match against
* @param {boolean} [forceCaseSensitivity=false] - Whether to force case sensitivity
* @returns {boolean} True if value matches query pattern
*/
function looseCaseInsensitiveCompare(value, query, forceCaseSensitivity = false) {
// Check for null or undefined
if (value == null || query == null) {
return false;
}
// Convert to string
value = String(value);
query = String(query);
// Make case-insensitive unless forceCaseSensitivity is true
if (!forceCaseSensitivity) {
value = value.toLowerCase();
query = query.toLowerCase();
}
// Handle wildcards
let regexStr = '';
for (let i = 0; i < query.length; i++) {
if (query[i] === '%') {
regexStr += '.*';
} else if (query[i] === '_') {
regexStr += '.';
} else {
// Escape special regex characters
regexStr += query[i].replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
}
try {
const regex = new RegExp(`^${regexStr}$`);
return regex.test(value);
} catch (error) {
return false;
}
}
/**
* Parses a function string into name and parameters
* @param {string} functionString - The function string to parse
* @returns {Object|string} Object with name and params properties, or original string if parsing fails
*/
function parseFunctionString(functionString) {
const openParenIndex = functionString.indexOf('(');
const closeParenIndex = functionString.lastIndexOf(')');
const functionName = openParenIndex === -1 ? functionString : functionString.substring(0, openParenIndex);
let params = [];
if (openParenIndex !== -1 && closeParenIndex > openParenIndex) {
const paramsString = functionString.substring(openParenIndex + 1, closeParenIndex).trim();
if (paramsString) {
// Regex to split by comma, but not inside quotes
const paramRegex = /(".*?"|'.*?'|[^,]+)/g;
let match;
while ((match = paramRegex.exec(paramsString)) !== null) {
let param = match[0].trim();
// Attempt to convert type if not quoted
if ((param.startsWith("'") && param.endsWith("'")) || (param.startsWith('"') && param.endsWith('"'))) {
params.push(param.substring(1, param.length - 1));
} else if (!isNaN(param) && param !== '') {
params.push(parseFloat(param));
} else if (param === 'true') {
params.push(true);
} else if (param === 'false') {
params.push(false);
} else {
params.push(param);
}
}
}
}
return {
name: functionName,
params: params
};
}
/**
* Evaluates a single operation (e.g., "1='smith'")
* @param {Array} data - The row data to evaluate against
* @param {Array} fields - The field names
* @param {string} operation - The operation string
* @param {Object} [queryFunctions={}] - Custom query functions
* @returns {string} 'true' or 'false'
*/
function evaluateSingleOperation(data, fields, operation, queryFunctions = {}) {
// Input validation
if (!Array.isArray(data)) {
return 'false';
}
if (!Array.isArray(fields)) {
return 'false';
}
if (typeof operation !== 'string') {
return 'false';
}
if (operation.trim() === '') {
return 'false';
}
const operatorPattern = /(>=|<=|!=|=|>|<)/g;
const parts = operation.split(operatorPattern);
if (parts.length !== 3) {
return 'false';
}
// Trim the parts to handle extraneous whitespace
const index = parts[0].trim();
const operator = parts[1].trim();
const value = parts[2].trim();
// Validate operator
if (!['>=', '<=', '!=', '=', '>', '<'].includes(operator)) {
return 'false';
}
// Handle case where value is not quoted - this is an ERROR condition
let cleanValue;
if (value.length >= 2 &&
((value.charAt(0) === '"' && value.charAt(value.length - 1) === '"') ||
(value.charAt(0) === "'" && value.charAt(value.length - 1) === "'"))) {
// Value is properly quoted
cleanValue = value.substring(1, value.length - 1);
} else {
// Value is not quoted - this is a malformed query that should error
return 'ERROR_UNQUOTED_VALUE';
}
// Verify that the index is valid
const columnIndex = parseInt(index);
if (isNaN(columnIndex) || columnIndex < 0 || columnIndex >= data.length) {
return 'false';
}
// Check if the corresponding field exists
if (columnIndex >= fields.length) {
return 'false';
}
// Handle null/undefined data values
const cellValue = data[columnIndex];
if (cellValue === null || cellValue === undefined) {
// For null/undefined values, only exact matches with empty string or specific null checks work
if (cleanValue === '' || cleanValue === 'null' || cleanValue === 'undefined') {
return operator === '=' ? 'true' : 'false';
}
return operator === '!=' ? 'true' : 'false';
}
let matchFound = false;
if (cleanValue.charAt(0) === '@') {
const functionString = cleanValue.substring(1);
// Validate function string is not empty
if (functionString.trim() === '') {
matchFound = false;
} else {
const functionParts = parseFunctionString(functionString);
// Validate function parts were parsed correctly
if (!functionParts || typeof functionParts.name !== 'string' || functionParts.name.trim() === '') {
matchFound = false;
} else {
const functionName = functionParts.name;
const functionParams = functionParts.params;
// Validate queryFunctions object
if (!queryFunctions || typeof queryFunctions !== 'object') {
matchFound = false;
} else if (typeof queryFunctions[functionName] === 'function') {
try {
const options = {
value: data[columnIndex],
field: fields[columnIndex],
params: functionParams,
row: fields.reduce((obj, field, index) => {
obj[field] = data[index];
return obj;
}, {})
};
const result = queryFunctions[functionName](options);
// Ensure function returns a boolean-like value
matchFound = Boolean(result);
} catch (error) {
console.error(`Error executing query function "${functionName}":`, error);
matchFound = false;
}
} else {
// Fallback for strings that start with @ but aren't functions, e.g. '@username'
matchFound = looseCaseInsensitiveCompare(data[columnIndex], cleanValue);
}
}
}
// Invert for '!=' only if it's a function or fallback string match
if (operator === '!=') {
matchFound = !matchFound;
}
} else {
const isComparison = ['>', '<', '>=', '<='].includes(operator);
if (isComparison) {
// For numeric comparisons, validate both values can be parsed as numbers
const cellValueStr = String(data[columnIndex]);
const numCellValue = parseFloat(cellValueStr);
const numQueryValue = parseFloat(cleanValue);
// Handle edge cases for numeric parsing
if (isNaN(numCellValue) || isNaN(numQueryValue)) {
matchFound = false;
} else if (!isFinite(numCellValue) || !isFinite(numQueryValue)) {
// Handle Infinity cases
matchFound = false;
} else {
// Perform numeric comparison
switch (operator) {
case '>': matchFound = numCellValue > numQueryValue; break;
case '<': matchFound = numCellValue < numQueryValue; break;
case '>=': matchFound = numCellValue >= numQueryValue; break;
case '<=': matchFound = numCellValue <= numQueryValue; break;
default: matchFound = false; break;
}
}
} else { // Operator is '=' or '!='
// For equality comparisons, handle various data types safely
try {
matchFound = looseCaseInsensitiveCompare(data[columnIndex], cleanValue);
if (operator === '!=') {
matchFound = !matchFound;
}
} catch (error) {
console.error(`Error in comparison operation:`, error);
matchFound = false;
}
}
}
return matchFound ? 'true' : 'false';
}
/**
* Replaces field expressions with boolean results
* @param {Array} data - The row data to evaluate against
* @param {Array} fields - The field names
* @param {string} query - The query string with field expressions
* @param {Object} [queryFunctions={}] - Custom query functions
* @returns {string} Query string with expressions replaced by boolean values
*/
function replaceAndEvaluateExpressions(data, fields, query, queryFunctions = {}) {
const regex = /\d+\s*(?:>=|<=|!=|=|>|<)\s*(?:"[^"]*"|'[^']*')/g;
let modifiedStr = query;
let match;
// Use a temporary string for replacement to avoid issues with repeated expressions
let tempQuery = query;
const replacements = [];
while ((match = regex.exec(tempQuery)) !== null) {
const expression = match[0];
const evaluatedValue = evaluateSingleOperation(data, fields, expression, queryFunctions);
// Check if evaluateSingleOperation returned an error
if (evaluatedValue.startsWith('ERROR_')) {
// Return error immediately instead of continuing
return evaluatedValue;
}
replacements.push({ expression, evaluatedValue });
}
// Check for unmatched comparison expressions (likely unquoted values)
const unquotedPattern = /\d+\s*(?:>=|<=|!=|=|>|<)\s*[^"'\s\(\)]+/g;
const unquotedMatches = query.match(unquotedPattern);
if (unquotedMatches) {
// Filter out matches that were already processed (quoted values)
const processedExpressions = replacements.map(r => r.expression);
const unprocessedMatches = unquotedMatches.filter(match =>
!processedExpressions.some(processed => processed.includes(match))
);
if (unprocessedMatches.length > 0) {
return 'ERROR_UNQUOTED_VALUE';
}
}
for (const rep of replacements) {
// Replace only the first occurrence to handle queries with identical conditions
modifiedStr = modifiedStr.replace(rep.expression, rep.evaluatedValue);
}
return modifiedStr;
}
/**
* Evaluates a logical expression with AND/OR operators
* @param {string} expression - The boolean expression to evaluate
* @returns {boolean} The result of the logical evaluation
*/
function evaluateLogicalExpression(expression) {
const orParts = expression.split(' OR ');
for (const orPart of orParts) {
const andParts = orPart.split(' AND ');
let andResult = true;
for (const andPart of andParts) {
if (andPart.trim() === 'false') {
andResult = false;
break;
}
}
if (andResult) {
return true;
}
}
return false;
}
/**
* Evaluates nested expressions with parentheses
* @param {string} expression - The expression to evaluate
* @returns {string} 'true' or 'false'
*/
function evaluateNestedExpression(expression) {
// Keep processing until no parentheses are left
while (expression.includes('(')) {
const closeParenIndex = expression.indexOf(')');
if (closeParenIndex === -1) break; // Malformed expression
const openParenIndex = expression.lastIndexOf('(', closeParenIndex);
if (openParenIndex === -1) break; // Malformed expression
const innerExpression = expression.substring(openParenIndex + 1, closeParenIndex);
const innerResult = evaluateLogicalExpression(innerExpression) ? 'true' : 'false';
expression = expression.substring(0, openParenIndex) + innerResult + expression.substring(closeParenIndex + 1);
}
// Evaluate the final flat expression
return evaluateLogicalExpression(expression) ? 'true' : 'false';
}
/**
* Expands the * operator to match all fields
* @param {string} query - The query string with potential * operators
* @param {Array<string>} fields - The field names to expand against
* @returns {string} Query string with * operators expanded
*/
function expandAllFields(query, fields) {
// Regular expression to match OR*=, OR*!=, AND*=, AND*!=
const pattern = /(OR|AND)\s*\*\s*(=|!=)\s*('[^']+'|"[^"]+")/g;
// Replacement function
const replaceMatch = (match, logicalOperator, operator, value) => {
// Create conditions for each field, ensuring field names are not part of the value
const conditions = fields.map(column => `${column}${operator}${value}`);
const joiner = logicalOperator.trim() === 'OR' ? ' OR ' : ' AND ';
return ` ${logicalOperator} (` + conditions.join(joiner) + ')';
};
// Perform the replacement
return query.replace(pattern, replaceMatch);
}
/**
* Parses ORDER BY clause string into fields and order directions
* @param {string} orderByClause - String like "field1 ASC, field2 DESC" or "ORDER BY field1 ASC, field2 DESC"
* @returns {Object} Object with fields array and desc array (boolean values), or error string
*/
function parseOrderByClause(orderByClause) {
// Check if the input is a string
if (typeof orderByClause !== 'string') {
return 'Error: Input must be a string';
}
// Trim the input and remove 'ORDER BY' if present
orderByClause = orderByClause.trim();
const orderByPattern = /^ORDER\s+BY\s+/i;
if (orderByPattern.test(orderByClause)) {
orderByClause = orderByClause.replace(orderByPattern, '');
}
// Split the clause by commas to get individual field clauses
const fieldClauses = orderByClause.split(/\s*,\s*/);
const fields = [];
const orders = [];
// Iterate over each field clause to extract field names and order
for (let i = 0; i < fieldClauses.length; i++) {
const fieldClause = fieldClauses[i].trim();
// Default order is ascending (false)
let order = false;
let fieldName = fieldClause;
// Check if the clause ends with DESC, indicating descending order
if (fieldClause.toUpperCase().endsWith(' DESC')) {
order = true;
fieldName = fieldClause.substring(0, fieldClause.length - 5).trim();
}
// Check if the clause ends with ASC, and remove it
else if (fieldClause.toUpperCase().endsWith(' ASC')) {
fieldName = fieldClause.substring(0, fieldClause.length - 4).trim();
}
// Add the field name and order to the respective arrays
fields.push(fieldName);
orders.push(order);
}
// Return the arrays in an object
return { fields: fields, desc: orders };
}
// --- END: Upgraded Query Engine Helpers ---
// --- The DataMaster Class ---
class DataMaster {
/**
* Constructor is simple, meant for internal use by the factories
* @param {Array<Array>} table - 2D array of data
* @param {Array<string>} fields - Field names
* @param {Object} options - Configuration options
*/
constructor(table, fields, options = {}) {
// Initialize with safe defaults first
this._table = [];
this._fields = [];
this._options = this._validateOptions(options);
// Validate and set table
if (Array.isArray(table)) {
let validTable = true;
for (let i = 0; i < table.length; i++) {
if (!Array.isArray(table[i])) {
validTable = false;
break;
}
}
if (validTable) {
this._table = table;
}
}
// Validate and set fields
if (Array.isArray(fields)) {
this._fields = fields.map((field, index) => {
if (typeof field === 'string') {
return field;
}
return field !== null && field !== undefined ? field.toString() : 'Field' + index;
});
}
}
/**
* Validates and normalizes options
* @private
*/
_validateOptions(options) {
if (typeof options !== 'object' || options === null) {
options = {};
}
const validErrorModes = ['standard', 'strict', 'silent'];
const errorMode = validErrorModes.includes(options.errorMode) ? options.errorMode : 'standard';
const onError = typeof options.onError === 'function' ? options.onError : null;
return {
errorMode,
onError
};
}
/**
* Executes a programmatic filter (object or function) against the DataMaster data
* @private
* @param {Object|Function} filter - Object with field-value pairs for AND filtering, or function that receives row object
* @returns {Object} Object with {success: boolean, table: Array<Array>, indices: Array<number>, error: string}
*/
_executeProgrammaticFilter(filter) {
if (!Array.isArray(this._table) || !Array.isArray(this._fields)) {
return { success: false, table: [], indices: [], error: 'Invalid table or fields state' };
}
if (typeof filter === 'undefined' || filter === null) {
return { success: false, table: [], indices: [], error: 'Filter parameter is required' };
}
try {
const resultData = [];
const resultIndices = [];
if (typeof filter === 'function') {
// Function-based filtering
for (let i = 0; i < this._table.length; i++) {
if (!Array.isArray(this._table[i])) continue;
const rowObject = this._fields.reduce((obj, field, index) => {
obj[field] = this._table[i][index];
return obj;
}, {});
try {
if (filter(rowObject)) {
resultData.push([...this._table[i]]);
resultIndices.push(i);
}
} catch (filterError) {
continue;
}
}
} else if (typeof filter === 'object' && filter !== null) {
// Object-based filtering (field-value pairs with AND logic)
const filterKeys = Object.keys(filter);
for (let i = 0; i < this._table.length; i++) {
if (!Array.isArray(this._table[i])) continue;
let rowMatches = true;
for (const key of filterKeys) {
const fieldIndex = this._fields.indexOf(key);
if (fieldIndex === -1) {
rowMatches = false;
break;
}
const cellValue = this._table[i][fieldIndex];
const filterValue = filter[key];
if (!looseCaseInsensitiveCompare(cellValue, filterValue)) {
rowMatches = false;
break;
}
}
if (rowMatches) {
resultData.push([...this._table[i]]);
resultIndices.push(i);
}
}
} else {
return { success: false, table: [], indices: [], error: 'Filter must be an object or function' };
}
return { success: true, table: resultData, indices: resultIndices, error: null };
} catch (error) {
return { success: false, table: [], indices: [], error: 'Failed to execute programmatic filter: ' + error.message };
}
}
/**
* Executes a WHERE clause against the DataMaster data
* @private
* @param {string} clauseString - The WHERE clause to execute
* @param {Object} [queryFunctions={}] - Custom query functions
* @returns {Object} Object with {success: boolean, table: Array<Array>, indices: Array<number>, error: string}
*/
_executeWhere(clauseString, queryFunctions = {}) {
if (!Array.isArray(this._table) || !Array.isArray(this._fields)) {
return { success: false, table: [], indices: [], error: 'Invalid table or fields state' };
}
if (typeof clauseString !== 'string') {
return { success: false, table: [], indices: [], error: 'Clause string must be a string' };
}
if (clauseString.length === 0) {
return { success: false, table: [], indices: [], error: 'Clause string cannot be empty' };
}
try {
// Run the expansion for the * operator
let query = expandAllFields(clauseString, this._fields);
const resultData = [];
const resultIndices = [];
// Convert field names to array indices
for (let i = 0; i < this._fields.length; i++) {
const fieldName = this._fields[i];
// Use a regex that captures the field name and any of the valid operators that follow it.
// The \b ensures we match whole words only, preventing "ID" from matching "ORDER BY ID".
const regex = new RegExp(`\\b${fieldName}\\b(\\s*(?:>=|<=|!=|=|>|<))`, 'g');
query = query.replace(regex, `${i}$1`);
}
// Loop through the data and add matches to the result
for (let d = 0; d < this._table.length; d++) {
if (!Array.isArray(this._table[d])) {
continue; // Skip invalid rows
}
// Convert the query into a string of boolean logic
const booleanExpression = replaceAndEvaluateExpressions(
this._table[d],
this._fields,
query,
queryFunctions
);
// Check if replaceAndEvaluateExpressions returned an error
if (booleanExpression.startsWith('ERROR_')) {
// Return error immediately instead of continuing
let errorMessage = 'Query evaluation failed';
if (booleanExpression === 'ERROR_UNQUOTED_VALUE') {
errorMessage = 'Query syntax error: Values must be quoted (e.g., field=\'value\' not field=value)';
}
return {
success: false,
table: [],
indices: [],
error: errorMessage
};
}
// E