series-extractor/test/test.ts

A TypeScript library for extracting data series from nested objects and arrays using a custom syntax.

import {
	ObjectNesting,
	ArrayNesting,
	TokenType,
	Token,
	Parser,
	seriesExtractor,
	SeriesSyntaxError,
	Value,
	Pair,
	ExtractableFlags
} from '../src/index';

describe('Parser', () => {

	const assertError = (syntax: string, err: string) => {
		expect(() => seriesExtractor(syntax)).toThrow(SeriesSyntaxError);
		expect(() => seriesExtractor(syntax)).toThrow(new RegExp(err));
	};

		it('should consume characters as expected', () => {
			const p = new Parser("  a\\b \\ \n\t\r ");
			expect(p.next()).toBe("a"); // Call public method
			p.backtrack();
			expect(p.next()).toBe("a");
			expect(p.next()).toBe("\\b");
			expect(p.next()).toBe("\\ ");
			p.backtrack();
			expect(p.next()).toBe("\\ ");
			expect(p.next()).toBeUndefined();
			p.backtrack();
			expect(p.next()).toBeUndefined();

			const p2 = new Parser("\\");
			expect(() => p2.next()).toThrow(/missing subsequent escaped character/);
		});
	it('should parse syntax 1 correctly', () => {
		const res = seriesExtractor("a.b.$c");
		const exp = new ObjectNesting().add(
			new Token("a", TokenType.STRING),
			new ObjectNesting().add(
				new Token("b", TokenType.STRING),
				new Token("c", TokenType.STRING | TokenType.DIMENSION)
			)
		);
		expect(res.equals(exp)).toBe(true);
	});

	it('should parse syntax 2 correctly', () => {
		const res = seriesExtractor("[#3.$a, 3:$a, #$3.$a, 3.$a, $3.$a, $3:$a]");
		const a = new Token("a", TokenType.STRING | TokenType.DIMENSION);
		const exp = new ArrayNesting()
			.add(
				new Token(0, TokenType.IMPLICIT_INTEGER),
				new ArrayNesting().add(new Token(3, TokenType.EXPLICIT_INTEGER), a)
			)
			.add(
				new Token(3, TokenType.IMPLICIT_INTEGER), a
			)
			.add(
				new Token(2, TokenType.IMPLICIT_INTEGER),
				new ArrayNesting().add(new Token('3', TokenType.EXPLICIT_INTEGER | TokenType.DIMENSION), a)
			)
			.add(
				new Token(3, TokenType.IMPLICIT_INTEGER),
				new ObjectNesting().add(new Token("3", TokenType.STRING), a)
			)
			.add(
				new Token(4, TokenType.IMPLICIT_INTEGER),
				new ObjectNesting().add(new Token("3", TokenType.STRING | TokenType.DIMENSION), a)
			)
			.add(
				new Token("3", TokenType.IMPLICIT_INTEGER | TokenType.DIMENSION), a
			);
		expect(res.equals(exp)).toBe(true);
	});

	it('should parse syntax 3 correctly', () => {
		const res = seriesExtractor("#$5.#5.5.$5");
		const exp = new ArrayNesting().add(
			new Token("5", TokenType.DIMENSION | TokenType.EXPLICIT_INTEGER),
			new ArrayNesting().add(
				new Token(5, TokenType.EXPLICIT_INTEGER),
				new ObjectNesting().add(
					new Token("5", TokenType.STRING),
					new Token("5", TokenType.STRING | TokenType.DIMENSION)
				)
			)
		);
		expect(res.equals(exp)).toBe(true);
	});

	it('should parse syntax 4 correctly', () => {
		const res = seriesExtractor("a.b{$c:[6:$d], e:\\#f.$g}");
		const exp = new ObjectNesting().add(
			new Token("a", TokenType.STRING),
			new ObjectNesting().add(
				new Token("b", TokenType.STRING),
				new ObjectNesting().add(
					new Token("c", TokenType.STRING | TokenType.DIMENSION),
					new ArrayNesting().add(
						new Token(6, TokenType.IMPLICIT_INTEGER),
						new Token("d", TokenType.STRING | TokenType.DIMENSION)
					)
				)
				.add(
					new Token("e", TokenType.STRING),
					new ObjectNesting().add(
						new Token("#f", TokenType.STRING),
						new Token("g", TokenType.STRING | TokenType.DIMENSION)
					)
				)
			)
		);
		expect(res.equals(exp)).toBe(true);
	});

	it('should parse syntax 5 correctly', () => {
		const res = seriesExtractor("[[$a],{b:$c}]");
		const exp = new ArrayNesting().add(
			new Token(0, TokenType.IMPLICIT_INTEGER),
			new ArrayNesting().add(
				new Token(0, TokenType.IMPLICIT_INTEGER),
				new Token("a", TokenType.STRING | TokenType.DIMENSION)
			)
		).add(
			new Token(1, TokenType.IMPLICIT_INTEGER),
			new ObjectNesting().add(
				new Token("b", TokenType.STRING),
				new Token("c", TokenType.STRING | TokenType.DIMENSION)
			)
		);
		expect(res.equals(exp)).toBe(true);
	});

	it('should handle invalid syntax', () => {
		assertError("[]", "Empty array");
		assertError("[a", "Unclosed array");
		assertError("[", "Expected array key or value");
		assertError("[#2:", "Do not use # for integer keys inside arrays");
		assertError("a.b.$c]", "Unexpected character after end of syntax");
	});

	it('should extract rows correctly - example 1', () => {
		const json = [{ 'a': [0, 1], 'b': [2] }, { 'c': [4, 5, 6] }];
		const syntax = "#$.$x.#$.$y";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		const expected = [
			{ 'x': 'a', 'y': 0 },
			{ 'x': 'a', 'y': 1 },
			{ 'x': 'b', 'y': 2 },
			{ 'x': 'c', 'y': 4 },
			{ 'x': 'c', 'y': 5 },
			{ 'x': 'c', 'y': 6 }
		];
		expect(data).toEqual(expected);
	});

	it('should extract rows correctly - example 2', () => {
		const json = { "a": 3, "b": 4, "c": 5 };
		const syntax = "{a:$x, b:$y, c:$z}";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		const expected = [
			{ 'x': 3, 'y': 4, 'z': 5 },
		];
		expect(data).toEqual(expected);
	});

	// Test anonymous dimensions (wildcard $)
	it('should handle anonymous dimension variables', () => {
		const json = { 'a': 1, 'b': 2, 'c': 3 };
		const syntax = "{$:$val}";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		expect(data).toEqual([
			{ 'val': 1 },
			{ 'val': 2 },
			{ 'val': 3 }
		]);
	});

	// Test shortcut with anonymous dimension
	it('should parse shortcut with anonymous dimension', () => {
		const json = { 'a': { 'x': 1 }, 'b': { 'x': 2 } };
		const syntax = "{$:{x:$val}}";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		expect(data).toEqual([
			{ 'val': 1 },
			{ 'val': 2 }
		]);
	});

	// Test escaped characters
	it('should handle escaped special characters', () => {
		const json = { 'a.b': { 'c': 10 } };
		const syntax = "a\\.b{c:$val}";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		expect(data).toEqual([{ 'val': 10 }]);
	});

	// Test sparse array definitions
	it('should handle sparse array definitions', () => {
		const json = ['x', 'y', 'z', 'w', 'v'];
		const syntax = "[4:$val, $other]";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		// Index 0 is for implicit (second element in syntax), index 4 is explicit
		expect(data).toEqual([
			{ 'other': 'y', 'val': 'v' }
		]);
	});

	// Test empty object error
	it('should throw error on empty object', () => {
		assertError("{}", "Empty object");
	});

	// Test more error cases
	it('should throw error on missing colon in object', () => {
		assertError("{a}", "Expected colon");
	});

	it('should throw error on unexpected character in parseShortcut', () => {
		// This would be caught by parser logic
		assertError("a,b", "Expected value");
	});

	// Test Token edge cases
	it('should handle Token with explicit integer and dimension', () => {
		const token = new Token("5", TokenType.EXPLICIT_INTEGER | TokenType.DIMENSION);
		expect(token.name).toBe("5");
		expect(token.flags).toBe(TokenType.EXPLICIT_INTEGER | TokenType.DIMENSION);
	});

	// Test finalized token
	it('should check if token is finalized', () => {
		const token1 = new Token("a", TokenType.STRING);
		expect(token1.finalized).toBe(true);

		const token2 = new Token("a", TokenType.STRING | TokenType.IMPLICIT_INTEGER);
		expect(token2.finalized).toBe(false);
	});

	// Test finalizeType errors
	it('should throw error when finalizing already finalized token', () => {
		const token = new Token("test", TokenType.STRING);
		expect(() => token.finalizeType(TokenType.STRING)).toThrow("token type already finalized");
	});

	it('should throw error on invalid finalizeType argument', () => {
		const token = new Token("test", TokenType.STRING | TokenType.IMPLICIT_INTEGER);
		expect(() => token.finalizeType(TokenType.EXPLICIT_INTEGER)).toThrow("invalid type argument");
	});

	// Test Token.fromBuilder errors
	it('should throw error on invalid flags in fromBuilder', () => {
		expect(() => Token.fromBuilder(["test"], TokenType.IMPLICIT_INTEGER)).toThrow("invalid flags argument");
	});

	it('should throw error on # without integer key', () => {
		expect(() => Token.fromBuilder(["#"], TokenType.STRING | TokenType.EXPLICIT_INTEGER)).toThrow("Expected integer key after #");
	});

	it('should throw error when explicit integer not allowed', () => {
		expect(() => Token.fromBuilder(["#", "5"], TokenType.STRING)).toThrow("Explicit integer key shortcut not allowed here");
	});

	// Test extraction with different data types
	it('should handle extraction with null values', () => {
		const json = { 'a': null };
		const syntax = "a.$val";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		expect(data).toEqual([{ 'val': null }]);
	});

	// Test that leaf values must be dimension variables
	it('should throw error when leaf value is not a dimension', () => {
		// Test via low-level API
		expect(() => {
			new ObjectNesting().add(
				new Token("key", TokenType.STRING),
				new Token("value", TokenType.STRING)  // Missing DIMENSION flag
			);
		}).toThrow("Leaf value must be a dimension variable");

		// Test via parser - leaf without $ should be rejected
		assertError("{a: b}", "Leaf value must be a dimension variable");
		assertError("[a]", "Leaf value must be a dimension variable");
		assertError("{a: {b: c}}", "Leaf value must be a dimension variable");
	});

	// Test that anonymous dimensions ARE allowed as leaf values
	it('should allow anonymous dimensions as leaf values', () => {
		// Anonymous leaf in object - yields single row with no fields
		const extractor1 = seriesExtractor("{a: $}");
		const data1 = Array.from(extractor1.extractRows({ a: 42 }));
		expect(data1).toEqual([{}]);

		// Anonymous leaf in array - only yields one row (anonymous values don't create multiple rows)
		const extractor2 = seriesExtractor("[$]");
		const data2 = Array.from(extractor2.extractRows([10, 20, 30]));
		expect(data2).toEqual([{}]);

		// Anonymous leaf in array of objects - still only ONE empty row
		const extractor4 = seriesExtractor("[$]");
		const data4 = Array.from(extractor4.extractRows([{a: 42}, {a: 43}]));
		expect(data4).toEqual([{}]); // One empty row, not two

		// With ANONYMOUS flag, extract() returns the anonymous values
		const extractor3 = seriesExtractor("{a: $}");
		const values = Array.from(extractor3.extract({ a: 42 }, ExtractableFlags.ANONYMOUS));
		expect(values.length).toBe(1);
		const firstValue = values[0];
		if (typeof firstValue !== 'string') { // Not PUSH or POP
			expect(firstValue.anonymous).toBe(true);
			expect(firstValue.value).toBe(42);
		}
	});

	// Test extract() with and without ANONYMOUS flag for array of values
	it('should return values based on ANONYMOUS flag', () => {
		const extractor = seriesExtractor("[$]");
		const testData = [10, 20, 30];

		// WITHOUT ANONYMOUS flag - returns 0 values (anonymous values excluded)
		const withoutFlag = Array.from(extractor.extract(testData));
		expect(withoutFlag.length).toBe(0);

		// WITH ANONYMOUS flag - returns only 1 value (from first/index 0)
		// [$] means anonymous key (doesn't iterate) + anonymous value
		const withFlag = Array.from(extractor.extract(testData, ExtractableFlags.ANONYMOUS));
		expect(withFlag.length).toBe(1);
		const val = withFlag[0];
		if (typeof val !== 'string') {
			expect(val.anonymous).toBe(true);
			expect(val.value).toBe(10); // First array element only
		}

		// Even with STACK flag, still only 1 value because [$] = [0: $]
		// To iterate through all elements, need dimension key: [$: ...]
		const withBothFlags = Array.from(extractor.extract(testData, ExtractableFlags.ANONYMOUS | ExtractableFlags.STACK));
		const values = withBothFlags.filter(v => typeof v !== 'string');
		expect(values.length).toBe(1);  // Still just index 0
	});

	// Test that [$: $] iterates through all array elements (dimension key)
	it('should iterate all array elements with dimension key [$: $]', () => {
		const extractor = seriesExtractor("[$: $]");
		const testData = [10, 20, 30];

		// WITHOUT ANONYMOUS flag - returns 0 values (both key and value are anonymous)
		const withoutFlag = Array.from(extractor.extract(testData));
		expect(withoutFlag.length).toBe(0);

		// WITH ANONYMOUS flag - returns 6 values: key+value for each element
		const withFlag = Array.from(extractor.extract(testData, ExtractableFlags.ANONYMOUS));
		expect(withFlag.length).toBe(6);  // 2 per element: anonymous key + anonymous value

		// Values should be: 0, 10, 1, 20, 2, 30
		const vals = withFlag as Value[];
		expect(vals[0].value).toBe(0);  // First anonymous key (index)
		expect(vals[1].value).toBe(10); // First anonymous value
		expect(vals[2].value).toBe(1);  // Second anonymous key
		expect(vals[3].value).toBe(20); // Second anonymous value
		expect(vals[4].value).toBe(2);  // Third anonymous key
		expect(vals[5].value).toBe(30); // Third anonymous value
	});

	// Test Nesting.has with invalid JSON types
	it('should handle has() method with non-object types', () => {
		const nesting = new ObjectNesting();
		expect(nesting.has(null, "key")).toBe(false);
		expect(nesting.has(undefined, "key")).toBe(false);
		expect(nesting.has("string", "key")).toBe(false);
	});

	// Test ArrayNesting.has with invalid array
	it('should handle ArrayNesting.has with non-array types', () => {
		const nesting = new ArrayNesting();
		expect(nesting.has({} as any, 0)).toBe(false);
		expect(nesting.has(null as any, 0)).toBe(false);
	});

	// Test extraction with missing keys
	it('should skip missing keys during extraction', () => {
		const json = { 'a': 1 };
		const syntax = "{a:$x, b:$y}";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		expect(data).toEqual([{ 'x': 1 }]);  // b is missing, so y is not included
	});

	// Test toString methods
	it('should have meaningful toString representations', () => {
		const token = new Token("test", TokenType.STRING | TokenType.DIMENSION);
		expect(token.toString()).toContain("test");

		const nesting = new ObjectNesting();
		expect(nesting.toString()).toContain("ObjectNesting");
	});

	// Test Value class
	it('should create and use Value objects correctly', () => {
		const token = new Token("myVar", TokenType.STRING | TokenType.DIMENSION);
		const value = new Value(token, 42);

		expect(value.name).toBe("myVar");
		expect(value.value).toBe(42);
		expect(value.anonymous).toBe(false);
		expect(value.toString()).toContain("myVar=42");
	});

	// Test anonymous Value
	it('should handle anonymous values', () => {
		const token = new Token(null, TokenType.STRING | TokenType.DIMENSION);
		const value = new Value(token, 100);

		expect(value.anonymous).toBe(true);
		expect(value.name).toBe(null);
	});

	// Test Pair equality
	it('should compare Pairs correctly', () => {
		const key1 = new Token("a", TokenType.STRING);
		const val1 = new Token("b", TokenType.STRING | TokenType.DIMENSION);
		const pair1 = new Pair(key1, val1);

		const key2 = new Token("a", TokenType.STRING);
		const val2 = new Token("b", TokenType.STRING | TokenType.DIMENSION);
		const pair2 = new Pair(key2, val2);

		expect(pair1.equals(pair2)).toBe(true);

		const key3 = new Token("c", TokenType.STRING);
		const pair3 = new Pair(key3, val1);
		expect(pair1.equals(pair3)).toBe(false);
	});

	// Test Nesting equals with different types
	it('should return false when comparing different nesting types', () => {
		const obj = new ObjectNesting().add(
			new Token("a", TokenType.STRING),
			new Token("b", TokenType.STRING | TokenType.DIMENSION)
		);
		const arr = new ArrayNesting().add(
			new Token(0, TokenType.IMPLICIT_INTEGER),
			new Token("b", TokenType.STRING | TokenType.DIMENSION)
		);

		expect(obj.equals(arr)).toBe(false);
	});

	// Test array with explicit integers in shortcut
	it('should parse array shortcut with explicit integer', () => {
		const json = [[1, 2], [3, 4]];
		const syntax = "#0[$val]";
		const nesting = seriesExtractor(syntax);
		const data = Array.from(nesting.extractRows(json));
		// Extracts from json[0], which is [1, 2]
		expect(data).toEqual([
			{ 'val': 1 }
		]);
	});

	// Test SeriesSyntaxError structure
	it('should provide structured error information', () => {
		const syntax = "a.b.$c]";
		try {
			seriesExtractor(syntax);
			fail('Should have thrown an error');
		} catch (error) {
			expect(error).toBeInstanceOf(SeriesSyntaxError);
			if (error instanceof SeriesSyntaxError) {
				// Check structured properties
				expect(error.line).toBe(1);
				expect(error.col).toBe(7);
				expect(error.char).toBe(7);
				expect(error.syntaxInput).toBe(syntax);
				expect(error.originalMessage).toBe("Unexpected character after end of syntax");

				// Check that context is formatted
				expect(error.context).toContain('a.b.$c]');
				expect(error.context).toContain('^');

				// Check that full message includes position info
				expect(error.message).toContain('line #1');
				expect(error.message).toContain('col #7');
				expect(error.message).toContain('char #7');
				expect(error.message).toContain('a.b.$c]');
			}
		}
	});

	// Test SeriesSyntaxError formatting
	it('should format error message nicely', () => {
		const syntax = "[#2:";
		try {
			seriesExtractor(syntax);
			fail('Should have thrown an error');
		} catch (error) {
			expect(error).toBeInstanceOf(SeriesSyntaxError);
			if (error instanceof SeriesSyntaxError) {
				// The message should be human-readable
				expect(error.message).toContain('Invalid series syntax');
				expect(error.message).toContain('[#2:');
				expect(error.originalMessage).toContain('Do not use # for integer keys inside arrays');
			}
		}
	});

	// Test mixing dimension keys with constant keys in same object
	it('should handle mixing dimension keys with constant keys', () => {
		const data = {
			key1: { a: 10, b: 20, c: 30 },
			key2: { a: 40, b: 50, c: 60 }
		};

		const extractor = seriesExtractor(`{$keyName: {$a: $x, b: $y}}`);
		const result = Array.from(extractor.extractRows(data));

		// Expected behavior: for each outer key (key1, key2):
		// - Extract constant key 'b' once
		// - Iterate through all keys for dimension $a
		expect(result).toEqual([
			{ keyName: 'key1', a: 'a', x: 10, y: 20 },
			{ keyName: 'key1', a: 'b', x: 20, y: 20 },
			{ keyName: 'key1', a: 'c', x: 30, y: 20 },
			{ keyName: 'key2', a: 'a', x: 40, y: 50 },
			{ keyName: 'key2', a: 'b', x: 50, y: 50 },
			{ keyName: 'key2', a: 'c', x: 60, y: 50 }
		]);
	});

	// Test nested object with double nested arrays
	it('should handle nested object with double nested arrays', () => {
		const data = {
			outer: {
				"key-A": { inner: [[1, 2], [3]] },
				"key-B": { inner: [[4]] }
			}
		};

		const extractor = seriesExtractor(`
			outer {
				$key: inner [$: [$: $val]]
			}
		`);

		const result = Array.from(extractor.extractRows(data));

		// Should extract all values from both keys
		expect(result).toEqual([
			{ key: 'key-A', val: 1 },
			{ key: 'key-A', val: 2 },
			{ key: 'key-A', val: 3 },
			{ key: 'key-B', val: 4 }
		]);
	});

	// Test README example - validates the documentation example works correctly
	it('should extract data correctly for README example', () => {
		const sensorData = {
			buildings: {
				"building-A": {
					floors: [
						[{ temp: 68, humidity: 45 }, { temp: 70, humidity: 48 }],
						[{ temp: 72, humidity: 50 }]
					],
					roof: { temp: 90, humidity: 55 }
				},
				"building-B": {
					floors: [[{ temp: 69, humidity: 46 }]],
					roof: { temp: 85, humidity: 52 }
				}
			}
		};

		const extractor = seriesExtractor(`
			buildings.$building {
				floors: #$floor.#$ {
					temp: $temp,
					humidity: $humidity
				},
				roof: {
					temp: $temp,
					humidity: $humidity
				}
			}
		`);

		const data = Array.from(extractor.extractRows(sensorData));

		expect(data).toEqual([
			{ building: 'building-A', floor: 0, temp: 68, humidity: 45 },
			{ building: 'building-A', floor: 0, temp: 70, humidity: 48 },
			{ building: 'building-A', floor: 1, temp: 72, humidity: 50 },
			{ building: 'building-A', temp: 90, humidity: 55 },
			{ building: 'building-B', floor: 0, temp: 69, humidity: 46 },
			{ building: 'building-B', temp: 85, humidity: 52 }
		]);
	});

	// Test mixing constant key with anonymous dimension key at same level
	it('should combine constant and dimension keys into same rows', () => {
		const data = {
			x: { a: 1, b: 2 },
			y: { a: 3, b: 4 },
			c: 5
		};

		const extractor = seriesExtractor("{$: {a: $a, b: $b}, c: $c}");
		const result = Array.from(extractor.extractRows(data));

		// The constant key 'c' is processed first and adds c=5 to the row context
		// Then the anonymous dimension '$:' iterates over ALL keys (x, y, c)
		// For x: extracts a=1, b=2, yields row {c: 5, a: 1, b: 2}
		// For y: extracts a=3, b=4, yields row {c: 5, a: 3, b: 4}
		// For c: value is 5 (not an object), cannot extract a/b, no row generated

		expect(result).toEqual([
			{ c: 5, a: 1, b: 2 },
			{ c: 5, a: 3, b: 4 }
		]);

		// Note: c appears in both rows because it's a constant key at the same level
		// The dimension iteration happens AFTER constant keys are processed
	});

	// Contrast: same test but with named dimension $nest instead of anonymous $
	it('should include named dimension in output rows (contrast with anonymous)', () => {
		const data = {
			x: { a: 1, b: 2 },
			y: { a: 3, b: 4 },
			c: 5
		};

		const extractor = seriesExtractor("{$nest: {a: $a, b: $b}, c: $c}");
		const result = Array.from(extractor.extractRows(data));

		// Same behavior as anonymous dimension test, BUT now 'nest' appears in output
		// The constant key 'c' is processed first and adds c=5 to the row context
		// Then the named dimension '$nest:' iterates over ALL keys (x, y, c)
		// For x: nest='x', extracts a=1, b=2, yields row {c: 5, nest: 'x', a: 1, b: 2}
		// For y: nest='y', extracts a=3, b=4, yields row {c: 5, nest: 'y', a: 3, b: 4}
		// For c: value is 5 (not an object), cannot extract a/b, no row generated

		expect(result).toEqual([
			{ c: 5, nest: 'x', a: 1, b: 2 },
			{ c: 5, nest: 'y', a: 3, b: 4 }
		]);

		// Key difference: 'nest' field appears in output because it's a NAMED dimension
		// Anonymous dimensions ($) don't appear in the output, only named ones do
	});

	// Test mixing constant array indices with dimension iteration (arrays behave like objects)
	it('should combine constant array indices with dimension iteration', () => {
		const data = [
			{ x: 10, d: 100 },  // index 0
			{ x: 20, d: 200 },  // index 1
			{ x: 30, d: 300 },  // index 2
			{ x: 40, d: 400 }   // index 3
		];

		const extractor = seriesExtractor("[0:{x:$a}, 3:{x:$b}, $:{d:$d}]");
		const result = Array.from(extractor.extractRows(data));

		// Constants 0 and 3 are processed first:
		// - index 0: extract x → a=10
		// - index 3: extract x → b=40
		// Then dimension $: iterates over ALL indices (0, 1, 2, 3):
		// - For index 0: extract d=100, yield {a: 10, b: 40, d: 100}
		// - For index 1: extract d=200, yield {a: 10, b: 40, d: 200}
		// - For index 2: extract d=300, yield {a: 10, b: 40, d: 300}
		// - For index 3: extract d=400, yield {a: 10, b: 40, d: 400}

		expect(result).toEqual([
			{ a: 10, b: 40, d: 100 },
			{ a: 10, b: 40, d: 200 },
			{ a: 10, b: 40, d: 300 },
			{ a: 10, b: 40, d: 400 }
		]);

		// Constant array indices behave the same as constant object keys:
		// they're processed first and persist in all rows from dimension iteration
	});

	// Test when multiple constants extract to the same field name (last wins)
	it('should overwrite field when multiple constants extract to same name', () => {
		const data = [
			{ x: 10, d: 100 },  // index 0
			{ x: 20, d: 200 },  // index 1
			{ x: 30, d: 300 },  // index 2
			{ x: 40, d: 400 }   // index 3
		];

		const extractor = seriesExtractor("[0:{x:$a}, 3:{x:$a}, $:{d:$d}]");
		const result = Array.from(extractor.extractRows(data));

		// Constants are processed in order:
		// - index 0: extract x → a=10 (sets row.a = 10)
		// - index 3: extract x → a=40 (overwrites row.a = 40)
		// Then dimension $: iterates, yielding rows with a=40 (last value wins)

		expect(result).toEqual([
			{ a: 40, d: 100 },
			{ a: 40, d: 200 },
			{ a: 40, d: 300 },
			{ a: 40, d: 400 }
		]);

		// Last constant wins when multiple constants write to same field
	});

	// Explore what constitutes a "row" - siblings at same level
	it('should group siblings at same level into one row', () => {
		const data = { a: 1, b: 2, c: 3 };
		const extractor = seriesExtractor("{a: $a, b: $b, c: $c}");
		const result = Array.from(extractor.extractRows(data));

		// All siblings at same level, no dimension variables (all constants)
		// Yields one row with all three values
		expect(result).toEqual([{ a: 1, b: 2, c: 3 }]);
	});

	// Explore what constitutes a "row" - ancestors in a chain
	it('should group ancestor chain into rows (one per leaf)', () => {
		const data = { a: { b: { c: 1, d: 2 } } };
		const extractor = seriesExtractor("a.b{c: $c, d: $d}");
		const result = Array.from(extractor.extractRows(data));

		// All constants in a chain, leaf values at same level
		// One row with both leaf values
		expect(result).toEqual([{ c: 1, d: 2 }]);
	});

	// Explore what constitutes a "row" - metadata + dimension iteration
	it('should broadcast constant metadata to all dimension iteration rows', () => {
		const data = {
			metadata: { station: "Station-A", region: "North" },
			readings: [
				{ time: 1, temp: 20 },
				{ time: 2, temp: 22 }
			]
		};

		const extractor = seriesExtractor(`{
			metadata: {station: $station, region: $region},
			readings: [$: {time: $time, temp: $temp}]
		}`);
		const result = Array.from(extractor.extractRows(data));

		// Constants (metadata) are broadcast to all rows from dimension iteration
		// Each row = one reading with its metadata context
		expect(result).toEqual([
			{ station: "Station-A", region: "North", time: 1, temp: 20 },
			{ station: "Station-A", region: "North", time: 2, temp: 22 }
		]);

		// Makes sense: each reading row includes its metadata context
	});

	// Explore what constitutes a "row" - nested constant under dimension
	it('should include nested constants within dimension iteration', () => {
		const data = {
			eng: { budget: 100, employees: [{ name: "Alice" }, { name: "Bob" }] },
			sales: { budget: 200, employees: [{ name: "Charlie" }] }
		};

		const extractor = seriesExtractor(`{
			$dept: {
				budget: $budget,
				employees: [$: {name: $name}]
			}
		}`);
		const result = Array.from(extractor.extractRows(data));

		// Each row = one employee with their department and department budget
		// budget is a constant under the dimension, gets included in nested iterations
		expect(result).toEqual([
			{ dept: "eng", budget: 100, name: "Alice" },
			{ dept: "eng", budget: 100, name: "Bob" },
			{ dept: "sales", budget: 200, name: "Charlie" }
		]);

		// Makes sense: budget is part of the dept context, shared by all employees
	});

	// Explore what constitutes a "row" - independent dimension siblings
	it('should yield separate rows for independent dimension siblings', () => {
		const data = {
			users: [{ name: "Alice" }, { name: "Bob" }],
			products: [{ name: "Widget" }, { name: "Gadget" }]
		};

		const extractor = seriesExtractor(`{
			users: [$: {name: $userName}],
			products: [$: {name: $productName}]
		}`);
		const result = Array.from(extractor.extractRows(data));

		// Two independent dimension iterations
		// Each dimension yields its own rows separately
		expect(result).toEqual([
			{ userName: "Alice" },
			{ userName: "Bob" },
			{ productName: "Widget" },
			{ productName: "Gadget" }
		]);

		// Makes sense: these are independent dimensions, not related
	});

	// Complex case: constants with dimension intermixed at different levels
	it('should handle dimension intermixed with constants at same nesting', () => {
		const data = {
			a: { b: { c: 10 } },
			b: { c: { d: 20, key1: 100, key2: 200 } },
			c: { x: 30 }
		};

		const extractor = seriesExtractor("{a: b.c.$x, b: c{d:$y, $e:$f}, c: x.$z}");
		const result = Array.from(extractor.extractRows(data));

		// Actual behavior: constants are processed in order, dimensions yield immediately
		// - a.b.c → x=10 (added to row context)
		// - b.c.d → y=20 (added to row context)
		// - b.c.$e:$f → iterates, yields 3 rows with x,y,e,f (z NOT included yet!)
		// - c.x → z=30 (added to row context)
		// - End of extraction → yields final row with x,y,z (e,f were popped)

		expect(result).toEqual([
			{ x: 10, y: 20, e: 'd', f: 20 },       // z missing!
			{ x: 10, y: 20, e: 'key1', f: 100 },  // z missing!
			{ x: 10, y: 20, e: 'key2', f: 200 },  // z missing!
			{ x: 10, y: 20, z: 30 }                // e,f missing!
		]);

		// Problem: constants after a dimension don't appear in that dimension's rows
		// This is confusing and probably not what users expect
	});

	// Test extraction order with dimensions before/between/after constants
	it('should demonstrate extraction order with mixed constants and dimensions', () => {
		const data = {
			dimBefore: [{ val: 1 }, { val: 2 }],
			constA: 100,
			dimMiddle: [{ val: 3 }, { val: 4 }],
			constB: 200,
			dimAfter: [{ val: 5 }, { val: 6 }],
			constC: 300
		};

		const extractor = seriesExtractor(`{
			dimBefore: [$idx: {val: $before}],
			constA: $a,
			dimMiddle: [$idx: {val: $middle}],
			constB: $b,
			dimAfter: [$idx: {val: $after}],
			constC: $c
		}`);
		const result = Array.from(extractor.extractRows(data));

		// ACTUAL traversal order (all keys are constants, dimensions are in the values):
		// 1. dimBefore (constant key) → ArrayNesting with $idx dimension → yields 2 rows
		//    - No other constants processed yet, so rows have only: idx, before
		// 2. constA (constant key) → $a value → adds a=100 to row context
		// 3. dimMiddle (constant key) → ArrayNesting with $idx dimension → yields 2 rows
		//    - constA already processed, so rows have: a, idx, middle (but NOT b or c!)
		// 4. constB (constant key) → $b value → adds b=200 to row context
		// 5. dimAfter (constant key) → ArrayNesting with $idx dimension → yields 2 rows
		//    - constA and constB processed, so rows have: a, b, idx, after (but NOT c!)
		// 6. constC (constant key) → $c value → adds c=300 to row context
		// 7. End of extraction → yields final row with all constants (a, b, c)

		expect(result).toEqual([
			{ idx: 0, before: 1 },                    // Only dimBefore's values
			{ idx: 1, before: 2 },
			{ a: 100, idx: 0, middle: 3 },            // constA + dimMiddle
			{ a: 100, idx: 1, middle: 4 },
			{ a: 100, b: 200, idx: 0, after: 5 },     // constA, constB + dimAfter
			{ a: 100, b: 200, idx: 1, after: 6 },
			{ a: 100, b: 200, c: 300 }                // Final row at end with all constants
		]);

		// Key insight: Constant keys are processed in order, and nested dimensions
		// yield rows immediately with only the constants processed BEFORE them
	});

	// Test extraction order with nested structures
	it('should demonstrate extraction order with nested constants and dimensions', () => {
		const data = {
			obj: {
				const1: 10,
				arr: [{ x: 1 }, { x: 2 }],
				const2: 20
			}
		};

		const extractor = seriesExtractor(`{
			obj: {
				const1: $c1,
				arr: [$: {x: $x}],
				const2: $c2
			}
		}`);
		const result = Array.from(extractor.extractRows(data));

		console.log("Nested extraction order:", JSON.stringify(result, null, 2));

		// Within obj nesting:
		// - const1 processed first → c1=10
		// - arr dimension iterates → yields 2 rows with c1, x (c2 NOT yet processed!)
		// - const2 processed after → c2=20
		// - End of extraction → yields final row with c1, c2 (x popped)

		expect(result).toEqual([
			{ c1: 10, x: 1 },      // c2 not included yet!
			{ c1: 10, x: 2 },      // c2 not included yet!
			{ c1: 10, c2: 20 }     // Final row with both constants, x popped
		]);

		// Same issue: constants AFTER a dimension don't appear in that dimension's rows
		// They only appear in the final row (or subsequent dimensions' rows)
	});

	// Test: constant keys vs dimension keys at same level
	it('should process constant keys before dimension keys', () => {
		const data = {
			constKey1: 100,
			dimKey1: { a: 1, b: 2 },
			constKey2: 200,
			dimKey2: { a: 3, b: 4 }
		};

		// $dim1 and $dim2 are dimension KEYS (not values)
		const extractor = seriesExtractor(`{
			constKey1: $c1,
			$dim1: {a: $x, b: $y},
			constKey2: $c2,
			$dim2: {a: $x, b: $y}
		}`);
		const result = Array.from(extractor.extractRows(data));

		// Per user's docs: "constant keys are traversed before variable keys"
		// 1. ALL constant keys processed first: constKey1 → c1=100, constKey2 → c2=200
		// 2. THEN dimension keys iterate over DATA keys (not syntax keys!)
		//    - $dim1 and $dim2 both iterate over: dimKey1, dimKey2
		//    - This creates cartesian product: $dim1 × $dim2
		// 3. Each row generated after visiting each dimension iteration

		expect(result).toEqual([
			{ c1: 100, c2: 200, dim1: 'dimKey1', x: 1, y: 2 },  // $dim1=dimKey1, $dim2 hasn't iterated yet
			{ c1: 100, c2: 200, dim2: 'dimKey1', x: 1, y: 2 },  // $dim2=dimKey1, $dim1 was reset
			{ c1: 100, c2: 200, dim1: 'dimKey2', x: 3, y: 4 },  // $dim1=dimKey2, $dim2 hasn't iterated yet
			{ c1: 100, c2: 200, dim2: 'dimKey2', x: 3, y: 4 }   // $dim2=dimKey2, $dim1 was reset
		]);

		// Confirms: ALL constant keys appear in all rows because they're processed first
	});

	// Test: verify dimension keys iterate over data keys, not syntax order
	it('should iterate dimension keys in data order, not syntax order', () => {
		const data = {
			zebra: { val: 1 },
			apple: { val: 2 },
			monkey: { val: 3 }
		};

		// Dimension key $name will iterate over data keys (zebra, apple, monkey)
		// in the order they appear in the data object
		const extractor = seriesExtractor(`{$name: {val: $val}}`);
		const result = Array.from(extractor.extractRows(data));

		console.log("Data order iteration:", JSON.stringify(result, null, 2));

		// JavaScript object iteration order for string keys: insertion order
		// So should be: zebra, apple, monkey (in the order defined in data)
		expect(result).toEqual([
			{ name: 'zebra', val: 1 },
			{ name: 'apple', val: 2 },
			{ name: 'monkey', val: 3 }
		]);
	});

	// Test: verify rows are generated after visiting dimension key
	it('should generate row after visiting each dimension key iteration', () => {
		const data = {
			outer1: { inner1: { x: 1 }, inner2: { x: 2 } },
			outer2: { inner1: { x: 3 }, inner2: { x: 4 } }
		};

		// Two levels of dimension keys: $outer iterates outer1/outer2,
		// then $inner iterates inner1/inner2 within each
		const extractor = seriesExtractor(`{$outer: {$inner: {x: $x}}}`);
		const result = Array.from(extractor.extractRows(data));

		console.log("Nested dimension keys:", JSON.stringify(result, null, 2));

		// After visiting each $inner dimension, a row is generated
		// Nested variables ($x) are reset after each $inner
		expect(result).toEqual([
			{ outer: 'outer1', inner: 'inner1', x: 1 },
			{ outer: 'outer1', inner: 'inner2', x: 2 },
			{ outer: 'outer2', inner: 'inner1', x: 3 },
			{ outer: 'outer2', inner: 'inner2', x: 4 }
		]);
	});

	// Test: emission order for sparse array with dimensions
	it('should emit rows in syntax order for sparse array [4:$.$a, $.$b]', () => {
		const data = [
			[100, 101],  // index 0
			[200, 201],  // index 1
			[300, 301],  // index 2
			[400, 401],  // index 3
			[500, 501]   // index 4
		];

		// [4:[$:$a], [$:$b]] means:
		// - Index 4 (constant): iterate with anonymous dimension key, extract $a
		// - Implicit index (constant): iterate with anonymous dimension key, extract $b
		const extractor = seriesExtractor("[4:[$:$a], [$:$b]]");
		const result = Array.from(extractor.extractRows(data));

		console.log("Sparse array order:", JSON.stringify(result, null, 2));

		// Both index 4 and implicit index (which would be 1) are constant keys
		// They should be processed in syntax order:
		// 1. Index 4 first → data[4] = [500, 501] → yields {a: 500}, {a: 501}
		// 2. Implicit index 1 → data[1] = [200, 201] → yields {b: 200}, {b: 201}

		expect(result).toEqual([
			{ a: 500 },  // data[4][0]
			{ a: 501 },  // data[4][1]
			{ b: 200 },  // data[1][0]
			{ b: 201 }   // data[1][1]
		]);

		// Confirms: constant keys (including sparse array indices) are processed
		// in the order they appear in the syntax
	});
});