n8n-nodes-arubacentral/dist/nodes/ArubaCentral/helpers/pagination.ts

n8n community node for Aruba Central API integration with comprehensive monitoring, configuration, and management capabilities

// ArubaCentral/helpers/pagination.ts
import {
	IExecuteFunctions,
	IHttpRequestMethods,
	IDataObject,
	INodeExecutionData,
} from 'n8n-workflow';
import { apiRequest } from './apiRequest';

/**
 * Helper function to handle pagination for Aruba Central API endpoints
 * @param this The IExecuteFunctions context
 * @param endpoint API endpoint to call
 * @param method HTTP method to use
 * @param body Request body (if applicable)
 * @param queryParams Query parameters
 * @param dataLocator Information to locate data in the response
 * @param returnAll Whether to return all results or respect limit parameter
 * @param limit Maximum number of results to return if not returning all
 * @param batchSize Optional batch size for pagination (if endpoint has specific limits)
 * @returns Array of INodeExecutionData objects containing all results
 */
export async function handlePagination(
	this: IExecuteFunctions,
	endpoint: string,
	method: IHttpRequestMethods,
	body: IDataObject,
	queryParams: IDataObject,
	dataLocator: {
		path: string[]; // Path to data array in the response (e.g., ['aps'] or ['data', 'aps'])
		fallbackPaths?: string[][]; // Fallback paths to try if primary path doesn't exist
	},
	returnAll: boolean,
	limit?: number,
	batchSize?: number,
): Promise<INodeExecutionData[]> {
	console.log(`Starting enhanced pagination for endpoint ${endpoint}`);

	// Deep clone query params to avoid modifying the original
	const qs = JSON.parse(JSON.stringify(queryParams)) as IDataObject;

	// Initialize variables
	const allItems: IDataObject[] = [];
	let hasMoreItems = true;
	let offset = (qs.offset as number) || 0;

	// Use the provided batch size or fall back to the default of 1000
	const effectiveBatchSize = batchSize || 1000;
	console.log(
		`Pagination configuration: returnAll=${returnAll}, requestedLimit=${limit}, batchSize=${effectiveBatchSize}`,
	);

	// Set initial limit for API requests
	qs.limit = effectiveBatchSize;

	// For logging path attempts
	const logDataPath = (path: string[]) => path.join('.');

	// Continue fetching batches while there are more items and we haven't reached the limit
	while (hasMoreItems && (!limit || allItems.length < limit)) {
		// Update offset for this request
		qs.offset = offset;
		console.log(`Making request with offset=${offset}, limit=${effectiveBatchSize}`);

		// Make API call
		const response = await apiRequest.call(this, method, endpoint, body, qs);
		console.log('Response received. Extracting data...');

		// Debug log the response keys
		if (typeof response === 'object' && response !== null) {
			console.log('Response top-level keys:', Object.keys(response));
			// If there's a data property, log its keys too
			if (response.data && typeof response.data === 'object') {
				console.log('Response data keys:', Object.keys(response.data));
			}
		}

		// Function to extract data from response based on path
		const extractData = (obj: any, path: string[]): any[] | null => {
			let current = obj;

			// Follow the path to the data
			for (const key of path) {
				if (current === undefined || current === null) return null;
				current = current[key];
			}

			// If we found an array, return it
			if (Array.isArray(current)) {
				return current;
			}

			return null;
		};

		// Try to extract data using the primary path
		let items = extractData(response, dataLocator.path);
		console.log(
			`Looking for data at path ${logDataPath(dataLocator.path)}: ${items ? 'Found' : 'Not found'}`,
		);

		// If not found, try fallback paths
		if (!items && dataLocator.fallbackPaths) {
			for (const fallbackPath of dataLocator.fallbackPaths) {
				items = extractData(response, fallbackPath);
				if (items) {
					console.log(`Found data at fallback path ${logDataPath(fallbackPath)}`);
					break;
				}
			}
		}

		// If no items found through defined paths, attempt auto-detection
		if (!items) {
			console.log('Could not find data at specified paths, attempting auto-detection');

			// Try to find any array in the response
			const findArrays = (obj: any, currentPath: string[] = []): string[][] => {
				if (!obj || typeof obj !== 'object') return [];

				const paths: string[][] = [];

				Object.keys(obj).forEach((key) => {
					const value = obj[key];
					const newPath = [...currentPath, key];

					if (Array.isArray(value) && value.length > 0) {
						// Found an array with items
						paths.push(newPath);
					} else if (value && typeof value === 'object') {
						// Continue searching nested objects
						paths.push(...findArrays(value, newPath));
					}
				});

				return paths;
			};

			const possiblePaths = findArrays(response);

			if (possiblePaths.length > 0) {
				console.log('Auto-detected possible data paths:', possiblePaths.map(logDataPath));

				// Use the first detected array
				items = extractData(response, possiblePaths[0]);
				console.log(`Using auto-detected path ${logDataPath(possiblePaths[0])}`);
			}
		}

		// If we found items, process them
		if (items && items.length > 0) {
			console.log(`Retrieved ${items.length} items in this batch`);

			// Add items to our result array, respecting the limit if specified
			if (limit) {
				const remainingCapacity = limit - allItems.length;
				if (remainingCapacity < items.length) {
					// Only add what we need to reach the limit
					allItems.push(...items.slice(0, remainingCapacity));
					console.log(`Added ${remainingCapacity} items to reach requested limit`);
					hasMoreItems = false; // Stop pagination since we've reached the limit
				} else {
					allItems.push(...items);
					console.log(`Added all ${items.length} items to results`);
				}
			} else {
				allItems.push(...items);
				console.log(`Added all ${items.length} items to results`);
			}

			// Check if we need to continue pagination
			if (items.length < effectiveBatchSize) {
				// Received fewer items than requested, so we've reached the end
				console.log(
					`Received fewer items (${items.length}) than requested (${effectiveBatchSize}), pagination complete`,
				);
				hasMoreItems = false;
			} else {
				// Prepare for next batch
				offset += items.length;
				console.log(`Moving to next batch, new offset: ${offset}`);
			}
		} else {
			// No items in this response, end pagination
			console.log('No items found in response, ending pagination');
			hasMoreItems = false;
		}
	}

	console.log(`Pagination complete. Total items retrieved: ${allItems.length}`);

	// Transform the items into INodeExecutionData objects
	return allItems.map((item) => ({ json: item }));
}