n8n-nodes-aiscraper/dist/nodes/AIScraper/postReceive.js

n8n node to call Parsera API for AI Scraping

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.unpackResponseData = unpackResponseData;
exports.pollAsyncRunResult = pollAsyncRunResult;
exports.pollAgentExtractResult = pollAgentExtractResult;
const n8n_workflow_1 = require("n8n-workflow");
async function unpackResponseData(items, response) {
    const responseBody = response.body;
    if (typeof responseBody === 'object' && responseBody !== null && 'data' in responseBody) {
        const extractedContent = responseBody.data;
        if (Array.isArray(extractedContent)) {
            return extractedContent.map(item => ({ json: item }));
        }
        else if (typeof extractedContent === 'string') {
            return [{ json: { data: extractedContent } }];
        }
        else if (typeof extractedContent === 'object' && extractedContent !== null) {
            return [{ json: extractedContent }];
        }
    }
    return items;
}
async function pollAsyncRunResult(items, response) {
    const body = response.body;
    const runId = body.run_id;
    if (!runId) {
        throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Async run did not return a run_id');
    }
    const baseURL = 'https://api.parsera.org/v1';
    const pollUrl = `${baseURL}/scrapers/run_async/${runId}`;
    const pollIntervalMs = 2000;
    const maxWaitMs = 1800000;
    const startTime = Date.now();
    while (Date.now() - startTime < maxWaitMs) {
        await (0, n8n_workflow_1.sleep)(pollIntervalMs);
        const pollResponse = await this.helpers.httpRequestWithAuthentication.call(this, 'aiScraperApi', {
            method: 'GET',
            url: pollUrl,
            json: true,
        });
        const status = pollResponse.status;
        if (status === 'completed' || status === 'completed_partial') {
            const data = pollResponse.data;
            let rows;
            if (Array.isArray(data)) {
                rows = data;
            }
            else if (typeof data === 'object' && data !== null) {
                rows = [];
                for (const key of Object.keys(data)) {
                    const val = data[key];
                    if (Array.isArray(val)) {
                        rows.push(...val);
                    }
                }
            }
            else {
                rows = [];
            }
            if (rows.length === 0) {
                return [{ json: {} }];
            }
            return rows.map((row) => ({ json: row }));
        }
        if (status === 'failed') {
            const errorMsg = pollResponse.error || pollResponse.message || 'Scraper run failed';
            throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Scraper run failed: ${errorMsg}`);
        }
    }
    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Scraper run timed out after ${maxWaitMs / 1000} seconds (run_id: ${runId})`);
}
async function pollAgentExtractResult(items, response) {
    const body = response.body;
    const taskId = body.task_id;
    if (!taskId) {
        throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Agent extract did not return a task_id');
    }
    const baseURL = 'https://api.parsera.org/v1';
    const pollUrl = `${baseURL}/agent/extract/${taskId}`;
    const pollIntervalMs = 2000;
    const maxWaitMs = 1800000;
    const startTime = Date.now();
    while (Date.now() - startTime < maxWaitMs) {
        await (0, n8n_workflow_1.sleep)(pollIntervalMs);
        const pollResponse = await this.helpers.httpRequestWithAuthentication.call(this, 'aiScraperApi', {
            method: 'GET',
            url: pollUrl,
            json: true,
        });
        const status = pollResponse.status;
        if (status === 'completed') {
            const data = pollResponse.data;
            let rows;
            if (Array.isArray(data)) {
                rows = data;
            }
            else if (typeof data === 'object' && data !== null) {
                rows = [data];
            }
            else {
                rows = [];
            }
            if (rows.length === 0) {
                return [{ json: {} }];
            }
            return rows.map((row) => ({ json: row }));
        }
        if (status === 'failed') {
            const errorMsg = pollResponse.error || pollResponse.message || 'Agent extract failed';
            throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Agent extract failed: ${errorMsg}`);
        }
    }
    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Agent extract timed out after ${maxWaitMs / 1000} seconds (task_id: ${taskId})`);
}
//# sourceMappingURL=postReceive.js.map