@arcane-utils/ingestion/dist/history.d.cts

Ingestion utilities for Arcane

import winston from 'winston';
import { googleBigquery } from '@arcane-utils/bigquery';
import { IngestionHeader } from '@arcane-utils/types';

declare const getHistoricalTableName: (ingestionId: number) => string;
declare const HISTORICAL_TABLE_SCHEMA: {
    name: string;
    type: string;
}[];
/**
 * Stores execution details in a BigQuery table named historical_{ingestion_id}.
 * The table is created if it does not exist.
 * This function handles both changed files (with jobId and jobRows details) and unchanged files (same hash, without job details)
 *
 * @param bigqueryClient - Instance of the BigQuery client.
 * @param ingestionId - Ingestion Id.
 * @param monitoringId - monitoring Id of the execution.
 * @param headers - Array of ingestion headers to be stored. Will stored key values jsonified.
 * @param jobRows - Number of rows processed by the load job. 0 for unchanged files.
 * @param jobId - Load job Id to be stored. Empty string for unchanged files.
 * @param fileHash - Hash of the file being processed.
 * @param logger - Instance of the logger.
 * @returns {Promise<void>} - Resolves when the operation is complete.
 */
declare const storeExecutionDetails: (bigqueryClient: googleBigquery.BigQuery, ingestionId: number, monitoringId: string, headers: IngestionHeader[] | undefined, jobRows: number, jobId: string, fileHash: string, logger: winston.Logger) => Promise<void>;

export { HISTORICAL_TABLE_SCHEMA, getHistoricalTableName, storeExecutionDetails };