@naturalcycles/db-lib/dist/pipeline/dbPipelineBackup.d.ts

Lowest Common Denominator API to supported Databases

import { ErrorMode } from '@naturalcycles/js-lib/error';
import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-lib/types';
import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
import type { CommonDB } from '../commondb/common.db.js';
import { DBQuery } from '../query/dbQuery.js';
export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
    /**
     * DB to dump data from.
     */
    db: CommonDB;
    /**
     * List of tables to dump. If undefined - will call CommonDB.getTables() and dump ALL tables returned.
     */
    tables?: string[];
    /**
     * How many tables to dump in parallel.
     *
     * @default 16
     * Set to `1` for serial (1 at a time) processing or debugging.
     */
    concurrency?: number;
    /**
     * @default ErrorMode.SUPPRESS
     *
     * Used in high-level pMap(tables, ...)
     * Also used as default option for TransformMapOptions
     */
    errorMode?: ErrorMode;
    /**
     * @default undefined
     * If set - will dump maximum that number of rows per table
     */
    limit?: number;
    /**
     * If set - will do "incremental backup" (not full), only for entities that updated >= `sinceUpdated`
     */
    sinceUpdated?: UnixTimestamp;
    /**
     * Map for each table a `sinceUpdated` timestamp, or `undefined`.
     * If set - will do "incremental backup" (not full), only for entities that updated >= `sinceUpdated` (on a per table basis)
     */
    sinceUpdatedPerTable?: StringMap<UnixTimestamp>;
    /**
     * By default, dbPipelineBackup creates a Query based on sinceUpdated.
     * But if queryPerTable is set for a table - it will override the Query that is ran for that table
     * (and ignore sinceUpdated, sinceUpdatedPerTable, limit, and any other properties that modify the query).
     */
    queryPerTable?: StringMap<DBQuery<any>>;
    /**
     * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
     * All parent directories will be created.
     *
     * @default to process.cwd()
     */
    outputDirPath: string;
    /**
     * @default false
     * If true - will fail if output file already exists.
     */
    protectFromOverwrite?: boolean;
    /**
     * Compress as .zst
     * @default true
     */
    zst?: boolean;
    /**
     * Only applicable if `gzip` is enabled
     * Currently not available.
     */
    /**
     * Optionally you can provide mapper that is going to run for each table.
     *
     * @default `{}`
     * Default mappers will be "passthroughMapper" (pass all data as-is).
     */
    mapperPerTable?: StringMap<AsyncMapper>;
    /**
     * If defined - it'll use that `logEvery` for that table.
     * Default logEvery is 1000.
     */
    logEveryPerTable?: StringMap<number>;
    /**
     * You can alter default `transformMapOptions` here.
     *
     * @default (see the code)
     * The goal to have default values that are reasonable for such a job to provide resilient output (forgiving individual errors).
     * `metric` will be set to table name
     */
    transformMapOptions?: TransformMapOptions;
    /**
     * @default false
     * If true - will use CommonSchemaGenerator to detect schema from input data.
     */
    /**
     * @default false
     * If true - will use CommonDB.getTableSchema() and emit schema.
     */
    emitSchemaFromDB?: boolean;
}
/**
 * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
 * File is overwritten (by default).
 * Input stream can be a stream from CommonDB.streamQuery()
 * Allows to define a mapper and a predicate to map/filter objects between input and output.
 * Handles backpressure.
 *
 * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
 */
export declare function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<NDJsonStats>;