@naturalcycles/db-lib/dist/pipeline/dbPipelineRestore.d.ts

Lowest Common Denominator API to supported Databases

import { AsyncMapper, ErrorMode, UnixTimestamp } from '@naturalcycles/js-lib';
import { NDJsonStats, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
import { CommonDB } from '../common.db';
import { CommonDBSaveOptions } from '../index';
export interface DBPipelineRestoreOptions extends TransformLogProgressOptions {
    /**
     * DB to save data to.
     */
    db: CommonDB;
    /**
     * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
     * All parent directories will be created.
     */
    inputDirPath: string;
    /**
     * List of tables to dump. If undefined - will dump all files that end with .ndjson (or .ndjson.gz) extension.
     */
    tables?: string[];
    /**
     * How many tables to dump in parallel.
     *
     * @default 16
     * Set to `1` for serial (1 at a time) processing or debugging.
     */
    concurrency?: number;
    /**
     * @default 100
     *
     * Determines the size of .saveBatch()
     */
    batchSize?: number;
    /**
     * @default ErrorMode.SUPPRESS
     *
     * Used in high-level pMap(tables, ...)
     * Also used as default option for TransformMapOptions
     */
    errorMode?: ErrorMode;
    /**
     * @default undefined
     * If set - will dump maximum that number of rows per table
     */
    limit?: number;
    /**
     * If set - will do "incremental backup" (not full), only for entities that updated >= `sinceUpdated`
     *
     * @default undefined
     */
    sinceUpdated?: UnixTimestamp;
    /**
     * @default false
     * If true - will read ${table}.schema.json files and recreate tables before importing.
     * Caution! Will do `drop table if exists`!!!
     */
    recreateTables?: boolean;
    /**
     * Optionally you can provide mapper that is going to run for each table.
     *
     * @default `{}`
     * Default mappers will be "passthroughMapper" (pass all data as-is).
     */
    mapperPerTable?: Record<string, AsyncMapper>;
    /**
     * You can alter default `transformMapOptions` here.
     *
     * @default (see the code)
     * The goal to have default values that are reasonable for such a job to provide resilient output (forgiving individual errors).
     * `metric` will be set to table name
     */
    transformMapOptions?: TransformMapOptions;
    saveOptionsPerTable?: Record<string, CommonDBSaveOptions<any>>;
}
/**
 * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
 * Allows to define a mapper and a predicate to map/filter objects between input and output.
 * Handles backpressure.
 *
 * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
 */
export declare function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<NDJsonStats>;