@cloudquery/plugin-sdk-javascript/dist/schema/table.js

This is the high-level package to use for developing CloudQuery plugins in JavaScript

import { Table as ArrowTable, tableFromIPC, tableToIPC, Schema } from '@apache-arrow/esnext-esm';
import { isMatch } from 'matcher';
import { TableError } from '../errors/errors.js';
import * as arrow from './arrow.js';
import { fromArrowField, toArrowField } from './column.js';
export const createTable = ({ name = '', title = '', description = '', columns = [], relations = [], transform = () => { }, resolver = () => Promise.resolve(), multiplexer = (client) => [client], postResourceResolver = () => Promise.resolve(), preResourceResolver = () => Promise.resolve(), isIncremental = false, ignoreInTests = false, parent = null, pkConstraintName = '', } = {}) => ({
    name,
    title,
    description,
    columns,
    relations,
    transform,
    resolver,
    multiplexer,
    postResourceResolver,
    preResourceResolver,
    isIncremental,
    ignoreInTests,
    parent,
    pkConstraintName,
});
export const getTablesNames = (tables) => tables.map((table) => table.name);
export const getTopLevelTableByName = (tables, name) => tables.find((table) => table.name === name);
export const getTableByName = (tables, name) => {
    const table = tables.find((table) => table.name === name);
    if (table) {
        return table;
    }
    for (const table of tables) {
        const found = getTableByName(table.relations, name);
        if (found) {
            return found;
        }
    }
};
export const getPrimaryKeys = (table) => {
    return table.columns.filter((column) => column.primaryKey).map((column) => column.name);
};
export const flattenTables = (tables) => {
    return tables.flatMap((table) => [table, ...flattenTables(table.relations.map((c) => ({ ...c, parent: table })))]);
};
export const getAllParents = (table) => {
    if (table.parent === null) {
        return [];
    }
    return [table.parent, ...getAllParents(table.parent)];
};
const getAllChildren = (table) => {
    return table.relations.flatMap((relation) => [relation, ...getAllChildren(relation)]);
};
const filterByNamesRecursive = (tables, names) => {
    const filtered = tables
        .filter((table) => names.includes(table.name))
        .map((table) => ({ ...table, relations: filterByNamesRecursive(table.relations, names) }));
    return filtered;
};
const deduplicate = (tables) => {
    return tables.filter((table, index, array) => array.findIndex((t) => t.name === table.name) === index);
};
export const filterTables = (tables, include, skip, skipDependantTables) => {
    const flattened = flattenTables(tables);
    const withIncludes = flattened.filter((table) => isMatch(table.name, include));
    // Include all children of included tables if skipDependantTables is false
    const withChildren = skipDependantTables
        ? withIncludes
        : deduplicate(withIncludes.flatMap((table) => [table, ...getAllChildren(table)]));
    // If a child was included, include the parent as well
    const withParents = deduplicate(withChildren.flatMap((table) => [...getAllParents(table), table]));
    const withSkipped = withParents.filter((table) => {
        return !isMatch(table.name, skip) && !getAllParents(table).some((parent) => isMatch(parent.name, skip));
    });
    const skippedParents = withParents
        .filter((table) => table.parent && !withSkipped.includes(table.parent))
        .map((table) => table.parent.name);
    if (skippedParents.length > 0) {
        throw new TableError(`Can't skip parent table when child table is included. Skipped parents are: ${skippedParents.join(', ')}`);
    }
    const filtered = filterByNamesRecursive(tables, withSkipped.map((table) => table.name));
    return filtered;
};
export const toArrowSchema = (table) => {
    const metadata = new Map();
    metadata.set(arrow.METADATA_TABLE_NAME, table.name);
    metadata.set(arrow.METADATA_TABLE_DESCRIPTION, table.description);
    metadata.set(arrow.METADATA_TABLE_TITLE, table.title);
    metadata.set(arrow.METADATA_CONSTRAINT_NAME, table.pkConstraintName);
    if (table.isIncremental) {
        metadata.set(arrow.METADATA_INCREMENTAL, arrow.METADATA_TRUE);
    }
    if (table.parent) {
        metadata.set(arrow.METADATA_TABLE_DEPENDS_ON, table.parent.name);
    }
    const fields = table.columns.map((c) => toArrowField(c));
    return new Schema(fields, metadata);
};
export const fromArrowSchema = (schema) => {
    return createTable({
        name: schema.metadata.get(arrow.METADATA_TABLE_NAME) || '',
        title: schema.metadata.get(arrow.METADATA_TABLE_TITLE) || '',
        description: schema.metadata.get(arrow.METADATA_TABLE_DESCRIPTION) || '',
        pkConstraintName: schema.metadata.get(arrow.METADATA_CONSTRAINT_NAME) || '',
        isIncremental: schema.metadata.get(arrow.METADATA_INCREMENTAL) === arrow.METADATA_TRUE,
        // dependencies: schema.metadata.get(arrow.METADATA_TABLE_DEPENDS_ON) || '',
        columns: schema.fields.map((f) => fromArrowField(f)),
    });
};
export const encodeTable = (table) => {
    const schema = toArrowSchema(table);
    const arrowTable = new ArrowTable(schema);
    return tableToIPC(arrowTable);
};
export const encodeTables = (tables) => {
    return tables.map((table) => encodeTable(table));
};
export const decodeTable = (bytes) => {
    const arrowTable = tableFromIPC(bytes);
    return fromArrowSchema(arrowTable.schema);
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export const decodeRecord = (bytes) => {
    const arrowTable = tableFromIPC(bytes);
    return [(arrowTable.schema.metadata.get(arrow.METADATA_TABLE_NAME) || ''), arrowTable.batches];
};
//# sourceMappingURL=table.js.map