@getanthill/datastore/scripts/metabase.js

Event-Sourced Datastore

#!/usr/bin/env node

/**
 * You must use credentials of an admin user on Metabase. The
 * database name must be snakcased: "My Data" is becoming "my_data".
 *
 * This script is performing a synchronization of the
 * data model available in the datastore with the tables
 * and fields available in Metabase.
 *
 * Current limitations:
 * - Metabase does not allow to create fields on a table
 *   so the data available in your database will be used
 *   during the discovery process and some fields might
 *   not be available in metabase if this process is not
 *   making the field up.
 *
 * Usage:
 * METABASE_URL=http://localhost:3000 \
 *   METABASE_USERNAME=john@doe.org \
 *   METABASE_PASSWORD='my-metabase-password' \
 *   ./scripts/metabase.js <database>
 *
 * Example:
 * METABASE_URL=http://localhost:3000 \
 *   METABASE_USERNAME=john@doe.org \
 *   METABASE_PASSWORD='abcdef' \
 *   ./scripts/metabase.js models
 */

process.removeAllListeners('warning');

const fs = require('fs');
const { argv } = require('process');

const { Datastore, Metabase } = require('../dist/sdk');

const METABASE_SESSION_TOKEN_PATH = '/tmp/metabase_session_token.json';

async function authenticate(metabase) {
  if (fs.existsSync(METABASE_SESSION_TOKEN_PATH)) {
    const token = require(METABASE_SESSION_TOKEN_PATH);

    metabase.setSessionToken(token);
  } else {
    await metabase.authenticate();

    fs.writeFileSync(
      METABASE_SESSION_TOKEN_PATH,
      JSON.stringify(metabase.sessionToken),
    );
  }

  return metabase;
}

async function main(dbName, onlyCollection = null, onlyField = null) {
  console.log('[metabase] Starting metabase sync...', {
    db_name: dbName,
  });

  console.log('[metabase] Metabase authentication');
  const metabase = new Metabase({
    namespace: process.env.METABASE_NAMESPACE,
    baseUrl: process.env.METABASE_URL || 'http://localhost:9000',
    username: process.env.METABASE_USERNAME,
    password: process.env.METABASE_PASSWORD,
  });
  const datastore = new Datastore({
    baseUrl: process.env.DATASTORE_API_URL || 'http://localhost:3001',
    token: process.env.DATASTORE_ACCESS_TOKEN || 'token',
    debug: false,
  });

  await authenticate(metabase);

  const { data: models } = await datastore.getModels();
  const { data: graph } = await datastore.getGraph();
  console.log('[metabase] Available models', {
    count: Object.values(models).length,
  });

  metabase.setModels(Object.values(models));
  metabase.setGraph(graph);

  const { data: databases } = await metabase.getDatabases();
  console.log('[metabase] Available databases', {
    count: databases.length,
    names: databases.map((d) => Metabase.getNormalizedName(d)),
  });

  const database = databases.find(
    (d) => Metabase.getNormalizedName(d) === dbName,
  );
  if (!database) {
    throw new Error('Database not found!');
  }

  console.log('[metabase] Database found');

  const tables = await metabase.getDatabaseTables(database.id);
  console.log('[metabase] Available tables', {
    count: tables.length,
  });
  metabase.setTables(tables);

  console.log("[metabase] Computing tables' updates...", {
    count: tables.length,
  });
  const tableUpdates = tables
    .filter(
      (table) =>
        onlyCollection === null ||
        Metabase.getNormalizedName(table) === onlyCollection,
    )
    .map((table) => [table.id, metabase.getTableUpdatePayload(table)])
    .filter(([, p]) => p !== null);
  console.log('[metabase] Tables to update..', {
    count: tableUpdates.length,
  });

  console.log('[metabase] Fetching available fields..', {});
  const fields = [];
  for (const [tableId, table] of tableUpdates) {
    const _fields = metabase.getTableFields(tableId);

    fields.push(..._fields);
  }
  console.log('[metabase] Available fields', {
    count: fields.length,
  });
  metabase.setFields(fields);

  console.log("[metabase] Computing fields' updates...", {
    count: fields.length,
  });
  const fieldUpdates = fields
    .filter(
      (field) =>
        (onlyCollection === null ||
          Metabase.getNormalizedName({
            name: metabase.getTableById(field.table_id).name,
          }) === onlyCollection) &&
        (onlyField === null || Metabase.getNormalizedName(field) === onlyField),
    )
    .map((field) => [field.id, metabase.getFieldUpdatePayload(field)])
    .filter(([, p]) => p !== null);
  console.log('[metabase] Fields to update..', {
    count: fieldUpdates.length,
  });

  console.log('[metabase] Updating tables...', {
    count: tables.length,
  });
  for (const [id, payload] of tableUpdates) {
    await metabase.updateTable(id, payload);
  }

  console.log('[metabase] Updating fields...', {
    count: fields.length,
  });
  for (const [id, payload] of fieldUpdates) {
    await metabase.updateField(id, payload);
  }

  // if (onlyCollection !== null) {
  //   console.log(
  //     graph.edges.filter(
  //       (e) => e.source === onlyCollection || e.target === onlyCollection,
  //     ),
  //   );
  //   console.log(tableUpdates);
  //   console.log(
  //     ...fieldUpdates.map(([, field]) => {
  //       if ('fk_target_field_id' in field) {
  //         return [field, fields.find((f) => f.id === field.fk_target_field_id)];
  //       }

  //       return [field];
  //     }),
  //   );
  // }
}

main(...argv.slice(2)).catch((err) => {
  console.error(err, err?.response?.data);
  process.exit(1);
});