@qvalia/knex-aws-data-api/src/data-api/data-api-client.js

Knex plugin that uses AWS Data API internally to execute SQL queries. Postgres & Mysql

/* eslint-disable func-names */
/* eslint-disable no-nested-ternary */
/* eslint-disable no-await-in-loop */
/* eslint-disable no-param-reassign */

/*
 * This module provides a simplified interface into the Aurora Serverless
 * Data API by abstracting away the notion of field values.
 *
 * This module is mostly copied over from https://github.com/markusahlstrand/knex-data-api-client
 * and added support for AWS SDK V3 and it's new structure changes
 *
 * More detail regarding the Aurora Serverless Data APIcan be found here:
 * https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/data-api.html
 *
 */

const {
  RDSDataClient,
  ExecuteStatementCommand,
  BatchExecuteStatementCommand,
  BeginTransactionCommand,
  CommitTransactionCommand,
  RollbackTransactionCommand
} = require('@aws-sdk/client-rds-data');

// Supported value types in the Data API
const supportedTypes = [
  'arrayValue',
  'blobValue',
  'booleanValue',
  'doubleValue',
  'isNull',
  'longValue',
  'stringValue',
  'structValue'
];

/** ***************************************************************** */
/**  PRIVATE METHODS                                               * */
/** ***************************************************************** */

// Simple error function
const error = (...err) => {
  throw Error(...err);
};

// Picked from sqlString module to reduce the dependency chain
function escapeId(val, forbidQualified) {
  const ID_GLOBAL_REGEXP = /`/g;
  const QUAL_GLOBAL_REGEXP = /\./g;
  if (Array.isArray(val)) {
    let sql = '';

    for (let i = 0; i < val.length; i++) {
      sql += (i === 0 ? '' : ', ') + escapeId(val[i], forbidQualified);
    }

    return sql;
  } if (forbidQualified) {
    return `\`${String(val).replace(ID_GLOBAL_REGEXP, '``')}\``;
  }
  return `\`${String(val).replace(ID_GLOBAL_REGEXP, '``').replace(QUAL_GLOBAL_REGEXP, '`.`')}\``;

};

// Parse SQL statement from provided arguments
const parseSQL = (args) =>
  typeof args[0] === 'string'
    ? args[0]
    : typeof args[0] === 'object' && typeof args[0].sql === 'string'
      ? args[0].sql
      : error(`No 'sql' statement provided.`);

// Parse the parameters from provided arguments
const parseParams = args =>
  Array.isArray(args[0].parameters)
    ? args[0].parameters
    : typeof args[0].parameters === 'object'
      ? [args[0].parameters]
      : Array.isArray(args[1])
        ? args[1]
        : typeof args[1] === 'object'
          ? [args[1]]
          : args[0].parameters
            ? error("'parameters' must be an object or array")
            : args[1]
              ? error("'parameters' must be an object or array")
              : [];

// Parse the supplied database, or default to config
const parseDatabase = (config, args) =>
  config.transactionId
    ? config.database
    : typeof args[0].database === 'string'
      ? args[0].database
      : args[0].database
        ? error("'database' must be a string.")
        : config.database
          ? config.database
          : undefined; // removed for #47 - error('No \'database\' provided.')

// Parse the supplied hydrateColumnNames command, or default to config
const parseHydrate = (config, args) =>
  typeof args[0].hydrateColumnNames === 'boolean'
    ? args[0].hydrateColumnNames
    : args[0].hydrateColumnNames
      ? error("'hydrateColumnNames' must be a boolean.")
      : config.hydrateColumnNames;

// Parse the supplied format options, or default to config
const parseFormatOptions = (config, args) =>
  typeof args[0].formatOptions === 'object'
    ? {
      deserializeDate:
        typeof args[0].formatOptions.deserializeDate === 'boolean'
          ? args[0].formatOptions.deserializeDate
          : args[0].formatOptions.deserializeDate
            ? error("'formatOptions.deserializeDate' must be a boolean.")
            : config.formatOptions.deserializeDate,
      treatAsLocalDate:
        typeof args[0].formatOptions.treatAsLocalDate === 'boolean'
          ? args[0].formatOptions.treatAsLocalDate
          : args[0].formatOptions.treatAsLocalDate
            ? error("'formatOptions.treatAsLocalDate' must be a boolean.")
            : config.formatOptions.treatAsLocalDate
    }
    : args[0].formatOptions
      ? error("'formatOptions' must be an object.")
      : config.formatOptions;


// Utility function for removing certain keys from an object
const omit = (obj, values) =>
  Object.keys(obj).reduce(
    (acc, x) =>
      values.includes(x) ? acc : Object.assign(acc, { [x]: obj[x] }),
    {}
  );

// Prepare method params w/ supplied inputs if an object is passed
const prepareParams = ({ secretArn, resourceArn }, args) => ({
  secretArn,
  resourceArn,
  ...(typeof args[0] === 'object' ? omit(args[0], ['hydrateColumnNames', 'parameters']) : {}) // merge any inputs
});

// Utility function for picking certain keys from an object
const pick = (obj, values) =>
  Object.keys(obj).reduce(
    (acc, x) =>
      values.includes(x) ? Object.assign(acc, { [x]: obj[x] }) : acc,
    {}
  );

// Utility function for flattening arrays
const flatten = arr => arr.reduce((acc, x) => acc.concat(x), []);

// Formats the (UTC) date to the AWS accepted YYYY-MM-DD HH:MM:SS[.FFF] format
// See https://docs.aws.amazon.com/rdsdataservice/latest/APIReference/API_SqlParameter.html
const formatToTimeStamp = (date, treatAsLocalDate) => {
  const pad = (val, num = 2) => '0'.repeat(num - String(val).length) + val;

  const year = treatAsLocalDate ? date.getFullYear() : date.getUTCFullYear();
  const month = (treatAsLocalDate ? date.getMonth() : date.getUTCMonth()) + 1; // Convert to human month
  const day = treatAsLocalDate ? date.getDate() : date.getUTCDate();

  const hours = treatAsLocalDate ? date.getHours() : date.getUTCHours();
  const minutes = treatAsLocalDate ? date.getMinutes() : date.getUTCMinutes();
  const seconds = treatAsLocalDate ? date.getSeconds() : date.getUTCSeconds();
  const ms = treatAsLocalDate
    ? date.getMilliseconds()
    : date.getUTCMilliseconds();

  const fraction = ms <= 0 ? '' : `.${pad(ms, 3)}`;

  return `${year}-${pad(month)}-${pad(day)} ${pad(hours)}:${pad(minutes)}:${pad(
    seconds
  )}${fraction}`;
};

// Converts the string value to a Date object.
// If standard TIMESTAMP format (YYYY-MM-DD[ HH:MM:SS[.FFF]]) without TZ + treatAsLocalDate=false then assume UTC Date
// In all other cases convert value to datetime as-is (also values with TZ info)
const formatFromTimeStamp = (value, treatAsLocalDate) =>
  !treatAsLocalDate &&
    /^\d{4}-\d{2}-\d{2}(\s\d{2}:\d{2}:\d{2}(\.\d{3})?)?$/.test(value)
    ? new Date(`${value}Z`)
    : new Date(value);

// Converts object params into name/value format
const splitParams = p =>
  Object.keys(p).reduce((arr, x) => arr.concat({ name: x, value: p[x] }), []);

const isDate = val => val instanceof Date;

// Gets the value type and returns the correct value field name
// TODO: Support more types as the are released
const getType = val =>
  typeof val === 'string'
    ? 'stringValue'
    : typeof val === 'boolean'
      ? 'booleanValue'
      : typeof val === 'number' && parseInt(val, 10) === val
        ? 'longValue'
        : typeof val === 'number' && parseFloat(val) === val
          ? 'doubleValue'
          : val === null
            ? 'isNull'
            : isDate(val)
              ? 'stringValue'
              : Buffer.isBuffer(val)
                ? 'blobValue'
                : // : Array.isArray(val) ? 'arrayValue' This doesn't work yet
                // TODO: there is a 'structValue' now for postgres
                typeof val === 'object' &&
                  Object.keys(val).length === 1 &&
                  supportedTypes.includes(Object.keys(val)[0])
                  ? null
                  : undefined;

// Hint to specify the underlying object type for data type mapping
const getTypeHint = val => (isDate(val) ? 'TIMESTAMP' : undefined);

// Creates a standard Data API parameter using the supplied inputs
const formatType = (name, value, type, typeHint, formatOptions) => Object.assign(
  typeHint != null ? { name, typeHint } : { name },
  type === null
    ? { value }
    : {
      value: {
        [type || error(`'${name}' is an invalid type`)]:
          type === 'isNull'
            ? true
            : isDate(value)
              ? formatToTimeStamp(
                value,
                formatOptions && formatOptions.treatAsLocalDate
              )
              : value
      }
    }
); // end formatType

// Converts parameter to the name/value format
const formatParam = (n, v, formatOptions) =>
  formatType(n, v, getType(v), getTypeHint(v), formatOptions);

// Normize parameters so that they are all in standard format
const normalizeParams = params =>
  params.reduce(
    (acc, p) =>
      Array.isArray(p)
        ? acc.concat([normalizeParams(p)])
        : (Object.keys(p).length === 2 &&
          p.name &&
          typeof p.value !== 'undefined') ||
          (Object.keys(p).length === 3 &&
            p.name &&
            typeof p.value !== 'undefined' &&
            p.cast)
          ? acc.concat(p)
          : acc.concat(splitParams(p)),
    []
  ); // end reduce

// Prepare parameters
const processParams = (
  engine,
  sql,
  sqlParams,
  params,
  formatOptions,
  row = 0
) => ({
  processedParams: params.reduce((acc, p) => {
    if (Array.isArray(p)) {
      const result = processParams(
        engine,
        sql,
        sqlParams,
        p,
        formatOptions,
        row
      );
      if (row === 0) {
        sql = result.escapedSql;
        row++;
      }
      return acc.concat([result.processedParams]);
    } if (sqlParams[p.name]) {
      if (sqlParams[p.name].type === 'n_ph') {
        if (p.cast) {
          const regex = new RegExp(`:${p.name}\\b`, 'g');
          sql = sql.replace(
            regex,
            engine === 'pg'
              ? `:${p.name}::${p.cast}`
              : `CAST(:${p.name} AS ${p.cast})`
          );
        }
        acc.push(formatParam(p.name, p.value, formatOptions));
      } else if (row === 0) {
        const regex = new RegExp(`::${p.name}\\b`, 'g');
        sql = sql.replace(regex, escapeId(p.value));
      }
      return acc;
    }
    return acc;
  }, []),
  escapedSql: sql
});

// Get all the sql parameters and assign them types
const getSqlParams = sql =>
  // TODO: probably need to remove comments from the sql
  // TODO: placeholders?
  // sql.match(/\:{1,2}\w+|\?+/g).map((p,i) => {
  (sql.match(/:{1,2}\w+/g) || [])
    .map(p =>
      // TODO: future support for placeholder parsing?
      // return p === '??' ? { type: 'id' } // identifier
      //   : p === '?' ? { type: 'ph', label: '__d'+i  } // placeholder
      p.startsWith('::')
        ? { type: 'n_id', label: p.substr(2) } // named id
        : { type: 'n_ph', label: p.substr(1) } // named placeholder
    )
    .reduce((acc, x) => Object.assign(acc, {
      [x.label]: {
        type: x.type
      }
    }), {}) // end reduce
  ;

// Format record value based on its value, the database column's typeName and the formatting options
const formatRecordValue = (value, typeName, formatOptions) =>
  formatOptions &&
    formatOptions.deserializeDate &&
    ['DATE', 'DATETIME', 'TIMESTAMP', 'TIMESTAMP WITH TIME ZONE'].includes(
      typeName
    )
    ? formatFromTimeStamp(
      value,
      (formatOptions && formatOptions.treatAsLocalDate) ||
      typeName === 'TIMESTAMP WITH TIME ZONE'
    )
    : value;

// Format updateResults and extract insertIds
const formatUpdateResults = res =>
  res.map(x => x.generatedFields && x.generatedFields.length > 0
    ? { insertId: x.generatedFields[0].longValue }
    : {});

// Merge configuration data with supplied arguments
const mergeConfig = (initialConfig, args) => Object.assign(initialConfig, args);

// Processes records and either extracts Typed Values into an array, or
// object with named column labels
const formatRecords = (recs, columns, hydrate, formatOptions) => {
  // Create map for efficient value parsing
  const fmap =
    recs && recs[0]
      ? recs[0].map((x, i) =>
      ({
        ...(columns
          ? { label: columns[i].label, typeName: columns[i].typeName }
          : {})
      }) // add column label and typeName
      )
      : {};

  // Map over all the records (rows)
  return recs
    ? recs.map(rec =>
      // Reduce each field in the record (row)
      rec.reduce(
        (acc, field, i) => {
          // If the field is null, always return null
          if (field.isNull === true) {
            return hydrate // object if hydrate, else array
              ? Object.assign(acc, { [fmap[i].label]: null })
              : acc.concat(null);

            // If the field is mapped, return the mapped field
          }
          if (fmap[i] && fmap[i].field) {
            const value = formatRecordValue(
              field[fmap[i].field],
              fmap[i].typeName,
              formatOptions
            );
            return hydrate // object if hydrate, else array
              ? Object.assign(acc, { [fmap[i].label]: value })
              : acc.concat(value);

            // Else discover the field type
          }
          // Look for non-null fields
          Object.keys(field).forEach(type => {
            if (type !== 'isNull' && field[type] !== null) {
              fmap[i].field = type;
            }
          });

          // Return the mapped field (this should NEVER be null)
          const value = formatRecordValue(
            field[fmap[i].field],
            fmap[i].typeName,
            formatOptions
          );
          return hydrate // object if hydrate, else array
            ? Object.assign(acc, { [fmap[i].label]: value })
            : acc.concat(value);
        },
        hydrate ? {} : []
      ) // init object if hydrate, else init array
    )
    : []; // empty record set returns an array
}; // end formatRecords

// Formats the results of a query response
const formatResults = (
  {
    // destructure results
    columnMetadata, // ONLY when hydrate or includeResultMetadata is true
    numberOfRecordsUpdated, // ONLY for executeStatement method
    records, // ONLY for executeStatement method
    generatedFields, // ONLY for INSERTS
    updateResults // ONLY on batchExecuteStatement
  },
  hydrate,
  includeMeta,
  formatOptions
) =>
  Object.assign(
    includeMeta ? { columnMetadata } : {},
    numberOfRecordsUpdated !== undefined && !records
      ? { numberOfRecordsUpdated }
      : {},
    records
      ? {
        records: formatRecords(
          records,
          columnMetadata,
          hydrate,
          formatOptions
        )
      }
      : {},
    updateResults ? { updateResults: formatUpdateResults(updateResults) } : {},
    generatedFields && generatedFields.length > 0
      ? { insertId: generatedFields[0].longValue }
      : {}
  );

/** ***************************************************************** */
/**  QUERY MANAGEMENT                                              * */
/** ***************************************************************** */

// Query function (use standard form for `this` context)
const query = async function (config, ..._args) {
  // Flatten array if nested arrays (fixes #30)
  const args = Array.isArray(_args[0]) ? flatten(_args) : _args;

  // Parse and process sql
  const sql = parseSQL(args);
  const sqlParams = getSqlParams(sql);

  // Parse hydration setting
  const hydrateColumnNames = parseHydrate(config, args);

  // Parse data format settings
  const formatOptions = parseFormatOptions(config, args);

  // Parse and normalize parameters
  const parameters = normalizeParams(parseParams(args));

  // Process parameters and escape necessary SQL
  const { processedParams, escapedSql } = processParams(
    config.engine,
    sql,
    sqlParams,
    parameters,
    formatOptions
  );

  // Determine if this is a batch request
  const isBatch =
    processedParams.length > 0 && Array.isArray(processedParams[0]);

  // Create/format the parameters
  const params = Object.assign(
    prepareParams(config, args),
    {
      database: parseDatabase(config, args), // add database
      sql: escapedSql // add escaped sql statement
    },
    // Only include parameters if they exist
    processedParams.length > 0
      ? // Batch statements require parameterSets instead of parameters
      { [isBatch ? 'parameterSets' : 'parameters']: processedParams }
      : {},
    // Force meta data if set and not a batch
    hydrateColumnNames && !isBatch ? { includeResultMetadata: true } : {},
    // If a transactionId is passed, overwrite any manual input
    config.transactionId ? { transactionId: config.transactionId } : {}
  ); // end params

  try {
    // attempt to run the query

    // Capture the result for debugging
    const result = await (isBatch
      ? config.RDS.send(new BatchExecuteStatementCommand(params))
      : config.RDS.send(new ExecuteStatementCommand(params)));

    // Format and return the results
    return formatResults(
      result,
      hydrateColumnNames,
      args[0].includeResultMetadata === true,
      formatOptions
    );
  } catch (e) {
    if (this && this.rollback) {
      const rollback = await config.RDS.send(
        new RollbackTransactionCommand(
          pick(params, ['resourceArn', 'secretArn', 'transactionId'])
        )
      );

      this.rollback(e, rollback);
    }
    // Throw the error
    throw e;
  }
}; // end query

/** ***************************************************************** */
/**  TRANSACTION MANAGEMENT                                        * */
/** ***************************************************************** */

// Commit transaction by running queries
const commit = async (config, queries, rollback) => {
  const results = []; // keep track of results

  // Start a transaction
  const { transactionId } = await config.RDS.send(
    new BeginTransactionCommand(
      pick(config, ['resourceArn', 'secretArn', 'database'])
    )
  );

  // Add transactionId to the config
  const txConfig = Object.assign(config, { transactionId });

  // Loop through queries
  for (let i = 0; i < queries.length; i++) {
    // Execute the queries, pass the rollback as context
    const result = await query.apply({ rollback }, [
      config,
      queries[i](results[results.length - 1], results)
    ]);
    // Add the result to the main results accumulator
    results.push(result);
  }

  // Commit our transaction
  const { transactionStatus } = await txConfig.RDS.send(
    new CommitTransactionCommand(
      pick(config, ['resourceArn', 'secretArn', 'transactionId'])
    )
  );

  // Add the transaction status to the results
  results.push({ transactionStatus });

  // Return the results
  return results;
};

// Init a transaction object and return methods
const transaction = (config, _args) => {
  const args = typeof _args === 'object' ? [_args] : [{}];
  const queries = []; // keep track of queries
  let rollback = () => { }; // default rollback event

  const txConfig = Object.assign(prepareParams(config, args), {
    database: parseDatabase(config, args), // add database
    hydrateColumnNames: parseHydrate(config, args), // add hydrate
    formatOptions: parseFormatOptions(config, args), // add formatOptions
    RDS: config.RDS // reference the RDSDataService instance
  });

  return {
    query(...argslist) {
      if (typeof argslist[0] === 'function') {
        queries.push(argslist[0]);
      } else {
        queries.push(() => [...argslist]);
      }
      return this;
    },
    rollback(fn) {
      if (typeof fn === 'function') {
        rollback = fn;
      }
      return this;
    },
    async commit() {
      return commit(txConfig, queries, rollback);
    }
  };
};

/** ***************************************************************** */
/**  INSTANTIATION                                                 * */
/** ***************************************************************** */

// Export main function
/**
 * Create a Data API client instance
 * @param {object} params
 * @param {'mysql'|'pg'} [params.engine=mysql] The type of database (MySQL or Postgres)
 * @param {string} params.resourceArn The ARN of your Aurora Serverless Cluster
 * @param {string} params.secretArn The ARN of the secret associated with your
 *   database credentials
 * @param {string} [params.database] The name of the database
 * @param {boolean} [params.hydrateColumnNames=true] Return objects with column
 *   names as keys
 * @param {object} [params.options={}] Configuration object passed directly
 *   into RDSDataService
 * @param {object} [params.formatOptions] Date-related formatting options
 * @param {boolean} [params.formatOptions.deserializeDate=false]
 * @param {boolean} [params.formatOptions.treatAsLocalDate=false]
 * @param {boolean} [params.keepAlive] DEPRECATED
 * @param {boolean} [params.sslEnabled=true] DEPRECATED
 * @param {string} [params.region] DEPRECATED
 *
 */
const init = params => {
  // Set the options for the RDSDataService
  const options =
    typeof params.options === 'object'
      ? params.options
      : params.options !== undefined
        ? error("'options' must be an object")
        : {};

  // Update the AWS http agent with the region
  if (typeof params.region === 'string') {
    options.region = params.region;
  }

  // Disable ssl if wanted for local development
  if (params.sslEnabled === false) {
    options.sslEnabled = false;
  }

  // Set the configuration for this instance
  const config = {
    // Require engine
    engine: typeof params.engine === 'string' ? params.engine : 'mysql',

    // Require secretArn
    secretArn:
      typeof params.secretArn === 'string'
        ? params.secretArn
        : error("'secretArn' string value required"),

    // Require resourceArn
    resourceArn:
      typeof params.resourceArn === 'string'
        ? params.resourceArn
        : error("'resourceArn' string value required"),

    // Load optional database
    database:
      typeof params.database === 'string'
        ? params.database
        : params.database !== undefined
          ? error("'database' must be a string")
          : undefined,

    // Load optional schema DISABLED for now since this isn't used with MySQL
    // schema: typeof params.schema === 'string' ? params.schema
    //   : params.schema !== undefined ? error(`'schema' must be a string`)
    //   : undefined,

    // Set hydrateColumnNames (default to true)
    hydrateColumnNames:
      typeof params.hydrateColumnNames === 'boolean'
        ? params.hydrateColumnNames
        : true,

    // Value formatting options. For date the deserialization is enabled and (re)stored as UTC
    formatOptions: {
      deserializeDate:
        !(typeof params.formatOptions === 'object' &&
          params.formatOptions.deserializeDate === false),
      treatAsLocalDate:
        typeof params.formatOptions === 'object' &&
        params.formatOptions.treatAsLocalDate
    },

    // Create an instance of RDSDataService
    RDS: new RDSDataClient(options)
  }; // end config

  // Return public methods
  return {
    // Query method, pass config and parameters
    query: (...x) => query(config, ...x),
    // Transaction method, pass config and parameters
    transaction: x => transaction(config, x),

    // Export promisified versions of the RDSDataService methods
    batchExecuteStatement: (args) =>
      config.RDS.send(
        new BatchExecuteStatementCommand(
          mergeConfig(pick(config, ['resourceArn', 'secretArn', 'database']), args)
        )
      ),
    beginTransaction: (args) =>
      config.RDS.send(
        new BeginTransactionCommand(
          mergeConfig(pick(config, ['resourceArn', 'secretArn', 'database']), args)
        )
      ),
    commitTransaction: (args) =>
      config.RDS.send(
        new CommitTransactionCommand(
          mergeConfig(pick(config, ['resourceArn', 'secretArn']), args)
        )
      ),
    executeStatement: (args) =>
      config.RDS.send(
        new ExecuteStatementCommand(
          mergeConfig(pick(config, ['resourceArn', 'secretArn', 'database']), args)
        )
      ),
    rollbackTransaction: (args) =>
      config.RDS.send(
        new RollbackTransactionCommand(
          mergeConfig(pick(config, ['resourceArn', 'secretArn']), args)
        )
      )
  };
}; // end exports

module.exports = init;