sql-fixtures
Version:
Populate a SQL database with fixture data
220 lines (193 loc) • 8.5 kB
JavaScript
/*
* Calls into knex to actually insert the records into the database
* and returns an array of promises that knex generated
*
* Also massages the result of knex's insert into entire hydrated records.
*
* Database differences
* ====================
* The insertion happens differently for postgres versus all the other supported
* databases. The return result of an insert in Postgres can be the actual records
* that were inserted. sql-fixtures takes advantage of that in order to return to
* the user their data. For MySQL et al, this is not true. The best you get is
* the id (primary key) of the last record that got inserted.
*
* So for non-postgres dbs, the insertions happen serially, and after each insert
* a select is done to grab the inserted record.
*
* This also means for non-postgres, if a table lacks a primary key, then it can
* lead into undefined behavior. Also see
* http://city41.github.io/node-sql-fixtures/#no-primary-key-warning
*/
var _ = require('lodash');
var bluebird = require('bluebird');
var isPostgres = require('./is-postgres');
var isSqlite = require('./is-sqlite');
function removeExtraKeys(trimmedRecord, value, key) {
if (key !== 'specId' && value !== null && typeof value !== 'undefined') {
trimmedRecord[key] = value;
}
}
function buildRawSqlPromises(knex, sqls) {
return sqls.map(function(rawSql) {
var sqlPromise = knex.raw(rawSql).then(function(result) {
return {};
});
return sqlPromise;
});
}
function getInsertableRecords(knex, tableName, candidateRecords, unique) {
if (unique) {
var checkIfEquivalentRecordExistsPromises = _.map(candidateRecords, function(candidateRecord) {
return knex(tableName).where(candidateRecord).then(function(result) {
if (result.length === 0) {
return candidateRecord;
}
});
});
return bluebird.all(checkIfEquivalentRecordExistsPromises).then(function(results) {
return _.compact(results);
});
} else {
return bluebird.resolve(candidateRecords);
}
}
function getAllKeys(records) {
var keys = _.reduce(records, function(keysResult, record) {
keysResult = keysResult.concat(_.keys(record));
return keysResult;
}, []);
return _.compact(_.uniq(keys));
}
/**
* insertRecordsSerially, the "Achilles heel" of sql-fixtures
*
* TL;DR: sql-fixtures was originally created for postgres and takes advantage
* of a postgres specific feature. Expanding to mysql, maria and sqlite has been
* a problem because they lack that feature. This method *mostly* addresses the problem
*
* The problem: After inserting a record sql-fixtures needs to retrieve the entire
* record in order to resolve downstream dependencies. If nothing else, the record's
* ID is needed, to resolve downstream relation dependencies, but autogenerated columns
* also need to be retrieved. With Postgres, the return value of an insert is
* the record that was created, which is awesome and works perfecty. Thank you Postgres!
* No other database does this (booooo). MySQL and Maria instead offer LAST_INSERT_ID()
* which you can then use to select the inserted record, but only if the table has a
* primary key. sqlite also has this feature using rowids. The problem emerges for tables
* that lack a singular primary key column
*
* MySQL/Maria with a singular ID column
* -----------
* if they have a singular ID column, this function works perfectly. We are forced
* to insert records serially, but we get expected results and it's solid
*
* MySQL/Maria without a singular ID column
* -----------
* trouble can brew here. Since there is no ID column, LAST_INSERT_ID() does not work.
* We fall back to doing a select using the record's spec and hope we get the right row.
* we *usually* get the right row and things are *usually* fine, but datetime rows
* can mess this up.
*
* Sqlite with rowids, still TODO
* -----------
* Sqlite has rowids, a secret primary key column that is added by default. It allows
* us to avoid this problem and get perfect results. If someone creates a sqlite table
* and specifies "without rowid", then sql-fixtures will not support that case.
* TODO: using rowids is still todo
*/
function insertRecordsSerially(knex, tableName, insertRecords, showWarning) {
var insertedRecords = [];
function onInsertedResult(index, result, showWarning) {
if (showWarning && (!result || result.length === 0)) {
// this happens if our fallback failed. Warn the user and move on, not
// much else we can do :-/
console.warn("Failed to retrieve the most recently inserted record for table " + tableName +
", you will probably get unexpected results" +
"see: http://city41.github.io/node-sql-fixtures/#no-primary-key-warning");
}
insertedRecords[index] = result[0];
return insertRecordAt(index + 1);
}
function insertRecordAt(index) {
if (index < insertRecords.length) {
return knex(tableName).insert(insertRecords[index]).then(function(insertResult) {
var selectPromise;
if (!insertResult || insertResult.length === 0 || insertResult[0] < 1) {
// table lacks an ID column, we are most likely in mysql/maria
// this noop promise allows us to go into the fallback (see below)
selectPromise = bluebird.resolve();
} else {
// we have a good ID column, awesome, all databases work well if we get here
// if sqlite, there is a hidden rowid column, which is what knex returned
// it's possible to get here without an id column, but not a rowid column,
// this makes sqlite 100% accurate, as long as the user does not use "without rowid"
// when creating their tables
var idColumn = isSqlite(knex) ? 'rowid' : 'id';
var idQuery = {};
idQuery[idColumn] = insertResult[0];
selectPromise = knex(tableName).where(idQuery);
}
return selectPromise.then(function(retrievedRecordResult) {
if (!retrievedRecordResult || retrievedRecordResult.length === 0) {
// fallback: no id column, failed to get the row, we will try with a
// generic where using the spec itself as the where clause. This
// *usually* works, but can fail in certain scenarios.
return knex(tableName).where(insertRecords[index]).limit(1).then(function(finalResult) {
return onInsertedResult(index, finalResult, showWarning);
});
} else {
return onInsertedResult(index, retrievedRecordResult, false);
}
});
});
} else {
return bluebird.resolve(insertedRecords);
}
}
return insertRecordAt(0);
}
function buildInsertPromise(knex, tableName, records, unique, showWarning) {
var insertRecords = _.map(records, function(record) {
return _.transform(record, removeExtraKeys);
});
if (unique) {
insertRecords = _.uniqBy(insertRecords, getAllKeys(insertRecords));
}
function assembleFinalResult(insertedRecords) {
var finalResult = {};
finalResult[tableName] = _.map(insertedRecords, function(insertedRecord, i) {
// only attach keys that were passed with the data
var recordResult = _.pick(insertedRecord, _.union(_.keys(records[i]), ['id']));
return _.extend(records[i], recordResult);
});
return finalResult;
}
return getInsertableRecords(knex, tableName, insertRecords, unique).then(function(insertableRecords) {
insertRecords = insertableRecords;
if (insertRecords.length === 0) {
return assembleFinalResult(insertRecords);
}
if (isPostgres(knex)) {
return knex(tableName).returning('*').insert(insertRecords).then(function(insertResults) {
return assembleFinalResult(insertResults);
});
} else {
return insertRecordsSerially(knex, tableName, insertRecords, showWarning).then(function(insertResults) {
return assembleFinalResult(insertResults);
});
}
});
}
module.exports = function insertRecords(knex, configs, unique, showWarning) {
var promises = [];
_.forIn(configs, function(records, table) {
if (table === 'sql') {
var sqlPromises = buildRawSqlPromises(knex, records);
promises = promises.concat(sqlPromises);
} else {
var insertPromise = buildInsertPromise(knex, table, records, unique, showWarning);
promises.push(insertPromise);
}
});
return promises;
};