dbgate-tools
Version:
Auxiliary tools for other DbGate packages.
146 lines (145 loc) • 6.86 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createBulkInsertStreamBase = void 0;
const intersection_1 = __importDefault(require("lodash/intersection"));
const fromPairs_1 = __importDefault(require("lodash/fromPairs"));
const getLogger_1 = require("./getLogger");
const tableTransforms_1 = require("./tableTransforms");
const rowProgressReporter_1 = require("./rowProgressReporter");
const stringTools_1 = require("./stringTools");
const logger = (0, getLogger_1.getLogger)('bulkStreamBase');
function createBulkInsertStreamBase(driver, stream, dbhan, name, options) {
const fullNameQuoted = name.schemaName
? `${driver.dialect.quoteIdentifier(name.schemaName)}.${driver.dialect.quoteIdentifier(name.pureName)}`
: driver.dialect.quoteIdentifier(name.pureName);
const writable = new stream.Writable({
objectMode: true,
});
writable.fullNameQuoted = fullNameQuoted;
writable.buffer = [];
writable.structure = null;
writable.columnNames = null;
writable.columnDataTypes = null;
writable.requireFixedStructure = driver.databaseEngineTypes.includes('sql');
writable.rowsReporter = new rowProgressReporter_1.RowProgressReporter(options.progressName);
writable.addRow = async (row) => {
if (writable.structure) {
writable.buffer.push(row);
}
else {
writable.structure = row;
await writable.checkStructure();
}
};
writable.checkStructure = async () => {
var _a;
try {
let structure = (_a = options.targetTableStructure) !== null && _a !== void 0 ? _a : (await driver.analyseSingleTable(dbhan, name));
if (structure) {
writable.structure = structure;
}
if (structure && options.dropIfExists) {
logger.info(`DBGM-00123 Dropping table ${fullNameQuoted}`);
await driver.script(dbhan, `DROP TABLE ${fullNameQuoted}`);
}
if (options.createIfNotExists && (!structure || options.dropIfExists)) {
const dmp = driver.createDumper();
const createdTableInfo = driver.adaptTableInfo((0, tableTransforms_1.prepareTableForImport)({ ...writable.structure, ...name }));
dmp.createTable(createdTableInfo);
logger.info({ sql: dmp.s }, `DBGM-00124 Creating table ${fullNameQuoted}`);
await driver.script(dbhan, dmp.s);
structure = await driver.analyseSingleTable(dbhan, name);
writable.structure = structure;
}
if (!writable.structure) {
throw new Error(`DBGM-00125 Error importing table - ${fullNameQuoted} not found`);
}
if (options.truncate) {
await driver.script(dbhan, `TRUNCATE TABLE ${fullNameQuoted}`);
}
writable.columnNames = (0, intersection_1.default)(structure.columns.map(x => x.columnName), writable.structure.columns.map(x => x.columnName));
writable.columnDataTypes = (0, fromPairs_1.default)(writable.columnNames.map(colName => {
var _a;
return [
colName,
(_a = writable.structure.columns.find(x => x.columnName == colName)) === null || _a === void 0 ? void 0 : _a.dataType,
];
}));
}
catch (err) {
logger.error((0, stringTools_1.extractErrorLogData)(err), 'DBGM-00184 Error during preparing bulk insert table, stopped');
writable.destroy(err);
}
};
writable.send = async () => {
const rows = writable.buffer;
writable.buffer = [];
try {
if (driver.dialect.allowMultipleValuesInsert) {
const dmp = driver.createDumper();
dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
dmp.putRaw(')\n VALUES\n');
let wasRow = false;
for (const row of rows) {
if (wasRow)
dmp.putRaw(',\n');
dmp.putRaw('(');
dmp.putCollection(',', writable.columnNames, col => { var _a; return dmp.putValue(row[col], (_a = writable.columnDataTypes) === null || _a === void 0 ? void 0 : _a[col]); });
dmp.putRaw(')');
wasRow = true;
}
dmp.putRaw(';');
// require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
// console.log(dmp.s);
if (rows.length > 0) {
await driver.query(dbhan, dmp.s, { discardResult: true });
}
writable.rowsReporter.add(rows.length);
}
else {
for (const row of rows) {
const dmp = driver.createDumper();
dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
dmp.putRaw(')\n VALUES\n');
dmp.putRaw('(');
dmp.putCollection(',', writable.columnNames, col => { var _a; return dmp.putValue(row[col], (_a = writable.columnDataTypes) === null || _a === void 0 ? void 0 : _a[col]); });
dmp.putRaw(')');
// console.log(dmp.s);
await driver.query(dbhan, dmp.s, { discardResult: true });
writable.rowsReporter.add(1);
}
}
if (options.commitAfterInsert) {
const dmp = driver.createDumper();
dmp.commitTransaction();
await driver.query(dbhan, dmp.s, { discardResult: true });
}
}
catch (err) {
logger.error((0, stringTools_1.extractErrorLogData)(err), 'DBGM-00185 Error during base bulk insert, insert stopped');
writable.destroy(err);
}
};
writable.sendIfFull = async () => {
if (writable.buffer.length > 100) {
await writable.send();
}
};
writable._write = async (chunk, encoding, callback) => {
await writable.addRow(chunk);
await writable.sendIfFull();
callback();
};
writable._final = async (callback) => {
await writable.send();
writable.rowsReporter.finish();
callback();
};
return writable;
}
exports.createBulkInsertStreamBase = createBulkInsertStreamBase;