@pivvenit/mysql-import/mysql-import.js

Import .sql into a MySQL database with Node.

/**
 * @pivvenit/mysql-import - v5.0.23
 * Import .sql into a MySQL database with Node.
 * @author pivvenit
 * @website https://github.com/pivvenit/mysql-import/blob/master/README.md
 * @license MIT
 */

'use strict';

const mysql = require('mysql2');
const fs = require('fs');
const path = require("path");
const stream = require('stream');


/**
 * mysql-import - Importer class
 * @version 5.0.23
 * https://github.com/Pamblam/mysql-import
 */

class Importer{
	
	/**
	 * new Importer(settings)
	 * @param {host, user, password[, database]} settings - login credentials
	 */
	constructor(settings){
		this._connection_settings = settings;
		this._conn = null;
		this._encoding = 'utf8';
		this._imported = [];
		this._progressCB = undefined;
		this._dumpCompletedCB = undefined;
		this._total_files = 0;
		this._current_file_no = 0;
	}
	
	/**
	 * Get an array of the imported files
	 * @returns {Array}
	 */
	getImported(){
		return this._imported.slice(0);
	}
	
	/**
	 * Set the encoding to be used for reading the dump files.
	 * @param string - encoding type to be used.
	 * @throws {Error} - if unsupported encoding type. 
	 * @returns {undefined}
	 */
	setEncoding(encoding){
		var supported_encodings = [
			'utf8',
			'ucs2',
			'utf16le',
			'latin1',
			'ascii',
			'base64',
			'hex'
		];
		if(!supported_encodings.includes(encoding)){
			throw new Error("Unsupported encoding: "+encoding);
		}
		this._encoding = encoding;
	}
	
	/**
	 * Set or change the database to be used
	 * @param string - database name
	 * @returns {Promise}
	 */
	use(database){
		return new Promise((resolve, reject)=>{
			if(!this._conn){
				this._connection_settings.database = database;
				resolve();
				return;
			}
			this._conn.changeUser({database}, err=>{
				if (err){
					reject(err);	
				}else{
					resolve();
				}
			});
		});
	}
	
	/**
	 * Set a progress callback
	 * @param {Function} cb - Callback function is called whenever a chunk of
	 *		the stream is read. It is provided an object with the folling properties:
	 *			- total_files: The total files in the queue. 
	 *			- file_no: The number of the current dump file in the queue. 
	 *			- bytes_processed: The number of bytes of the file processed.
	 *			- total_bytes: The size of the dump file.
	 *			- file_path: The full path to the dump file.
	 * @returns {undefined}
	 */
	onProgress(cb){
		if(typeof cb !== 'function') return;
		this._progressCB = cb;
	}
	
	/**
	 * Set a progress callback
	 * @param {Function} cb - Callback function is called whenever a dump
	 *		file has finished processing.
	 *			- total_files: The total files in the queue. 
	 *			- file_no: The number of the current dump file in the queue. 
	 *			- file_path: The full path to the dump file.
	 * @returns {undefined}
	 */
	onDumpCompleted(cb){
		if(typeof cb !== 'function') return;
		this._dumpCompletedCB = cb;
	}
	
	/**
	 * Import (an) .sql file(s).
	 * @param string|array input - files or paths to scan for .sql files
	 * @returns {Promise}
	 */
	import(...input){
		return new Promise(async (resolve, reject)=>{
			try{
				await this._connect();
				var files = await this._getSQLFilePaths(...input);
				this._total_files = files.length;
				this._current_file_no = 0;
				
				var error = null;
				await slowLoop(files, (file, index, next)=>{
					this._current_file_no++;
					if(error){
						next();
						return;
					}
					this._importSingleFile(file).then(()=>{
						next();
					}).catch(err=>{
						error = err;
						next();
					});
				});
				if(error) throw error;
				await this.disconnect();
				resolve();
			}catch(err){
				reject(err);
			}
		});
	};
	
	/**
	 * Disconnect mysql. This is done automatically, so shouldn't need to be manually called.
	 * @param bool graceful - force close?
	 * @returns {Promise}
	 */
	disconnect(graceful=true){
		return new Promise((resolve, reject)=>{
			if(!this._conn){
				resolve();
				return;
			}
			if(graceful){
				this._conn.end(err=>{
					if(err){
						reject(err);
						return;
					}
					this._conn = null;
					resolve();
				});
			}else{
				this._conn.destroy();
				resolve();
			}				
		});
	}
	
	////////////////////////////////////////////////////////////////////////////
	// Private methods /////////////////////////////////////////////////////////
	////////////////////////////////////////////////////////////////////////////
	
	/**
	 * Import a single .sql file into the database
	 * @param {object} fileObj - Object containing the following properties:
	 *		- file: The full path to the file
	 *		- size: The size of the file in bytes
	 * @returns {Promise}
	 */
	_importSingleFile(fileObj){
		return new Promise((resolve, reject)=>{
			
			var parser = new queryParser({
				db_connection: this._conn,
				encoding: this._encoding,
				onProgress: this._progressCB ? (progress) => {
					this._progressCB({
						total_files: this._total_files, 
						file_no: this._current_file_no, 
						bytes_processed: progress, 
						total_bytes: fileObj.size,
						file_path: fileObj.file
					});
				} : undefined
			});
			
			const dumpCompletedCB = this._dumpCompletedCB ? (err) => this._dumpCompletedCB({
				total_files: this._total_files, 
				file_no: this._current_file_no, 
				file_path: fileObj.file,
				error: err
			}) : undefined;

			let completedCalled = false;
			
			parser.on('finish', ()=>{
				this._imported.push(fileObj.file);
				if (dumpCompletedCB && !completedCalled){
					dumpCompletedCB(null);
					completedCalled = true;
				}
				resolve();
			});
			
			
			parser.on('error', (err)=>{
				if (dumpCompletedCB && !completedCalled) {
					dumpCompletedCB(err);
					completedCalled = true;
				}
				reject(err);
			});

			var readerStream = fs.createReadStream(fileObj.file);
			readerStream.setEncoding(this._encoding);
			
			/* istanbul ignore next */
			readerStream.on('error', (err)=>{
				if (dumpCompletedCB && !completedCalled) {
					dumpCompletedCB(err);
					completedCalled = true;
				}
				reject(err);
			});
			
			readerStream.pipe(parser);
		});
	}
	
	/**
	 * Connect to the mysql server
	 * @returns {Promise}
	 */
	_connect(){
		return new Promise((resolve, reject)=>{
			if(this._conn){
				resolve(this._conn);
				return;
			}
			var connection = mysql.createConnection(this._connection_settings);
			connection.connect(err=>{
				if (err){
					reject(err);	
				}else{
					this._conn = connection;
					resolve();
				}
			});
		});
	}
	
	/**
	 * Check if a file exists
	 * @param string filepath
	 * @returns {Promise}
	 */
	_fileExists(filepath){
		return new Promise((resolve, reject)=>{
			fs.access(filepath, fs.F_OK, err=>{
				if(err){
					reject(err);
				}else{
					resolve();
				}
			});
		});
	}

	/**
	 * Get filetype information
	 * @param string filepath
	 * @returns {Promise}
	 */
	_statFile(filepath){
		return new Promise((resolve, reject)=>{
			fs.lstat(filepath, (err, stat)=>{
				if(err){
					reject(err);
				}else{
					resolve(stat);
				}
			});
		});
	}
	
	/**
	 * Read contents of a directory
	 * @param string filepath
	 * @returns {Promise}
	 */
	_readDir(filepath){
		return new Promise((resolve, reject)=>{
			fs.readdir(filepath, (err, files)=>{
				if(err){
					reject(err);
				}else{
					resolve(files);
				}
			});
		});
	}

	/**
	 * Parses the input argument(s) for Importer.import into an array sql files.
	 * @param strings|array paths
	 * @returns {Promise}
	 */
	_getSQLFilePaths(...paths){
		return new Promise(async (resolve, reject)=>{
			var full_paths = [];
			var error = null;
			paths = [].concat.apply([], paths); // flatten array of paths
			await slowLoop(paths, async (filepath, index, next)=>{
				if(error){
					next();
					return;
				}
				try{
					await this._fileExists(filepath);
					var stat = await this._statFile(filepath);
					/* istanbul ignore else  */
					if(stat.isFile()){
						if(filepath.toLowerCase().substring(filepath.length-4) === '.sql'){
							full_paths.push({
								file: path.resolve(filepath),
								size: stat.size
							});
						}
						next();
					}else if(stat.isDirectory()){
						var more_paths = await this._readDir(filepath);
						more_paths = more_paths.map(p=>path.join(filepath, p));
						var sql_files = await this._getSQLFilePaths(...more_paths);
						full_paths.push(...sql_files);
						next();
					}else{
						/* istanbul ignore next */
						next();
					}
				}catch(err){
					error = err;
					next();
				}
			});
			if(error){
				reject(error);
			}else{
				resolve(full_paths);
			}
		});
	}
	
}

/**
 * Build version number
 */
Importer.version = '5.0.23';

module.exports = Importer;

/**
 * Execute the loopBody function once for each item in the items array, 
 * waiting for the done function (which is passed into the loopBody function)
 * to be called before proceeding to the next item in the array.
 * @param {Array} items - The array of items to iterate through
 * @param {Function} loopBody - A function to execute on each item in the array.
 *		This function is passed 3 arguments - 
 *			1. The item in the current iteration,
 *			2. The index of the item in the array,
 *			3. A function to be called when the iteration may continue.
 * @returns {Promise} - A promise that is resolved when all the items in the 
 *		in the array have been iterated through.
 */
function slowLoop(items, loopBody) {
	return new Promise(f => {
		/* istanbul ignore next */
		if(!items.length) return f();
		let done = arguments[2] || f;
		let idx = arguments[3] || 0;
		let cb = items[idx + 1] ? () => slowLoop(items, loopBody, done, idx + 1) : done;
		loopBody(items[idx], idx, cb);
	});
}


class queryParser extends stream.Writable{
	
	constructor(options){
		/* istanbul ignore next */
		options = options || {};
		super(options);
		
		// The number of bytes processed so far
		this.processed_size = 0;
		
		// The progress callback
		this.onProgress = options.onProgress;
		
		// the encoding of the file being read
		// No default value anymore, since it is always provided by the parent caller.
		this.encoding = options.encoding;
		
		// the encoding of the database connection
		this.db_connection = options.db_connection;
		
		// The quote type (' or ") if the parser 
		// is currently inside of a quote, else false
		this.quoteType = false;
		
		// An array of chars representing the substring
		// the is currently being parsed
		this.buffer = [];
		
		// Is the current char escaped
		this.escaped = false;
		
		// The string that denotes the end of a query
		this.delimiter = ';';

		// Literal string to look for while checking for a mysql delimiter set statement
		this.delimiterLiteral = 'delimiter ';

		// Temporary buffer used while checking for a mysql delimiter set statement
		this.delimiterParseBuffer = '';

		// Are we currently seeking new delimiter
		this.seekingDelimiter = false;
		
	}
	
	////////////////////////////////////////////////////////////////////////////
	// "Private" methods" //////////////////////////////////////////////////////
	////////////////////////////////////////////////////////////////////////////
	
	// handle piped data
	async _write(chunk, enc, next) {
		var query;
		chunk = chunk.toString(this.encoding);
		var error = null;
		for (let i = 0; i < chunk.length; i++) {
			let char = chunk[i];
			query = this.parseChar(char);
			try{
				if(query) await this.executeQuery(query);
			}catch(e){
				error = e;
				break;
			}
		}
		this.processed_size += chunk.length;
		if (this.onProgress) {
			this.onProgress(this.processed_size);
		}
		next(error);
	}
	
	// Execute a query, return a Promise
	executeQuery(query){
		return new Promise((resolve, reject)=>{
			this.db_connection.query(query, err=>{
				if (err){
					reject(err);
				}else{
					resolve();
				}
			});
		});
	}
	
	// Parse the next char in the string
	// return a full query if one is detected after parsing this char
	// else return false.
	parseChar(char){
		this.checkEscapeChar();
		this.buffer.push(char);
		this.checkNewDelimiter(char);
		this.checkQuote(char);
		return this.checkEndOfQuery();
	}
	
	// Check if the current char has been escaped
	// and update this.escaped
	checkEscapeChar(){
		if(!this.buffer.length) return;
		if(this.buffer[this.buffer.length - 1] === "\\"){
			this.escaped = !this.escaped;
		}else{
			this.escaped = false;
		}
	}
	
	// Check to see if a new delimiter is being assigned
	checkNewDelimiter(char){
		var lowerChar = char.toLowerCase();
		var isNewLine = lowerChar === "\n" || lowerChar === "\r";
		if(isNewLine && this.seekingDelimiter){
			// We found the new delimiter, reset locals
			this.seekingDelimiter = false;
			this.delimiterParseBuffer = '';
			// Clear buffer so we can continue looking for new queries
			this.buffer = [];
		}else if(this.seekingDelimiter){
			// Append the char to the delimiter string, we haven't encountered a newline yet
			this.delimiter += lowerChar;
		}

		// If the current character is the one we expect given the delimiter literal
		if(!this.quoteType && lowerChar == this.delimiterLiteral[this.delimiterParseBuffer.length]){
			this.delimiterParseBuffer += lowerChar;
		}else{
			// Reset temp buffer because current character didn't match, we have to start over again
			this.delimiterParseBuffer = '';
		}

		// We matched the whole literal so we can start looking for the delimiter value itself
		if(this.delimiterLiteral === this.delimiterParseBuffer.toString()){
			this.seekingDelimiter = true;
			// Reset the current delimiter because we now can start parsing a new one
			this.delimiter = '';
		}
	}
	
	// Check if the current char is a quote
	checkQuote(char){
		var isQuote = (char === '"' || char === "'") && !this.escaped;
		if (isQuote && this.quoteType === char){
			this.quoteType = false;
		}else if(isQuote && !this.quoteType){
			this.quoteType = char;
		}
	}
	
	// Check if we're at the end of the query
	// return the query if so, else return false;
	checkEndOfQuery(){
		if(this.seekingDelimiter){
			return false;
		}

		var query = false;
		var delimiterFound = false;
		if(!this.quoteType && this.buffer.length >= this.delimiter.length){
			delimiterFound = this.buffer.slice(-this.delimiter.length).join('') === this.delimiter;
		}

		if (delimiterFound) {
			// trim the delimiter off the end
			this.buffer.splice(-this.delimiter.length, this.delimiter.length);
			query = this.buffer.join('').trim();
			this.buffer = [];
		}
		
		return query;
	}
}