total.js/nosql.js

MVC framework for Node.js

// Copyright 2012-2020 (c) Peter Širka <petersirka@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.

/**
 * @module NoSQL
 * @version 3.4.4
 */

'use strict';

const Readable = require('stream').Readable;
const Fs = require('fs');
const Path = require('path');
const NoSQLStream = require('./nosqlstream');
const REG_FIELDS_CLEANER = /"|`|\||'|\s/g;

if (!global.framework_utils)
	global.framework_utils = require('./utils');

if (!global.framework_image)
	global.framework_image = require('./image');

if (!global.framework_nosql)
	global.framework_nosql = exports;

if (!global.framework_builders)
	global.framework_builders = require('./builders');

const EXTENSION = '.nosql';
const EXTENSION_TABLE = '.table';
const EXTENSION_TABLE_BACKUP = '.table-backup';
const EXTENSION_BINARY = '.nosql-binary';
const EXTENSION_LOG = '.nosql-log';
const EXTENSION_MAPREDUCE = '.nosql-mapreduce';
const EXTENSION_BACKUP = '.nosql-backup';
const EXTENSION_META = '.meta';
const EXTENSION_COUNTER = '-counter2';
const BINARY_HEADER_LENGTH = 2000;
const COUNTER_MMA = [0, 0];
const DIRECTORYLENGTH = 9;
const FLAGS_READ = ['get'];
const INMEMORY = {};
const JSONBOOL = '":true ';
const NEWLINE = '\n';
const REGBOOL = /":true/g; // for updates of boolean types
const REGCHINA = /[\u3400-\u9FBF]/;
const REGCLEAN = /^[\s]+|[\s]+$/g;
const REGTESCAPE = /\||\n|\r/g;
const REGTUNESCAPE = /%7C|%0D|%0A/g;
const REGTESCAPETEST = /\||\n|\r/;
const IMAGES = { gif: 1, jpg: 1, jpeg: 1, png: 1, svg: 1 };
const BINARYREADDATA = { start: BINARY_HEADER_LENGTH };
const BINARYREADDATABASE64 = { start: BINARY_HEADER_LENGTH, encoding: 'base64' };
const BINARYREADMETA = { start: 0, end: BINARY_HEADER_LENGTH - 1, encoding: 'binary' };
const BOOLEAN = { '1': 1, 'true': 1, 'on': 1 };
const TABLERECORD = { '+': 1, '-': 1, '*': 1 };
const CLUSTERMETA = {};
const UNKNOWN = 'unknown';
const MKDIR = { recursive: true };

const COMPARER = global.Intl ? global.Intl.Collator().compare : function(a, b) {
	return a.removeDiacritics().localeCompare(b.removeDiacritics());
};

const NEWLINEBUF = Buffer.from('\n', 'utf8');
const CACHE = {};

var JSONBUFFER = process.argv.findIndex(n => n.endsWith('nosqlworker.js')) === -1 ? 20 : 40;
var FORK;
var FORKCALLBACKS;

function clusterlock(db, method) {
	Fs.open(db.filenameLock, 'wx', function(err, fd) {

		if (err) {
			setTimeout(clusterlock, 100, db, method);
			return;
		}

		Fs.write(fd, F.id.toString(), function(err) {
			err && F.error('NoSQLStream.lock.write()', err);
			Fs.close(fd, function(err) {
				err && F.error('NoSQLStream.lock.close()', err);
				db.locked = true;
				db[method]();
			});
		});
	});
}

function clusterunlock(db) {
	if (db.locked) {
		db.locked = false;
		Fs.unlink(db.filenameLock, NOOP);
	}
}

function promise(fn) {
	var self = this;
	return new Promise(function(resolve, reject) {
		self.callback(function(err, result) {
			if (err)
				reject(err);
			else
				resolve(fn == null ? result : fn(result));
		});
	});
}

exports.kill = function(signal) {
	FORK && TRY(() => FORK && FORK.kill && FORK.kill(signal || 'SIGTERM'));
};

exports.pid = function() {
	return FORK ? FORK.pid : 0;
};

exports.worker = function() {

	if (FORK || F.isCluster)
		return;

	// Clears unhandled callbacks
	ON('service', function() {

		var keys = Object.keys(FORKCALLBACKS);
		if (!keys.length)
			return;

		var time = Date.now();

		for (var i = 0, length = keys.length; i < length; i++) {
			var key = keys[i];
			var item = FORKCALLBACKS[key];
			if (item && item.time) {
				var diff = time - item.time;
				if (diff >= 60000) {
					delete FORKCALLBACKS[key];
					var err = new Error('NoSQL worker timeout.');
					switch (item.type) {
						case 'find':
							item.builder && item.builder.$callback2(err, EMPTYARRAY, 0, EMPTYOBJECT);
							break;
						case 'count':
							item.builder && item.builder.$callback2(err, EMPTYOBJECT, 0, EMPTYOBJECT);
							break;
						case 'insert':
						case 'update':
						case 'remove':
							item.builder && item.builder.$callback && item.builder.$callback(err, EMPTYOBJECT, EMPTYOBJECT);
							break;
						case 'clean':
						case 'clear':
							item.callback && item.callback(err);
							break;
						case 'stream':
							item.callback && item.callback(err, EMPTYOBJECT, 0);
							break;
						default:
							item.callback && item.callback(err, EMPTYOBJECT, EMPTYOBJECT);
							break;
					}
				}
			}
		}
	});

	FORKCALLBACKS = {};
	FORK = require('child_process').fork(module.filename.replace(/\.js$/, '') + 'worker.js', [], { cwd: F.directory });
	FORK.send({ TYPE: 'init', directory: F.path.root() });
	FORK.on('message', function(msg) {
		switch (msg.TYPE) {
			case 'find':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.builder.$callback2(msg.err, msg.response, msg.count, msg.repository);
				break;
			case 'count':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.builder.$callback2(msg.err, msg.response, msg.count, msg.repository);
				break;
			case 'insert':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.builder.$callback && obj.builder.$callback(msg.err, msg.response, msg.repository);
				break;
			case 'update':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.builder.$callback && obj.builder.$callback(msg.err, msg.response, msg.repository);
				break;
			case 'remove':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.builder.$callback && obj.builder.$callback(msg.err, msg.response, msg.repository);
				break;
			case 'backup':
			case 'restore':
			case 'counter.read':
			case 'counter.stats':
			case 'counter.clear':
			case 'storage.stats':
			case 'storage.clear':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.callback && obj.callback(msg.err, msg.response);
				break;
			case 'stream':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.callback && obj.callback(msg.err, msg.repository || {}, msg.count);
				break;
			case 'storage.scan':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.callback && obj.callback(msg.err, msg.response, msg.repository);
				break;
			case 'callback':
				var obj = FORKCALLBACKS[msg.id];
				obj && obj.callback && obj.callback(msg.err);
				break;

		}
		delete FORKCALLBACKS[msg.id];
	});

	var CMD = {};

	function send(instance, type) {
		var obj = {};
		obj.type = type;
		obj.name = instance.name;
		obj.time = Date.now();
		obj.t = instance instanceof Table;

		if (arguments.length > 2) {
			obj.arg = [];
			for (var i = 2; i < arguments.length; i++)
				obj.arg.push(arguments[i]);
		}
		setImmediate(send2, obj);
		return obj;
	}

	function notify(instance, type) {
		var obj = {};
		obj.type = type;
		obj.name = instance.name;
		obj.time = Date.now();
		obj.t = instance instanceof Table;

		if (arguments.length > 2) {
			obj.arg = [];
			for (var i = 2; i < arguments.length; i++)
				obj.arg.push(arguments[i]);
		}
		setImmediate(send2, obj, false);
		return obj;
	}

	function send2(obj, callback) {
		CMD.TYPE = obj.type;
		CMD.arg = obj.arg;
		CMD.data = obj.builder ? obj.builder.stringify() : null;
		CMD.name = obj.name;
		CMD.t = obj.t;
		if (callback !== false) {
			CMD.id = Math.random().toString(32).substring(2);
			FORKCALLBACKS[CMD.id] = obj;
		}
		FORK.send(CMD);
	}

	var DP = Database.prototype;
	var CP = Counter.prototype;
	var SP = Storage.prototype;

	TP.once = TP.on = TP.emit = TP.removeListener = TP.removeAllListeners = DP.once = DP.on = DP.emit = DP.removeListener = DP.removeAllListeners = CP.on = CP.once = CP.emit = CP.removeListener = CP.removeAllListeners = function() {
		PRINTLN('ERROR --> NoSQL events are not supported in fork mode.');
	};

	TP.listing = TP.list = DP.listing = DP.list = function(builder) {
		if (builder instanceof DatabaseBuilder)
			builder.db = this;
		else
			builder = new DatabaseBuilder(this);
		builder.$options.listing = true;
		builder.$take = builder.$options.take = 100;
		return send(this, 'find').builder = builder;
	};

	TP.find = DP.find = function(builder) {
		if (builder instanceof DatabaseBuilder)
			builder.db = this;
		else
			builder = new DatabaseBuilder(this);
		return send(this, 'find').builder = builder;
	};

	TP.find2 = DP.find2 = function(builder) {
		if (builder instanceof DatabaseBuilder)
			builder.db = this;
		else
			builder = new DatabaseBuilder(this);
		return send(this, 'find2').builder = builder;
	};

	TP.top = DP.top = function(max) {
		var builder = new DatabaseBuilder(this);
		builder.take(max);
		return send(this, 'find').builder = builder;
	};

	TP.one = DP.one = function() {
		var builder = new DatabaseBuilder(this);
		builder.first();
		return send(this, 'one').builder = builder;
	};

	TP.insert = DP.insert = function(doc, unique) {

		var self = this;
		var builder;

		if (doc.$$schema)
			doc = doc.$clean();

		if (unique) {
			builder = self.one();

			var callback;

			builder.callback(function(err, d) {
				if (d)
					callback && callback(null, 0);
				else {
					var tmp = self.insert(doc);
					tmp.callback(callback);
					tmp.$options.log = builder.$options.log;
				}
			});

			builder.callback = function(fn) {
				callback = fn;
				return builder;
			};

			return builder;
		}

		return send(self, 'insert', doc).builder = new DatabaseBuilder2(self);
	};

	TP.count = DP.count = function() {
		var builder = new DatabaseBuilder(this);
		return send(this, 'count').builder = builder;
	};

	DP.view = function() {
		throw new Error('NoSQL Views are not supported.');
	};

	TP.update = DP.update = function(doc, insert) {
		var val = doc.$$schema ? doc.$clean() : doc;
		if (typeof(val) === 'function')
			val = val.toString();
		return send(this, 'update', val, insert).builder = new DatabaseBuilder(this);
	};

	TP.modify = DP.modify = function(doc, insert) {
		var val = doc.$$schema ? doc.$clean() : doc;
		if (typeof(val) === 'function')
			val = val.toString();
		return send(this, 'modify', val, insert).builder = new DatabaseBuilder(this);
	};

	DP.restore = function(filename, callback) {
		var obj = send(this, 'restore', filename);
		obj.callback = callback;
		return this;
	};

	DP.backup = function(filename, callback) {
		var obj = send(this, 'backup', filename);
		obj.callback = callback;
		return this;
	};

	DP.refresh = function() {
		return this;
	};

	DP.drop = function() {
		notify(this, 'drop');
		return this;
	};

	TP.clear = DP.clear = function(callback) {
		send(this, 'clear').callback = callback;
		return this;
	};

	TP.clean = DP.clean = function(callback) {
		send(this, 'clean').callback = callback;
		return this;
	};

	TP.ready = DP.ready = function(callback) {
		callback && callback();
		return this;
	};

	TP.remove = DP.remove = function(filename) {
		return send(this, 'remove', filename).builder = new DatabaseBuilder(this);
	};

	TP.stream = DP.stream = function(fn, repository, callback) {

		if (typeof(repository) === 'function')  {
			callback = repository;
			repository = undefined;
		}

		send(this, 'stream', fn.toString(), repository).callback = callback;
		return this;
	};

	CP.min = function(id, count) {
		notify(this.db, 'counter.min', id, count);
		return this;
	};

	CP.max = function(id, count) {
		notify(this.db, 'counter.max', id, count);
		return this;
	};

	CP.sum = CP.inc = CP.hit = function(id, count) {
		notify(this.db, 'counter.hit', id, count);
		return this;
	};

	CP.remove = function(id) {
		notify(this.db, 'counter.remove', id);
		return this;
	};

	CP.read = function(options, callback) {
		send(this.db, 'counter.read', options).callback = callback;
		return this;
	};

	CP.stats = CP.stats_sum = function(top, year, month, day, type, callback) {

		if (typeof(day) == 'function') {
			callback = day;
			day = null;
		} else if (typeof(month) == 'function') {
			callback = month;
			month = null;
		} else if (typeof(year) === 'function') {
			callback = year;
			year = month = null;
		}

		send(this.db, 'counter.stats', top, year, month, day, type).callback = callback;
		return this;
	};

	CP.clear = function(callback) {
		send(this.db, 'counter.clear').callback = callback;
		return this;
	};

	SP.insert = function(doc) {
		notify(this.db, 'storage.insert', doc.$$schema ? doc.$clean() : doc);
		return this;
	};

	SP.scan = function(beg, end, mapreduce, callback) {

		if (typeof(beg) === 'function') {
			mapreduce = beg;
			callback = end;
			beg = null;
			end = null;
		} else if (typeof(end) === 'function') {
			callback = mapreduce;
			mapreduce = end;
			end = null;
		}

		send(this.db, 'storage.scan', beg, end, mapreduce.toString()).callback = callback;
		return this;
	};

	SP.mapreduce = function(name, fn) {
		send(this.db, 'storage.mapreduce', name, fn);
		return this;
	};

	SP.stats = function(name, callback) {

		if (typeof(name) === 'function') {
			callback = name;
			name = undefined;
		}

		send(this.db, 'storage.stats', name).callback = callback;
		return this;
	};

	SP.clear = function(beg, end, callback) {

		if (typeof(beg) === 'function') {
			callback = end;
			beg = null;
			end = null;
		} else if (typeof(end) === 'function') {
			callback = end;
			end = null;
		}

		send(this.db, 'storage.clear', beg, end).callback = callback;
		return this;
	};
};

function Table(name, filename, readonly, specific) {
	var t = this;
	t.filename = readonly ? filename : filename + (specific ? '' : EXTENSION_TABLE);
	t.filenameBackup = readonly ? '' : filename + EXTENSION_TABLE_BACKUP;
	t.filenameCounter = readonly ? '' : filename + (specific ? '' : EXTENSION_TABLE) + EXTENSION_COUNTER;
	t.filenameMeta = readonly ? '' : filename + (specific ? '' : EXTENSION_TABLE) + '-meta';
	t.directory = Path.dirname(filename);
	t.filenameLock = t.filename + '-lock';
	t.name = name;
	t.$name = '$' + name;
	t.pending_reader = [];
	t.pending_reader2 = [];
	t.pending_update = [];
	t.pending_append = [];
	t.pending_reader = [];
	t.pending_remove = [];
	t.pending_streamer = [];
	t.pending_clean = [];
	t.pending_clear = [];
	t.pending_locks = [];
	t.$events = {};

	t.step = 0;
	t.ready = false;
	t.$free = true;
	t.$writting = false;
	t.$reading = false;
	t.$allocations = true;

	t.counter = readonly ? null : new Counter(t);
	t.$meta();

	var schema = CONF['table_' + name] || CONF['table.' + name];

	Fs.createReadStream(t.filename, { end: 1200 }).once('data', function(chunk) {

		if (schema) {
			t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|'));
			schema = t.stringifySchema();
		}

		t.parseSchema(chunk.toString('utf8').split('\n', 1)[0].split('|'));
		t.ready = true;

		if (schema && t.stringifySchema() !== schema) {
			t.$header = Buffer.byteLength(t.stringifySchema()) + 1;
			t.extend(schema);
		} else
			t.$header = Buffer.byteLength(schema ? schema : t.stringifySchema()) + 1;

		t.next(0);

	}).on('error', function(e) {
		if (schema) {
			t.parseSchema(schema.replace(/;|,/g, '|').trim().split('|'));
			var bschema = t.stringifySchema();
			t.$header = Buffer.byteLength(bschema) + 1;
			Fs.writeFileSync(t.filename, bschema + NEWLINE, 'utf8');
			t.ready = true;
			t.next(0);
		} else {
			t.readonly = true;
			t.pending_reader.length && (t.pending_reader = []);
			t.pending_update.length && (t.pending_update = []);
			t.pending_append.length && (t.pending_append = []);
			t.pending_reader.length && (t.pending_reader = []);
			t.pending_remove.length && (t.pending_remove = []);
			t.pending_streamer.length && (t.pending_streamer = []);
			t.pending_locks.length && (t.pending_locks = []);
			t.pending_clean.length && (t.pending_clean = []);
			t.pending_clear.length && (t.pending_clear = []);
			t.throwReadonly(e);
		}
	});
}

function Database(name, filename, readonly, specific) {

	var self = this;
	var http = filename.substring(0, 6);
	self.readonly = http === 'http:/' || http === 'https:';
	self.filename = self.readonly ? filename.format('') : readonly ? filename : filename + (specific ? '' : EXTENSION);
	self.directory = Path.dirname(filename);

	if (!readonly) {
		self.filenameLock = self.filename + '-lock';
		self.filenameCounter = self.readonly ? filename.format('counter', '-') : filename + (specific ? '' : EXTENSION) + EXTENSION_COUNTER;
		self.filenameLog = self.readonly || readonly ? '' : filename + EXTENSION_LOG;
		self.filenameBackup = self.readonly || readonly ? '' : filename + EXTENSION_BACKUP;
		self.filenameStorage = self.readonly || readonly ? '' : filename + '-storage/{0}' + (specific ? '' : EXTENSION);
		self.filenameMeta = filename + EXTENSION_META;
		self.filenameBackup2 = framework_utils.join(self.directory, name + '_backup' + (specific ? '' : EXTENSION));
		self.inmemory = {};
		self.inmemorylastusage;
		// self.metadata;
		self.$meta();
	}

	self.name = name;
	self.pending_update = [];
	self.pending_append = [];
	self.pending_reader = [];
	self.pending_remove = [];
	self.pending_reader2 = [];
	self.pending_streamer = [];
	self.pending_clean = [];
	self.pending_clear = [];
	self.pending_locks = [];
	self.step = 0;
	self.pending_drops = false;
	self.pending_reindex = false;
	self.binary = self.readonly || readonly ? null : new Binary(self, self.directory + '/' + self.name + '-binary/');
	self.storage = self.readonly || readonly ? null : new Storage(self, self.directory + '/' + self.name + '-storage/');
	self.counter = readonly ? null : new Counter(self);
	self.$timeoutmeta;
	self.$events = {};
	self.$free = true;
	self.$writting = false;
	self.$reading = false;
}

const TP = Table.prototype;
const DP = Database.prototype;

TP.memory = DP.memory = function(count, size) {
	var self = this;
	count && (self.buffercount = count + 1);      // def: 15 - count of stored documents in memory while reading/writing
	size && (self.buffersize = size * 1024);      // def: 32 - size of buffer in kB
	return self;
};

TP.view = DP.view = function() {
	throw new Error('NoSQL Views are not supported in this version.');
};

TP.emit = DP.emit = function(name, a, b, c, d, e, f, g) {
	var evt = this.$events[name];
	if (evt) {
		var clean = false;
		for (var i = 0, length = evt.length; i < length; i++) {
			if (evt[i].$once)
				clean = true;
			evt[i].call(this, a, b, c, d, e, f, g);
		}
		if (clean) {
			evt = evt.remove(n => n.$once);
			if (evt.length)
				this.$events[name] = evt;
			else
				this.$events[name] = undefined;
		}
	}
	return this;
};

TP.on = DP.on = function(name, fn) {

	if (!fn.$once)
		this.$free = false;

	if (this.$events[name])
		this.$events[name].push(fn);
	else
		this.$events[name] = [fn];
	return this;
};

TP.once = DP.once = function(name, fn) {
	fn.$once = true;
	return this.on(name, fn);
};

TP.removeListener = DP.removeListener = function(name, fn) {
	var evt = this.$events[name];
	if (evt) {
		evt = evt.remove(n => n === fn);
		if (evt.length)
			this.$events[name] = evt;
		else
			this.$events[name] = undefined;
	}
	return this;
};

TP.removeAllListeners = DP.removeAllListeners = function(name) {
	if (name === true)
		this.$events = EMPTYOBJECT;
	else if (name)
		this.$events[name] = undefined;
	else
		this.$events[name] = {};
	return this;
};

exports.Database = Database;
exports.DatabaseBuilder = DatabaseBuilder;
exports.DatabaseBuilder2 = DatabaseBuilder2;
exports.DatabaseCounter = Counter;
exports.DatabaseBinary = Binary;
exports.DatabaseStorage = Storage;
exports.DatabaseTable = Table;

exports.load = function(name, filename, specific) {
	return new Database(name, filename, undefined, specific);
};

exports.table = function(name, filename, specific) {
	return new Table(name, filename, undefined, specific);
};

exports.memory = exports.inmemory = function(name) {
	return INMEMORY[name] = true;
};

TP.get = DP.get = function(name) {
	return this.meta(name);
};

TP.set = DP.set = function(name, value) {
	return this.meta(name, value);
};

TP.meta = DP.meta = function(name, value, nosave) {
	var self = this;

	if (value === undefined)
		return self.metadata ? self.metadata[name] : undefined;

	if (!self.metadata)
		self.metadata = {};

	self.metadata[name] = value;
	clearTimeout(self.timeoutmeta);

	if (!nosave)
		self.timeoutmeta = setTimeout(() => self.$meta(true), 500);

	if (F.isCluster && !nosave) {
		CLUSTERMETA.ID = F.id;
		CLUSTERMETA.TYPE = (self instanceof Table ? 'table' : 'nosql') + '-meta';
		CLUSTERMETA.name = self.name;
		CLUSTERMETA.key = name;
		CLUSTERMETA.value = value;
		process.send(CLUSTERMETA);
	}

	return self;
};

TP.backups = DP.backups = function(filter, callback) {

	if (callback === undefined) {
		callback = filter;
		filter = null;
	}

	var self = this;
	var isTable = self instanceof Table;

	if (isTable && !self.ready) {
		setTimeout((self, filter, callback) => self.backups(filter, callback), 500, self, filter, callback);
		return self;
	}

	var stream = Fs.createReadStream(self.filenameBackup);
	var output = [];
	var tmp = {};

	tmp.keys = self.$keys;

	stream.on('data', U.streamer(NEWLINEBUF, function(item, index) {
		var end = item.indexOf('|', item.indexOf('|') + 2);
		var meta = item.substring(0, end);
		var arr = meta.split('|');
		var dv = arr[0].trim().replace(' ', 'T') + ':00.000Z';
		tmp.line = item.substring(end + 1).trim();
		if (isTable)
			tmp.line = tmp.line.split('|');
		var obj = { id: index + 1, date: dv.parseDate(), user: arr[1].trim(), data: self instanceof Table ? self.parseData(tmp) : tmp.line.parseJSON(true) };
		if (!filter || filter(obj))
			output.push(obj);
	}), stream);

	CLEANUP(stream, () => callback(null, output));

	return self;
};

function next_operation(self, type) {
	self.next(type);
}

DP.ready = function(fn) {
	var self = this;
	fn.call(self);
	return self;
};

DP.insert = function(doc, unique) {

	var self = this;
	var builder;

	self.readonly && self.throwReadonly();

	if (unique) {
		builder = self.one();
		var callback;

		builder.callback(function(err, d) {
			if (d)
				callback && callback(null, 0);
			else
				self.insert(doc).callback(callback);
		});

		builder.callback = function(fn) {
			callback = fn;
			return builder;
		};

		return builder;
	}

	builder = new DatabaseBuilder2(self);
	var json = doc.$$schema ? doc.$clean() : doc;
	self.pending_append.push({ doc: JSON.stringify(json).replace(REGBOOL, JSONBOOL), raw: doc, builder: builder });
	setImmediate(next_operation, self, 1);
	self.$events.insert && self.emit('insert', json);
	return builder;
};

DP.upsert = function(doc) {
	return this.insert(doc, true);
};

DP.update = function(doc, insert) {
	var self = this;
	self.readonly && self.throwReadonly();
	var builder = new DatabaseBuilder(self);
	var data = doc.$$schema ? doc.$clean() : doc;
	builder.$options.readertype = 1;
	if (typeof(data) === 'string')
		data = new Function('doc', 'repository', 'arg', data.indexOf('return ') === -1 ? ('return (' + data + ')') : data);
	self.pending_update.push({ builder: builder, doc: data, insert: insert === true ? data : insert });
	setImmediate(next_operation, self, 2);
	return builder;
};

DP.modify = function(doc, insert) {
	var self = this;
	self.readonly && self.throwReadonly();
	var builder = new DatabaseBuilder(self);
	var data = doc.$$schema ? doc.$clean() : doc;
	var keys = Object.keys(data);
	var inc = null;

	builder.$options.readertype = 1;

	if (keys.length) {
		var tmp;
		for (var i = 0; i < keys.length; i++) {
			var key = keys[i];
			switch (key[0]) {
				case '!':
				case '+':
				case '-':
				case '*':
				case '/':
					!inc && (inc = {});
					tmp = key.substring(1);
					inc[tmp] = key[0];
					doc[tmp] = doc[key];
					doc[key] = undefined;
					keys[i] = tmp;
					break;
				case '$':
					tmp = key.substring(1);
					doc[tmp] = new Function('val', 'doc', 'repository', 'arg', doc[key].indexOf('return ') === -1 ? ('return (' + doc[key] + ')') : doc[key]);
					doc[key] = undefined;
					keys[i] = tmp;
					break;
			}
		}
		self.pending_update.push({ builder: builder, doc: data, keys: keys, inc: inc, insert: insert === true ? data : insert });
		setImmediate(next_operation, self, 2);
	}

	return builder;
};

DP.restore = function(filename, callback) {
	var self = this;
	self.readonly && self.throwReadonly();
	U.wait(() => !self.type, function(err) {

		if (err)
			throw new Error('Database can\'t be restored because it\'s busy.');

		self.type = 9;

		F.restore(filename, F.path.root(), function(err, response) {
			self.type = 0;
			if (!err) {
				self.$meta();
				self.binary.$refresh();
				self.refresh();
				self.storage && self.storage.refresh();
			}
			self.$events.change && self.emit('change', 'restore');
			callback && callback(err, response);
		});

	});
	return self;
};

DP.backup = function(filename, callback) {

	var self = this;
	self.readonly && self.throwReadonly();

	var list = [];
	var pending = [];

	pending.push(function(next) {
		F.path.exists(self.filename, function(e) {
			e && list.push(Path.join(CONF.directory_databases, self.name + EXTENSION));
			next();
		});
	});

	pending.push(function(next) {
		F.path.exists(F.path.databases(self.name + EXTENSION_META), function(e) {
			e && list.push(Path.join(CONF.directory_databases, self.name + EXTENSION_META));
			next();
		});
	});

	pending.push(function(next) {
		F.path.exists(self.filenameBackup, function(e) {
			e && list.push(Path.join(CONF.directory_databases, self.name + EXTENSION_BACKUP));
			next();
		});
	});

	pending.push(function(next) {
		F.path.exists(self.filenameCounter, function(e) {
			e && list.push(Path.join(CONF.directory_databases, self.name + EXTENSION + EXTENSION_COUNTER));
			next();
		});
	});

	pending.push(function(next) {
		F.path.exists(self.filenameLog, function(e) {
			e && list.push(Path.join(CONF.directory_databases, self.name + EXTENSION_LOG));
			next();
		});
	});

	pending.push(function(next) {
		F.path.exists(F.path.databases(self.name + '-binary'), function(e, size, file) {
			e && !file && list.push(Path.join(CONF.directory_databases, self.name + '-binary'));
			next();
		});
	});

	pending.push(function(next) {
		F.path.exists(F.path.databases(self.name + '-storage'), function(e, size, file) {
			e && !file && list.push(Path.join(CONF.directory_databases, self.name + '-storage'));
			next();
		});
	});

	pending.push(function(next) {
		var filename = Path.join(CONF.directory_databases, self.name + EXTENSION_MAPREDUCE);
		F.path.exists(F.path.root(filename), function(e) {
			e && list.push(filename);
			next();
		});
	});

	pending.async(function() {
		if (list.length)
			F.backup(filename, list, callback);
		else
			callback(new Error('No files for backuping.'));
	});

	return self;
};

DP.backup2 = function(filename, remove) {

	if (typeof(filename) === 'boolean') {
		remove = filename;
		filename = '';
	}

	var self = this;
	self.readonly && self.throwReadonly();

	if (remove)
		return self.remove(filename || '');

	var builder = new DatabaseBuilder2(self);
	var stream = Fs.createReadStream(self.filename);

	stream.pipe(Fs.createWriteStream(filename || self.filenameBackup2));

	stream.on('error', function(err) {
		builder.$options.log && builder.log();
		builder.$callback && builder.$callback(errorhandling(err, builder));
		builder.$callback = null;
	});

	stream.on('end', function() {
		builder.$options.log && builder.log();
		builder.$callback && builder.$callback(errorhandling(null, builder, true), true);
		builder.$callback = null;
	});

	return builder;
};

DP.drop = function() {
	var self = this;
	self.readonly && self.throwReadonly();
	self.pending_drops = true;
	setImmediate(next_operation, self, 7);
	return self;
};

DP.free = function(force) {
	var self = this;
	if (!force && !self.$free)
		return self;
	self.counter.removeAllListeners(true);
	self.binary.removeAllListeners(true);
	self.removeAllListeners(true);
	self.binary = null;
	self.counter = null;
	delete F.databases[self.name];
	return self;
};

DP.release = function() {
	var self = this;
	self.inmemory = {};
	self.inmemorylastusage = undefined;
	return self;
};

TP.clear = DP.clear = function(callback) {
	var self = this;
	self.readonly && self.throwReadonly();
	self.pending_clear.push(callback || NOOP);
	setImmediate(next_operation, self, 12);
	return self;
};

TP.clean = DP.clean = function(callback) {
	var self = this;
	self.readonly && self.throwReadonly();
	self.pending_clean.push(callback || NOOP);
	setImmediate(next_operation, self, 13);
	return self;
};

TP.lock = DP.lock = function(callback) {
	var self = this;
	self.readonly && self.throwReadonly();
	self.pending_locks.push(callback || NOOP);
	setImmediate(next_operation, self, 14);
	return self;
};

DP.remove = function() {
	var self = this;
	self.readonly && self.throwReadonly();
	var builder = new DatabaseBuilder(self);
	self.pending_remove.push(builder);
	builder.$options.readertype = 1;
	setImmediate(next_operation, self, 3);
	return builder;
};

DP.listing = DP.list = function(builder) {
	var self = this;
	if (builder)
		builder.db = self;
	else
		builder = new DatabaseBuilder(self);
	builder.$options.listing = true;
	builder.$take = builder.$options.take = 100;
	self.pending_reader.push(builder);
	setImmediate(next_operation, self, 4);
	return builder;
};

DP.find = function(builder) {
	var self = this;
	if (builder instanceof DatabaseBuilder)
		builder.db = self;
	else
		builder = new DatabaseBuilder(self);
	self.pending_reader.push(builder);
	setImmediate(next_operation, self, 4);
	return builder;
};

DP.find2 = function(builder) {
	var self = this;
	if (builder instanceof DatabaseBuilder)
		builder.db = self;
	else {
		builder = new DatabaseBuilder(self);
		builder.$options.notall = true;
	}

	if (self.readonly)
		return self.find(builder);
	self.pending_reader2.push(builder);
	setImmediate(next_operation, self, 11);
	return builder;
};

DP.stream = function(fn, repository, callback) {
	var self = this;

	if (typeof(repository) === 'function') {
		callback = repository;
		repository = null;
	}

	self.pending_streamer.push({ fn: fn, callback: callback, repository: repository || {} });
	setImmediate(next_operation, self, 10);
	return self;
};

DP.throwReadonly = function(e) {
	throw new Error('Database "{0}" is readonly.'.format(this.name) + (e ? '\n' + e.toString() : ''));
};

DP.scalar = function(type, field) {
	return this.find().scalar(type, field);
};

DP.count = function() {
	var self = this;
	var builder = new DatabaseBuilder(self);
	builder.$options.readertype = 1;
	self.pending_reader.push(builder);
	setImmediate(next_operation, self, 4);
	return builder;
};

DP.one = DP.read = function() {
	var self = this;
	var builder = new DatabaseBuilder(self);
	builder.first();
	self.pending_reader.push(builder);
	setImmediate(next_operation, self, 4);
	return builder;
};

DP.one2 = DP.read2 = function() {
	var self = this;
	var builder = new DatabaseBuilder(self);
	builder.first();
	self.pending_reader2.push(builder);
	setImmediate(next_operation, self, 11);
	return builder;
};

DP.top = function(max) {
	var self = this;
	var builder = new DatabaseBuilder(self);
	builder.take(max);
	self.pending_reader.push(builder);
	setImmediate(next_operation, self, 4);
	return builder;
};

//  1 append
//  2 update
//  3 remove
//  4 reader
//  5 views
//  6 reader views
//  7 drop
//  8 backup
//  9 restore
// 10 streamer
// 11 reader reverse
// 12 clear
// 13 clean
// 14 locks

const NEXTWAIT = { 7: true, 8: true, 9: true, 12: true, 13: true, 14: true };

DP.next = function(type) {

	if (type && NEXTWAIT[this.step])
		return;

	if (F.isCluster && type === 0 && this.locked)
		clusterunlock(this);

	if (!this.$writting && !this.$reading) {

		if (this.step !== 12 && this.pending_clear.length) {
			if (!this.readonly && F.isCluster)
				clusterlock(this, '$clear');
			else if (INMEMORY[this.name])
				this.$clear_inmemory();
			else
				this.$clear();
			return;
		}

		if (this.step !== 13 && this.pending_clean.length) {
			if (!this.readonly && F.isCluster)
				clusterlock(this, '$clean');
			else
				this.$clean();
			return;
		}

		if (this.step !== 7 && !this.pending_reindex && this.pending_drops) {
			this.$drop();
			return;
		}

		if (this.step !== 14 && this.pending_locks.length) {
			this.$lock();
			return;
		}
	}

	if (!this.$writting) {

		if (this.step !== 1 && !this.pending_reindex && this.pending_append.length) {
			if (INMEMORY[this.name])
				this.$append_inmemory();
			else
				this.$append();
			return;
		}

		if (this.step !== 2 && !this.$writting && this.pending_update.length) {
			if (!this.readonly && F.isCluster)
				clusterlock(this, '$update');
			else if (INMEMORY[this.name])
				this.$update_inmemory();
			else
				this.$update();
			return;
		}

		if (this.step !== 3 && !this.$writting && this.pending_remove.length) {
			if (!this.readonly && F.isCluster)
				clusterlock(this, '$remove');
			if (INMEMORY[this.name])
				this.$remove_inmemory();
			else
				this.$remove();
			return;
		}

	}

	if (!this.$reading) {

		if (this.step !== 4 && this.pending_reader.length) {
			this.$reader();
			return;
		}

		if (this.step !== 11 && this.pending_reader2.length) {
			this.$reader3();
			return;
		}

		if (this.step !== 10 && this.pending_streamer.length) {
			this.$streamer();
			return;
		}
	}

	if (this.step !== type) {
		this.step = 0;
		setImmediate(next_operation, this, 0);
	}
};

// ======================================================================
// FILE OPERATIONS
// ======================================================================

// InMemory saving
DP.$save = function() {
	var self = this;
	setTimeout2('nosql.' + self.name, function() {
		var data = self.inmemory['#'] || EMPTYARRAY;
		var builder = [];
		for (var i = 0, length = data.length; i < length; i++)
			builder.push(JSON.stringify(data[i]).replace(REGBOOL, JSONBOOL));
		Fs.writeFile(self.filename, builder.join(NEWLINE) + NEWLINE, F.errorcallback);
	}, 50, 100);
	return self;
};

DP.$inmemory = function(callback) {

	var self = this;
	var view = '#';

	self.readonly && self.throwReadonly();

	// Last usage
	self.inmemorylastusage = global.F ? global.NOW : undefined;

	if (self.inmemory[view])
		return callback();

	var filename = self.filename;
	if (view !== '#')
		filename = filename.replace(/\.nosql/, '#' + view + '.nosql');

	self.inmemory[view] = [];

	Fs.readFile(filename, function(err, data) {
		if (err)
			return callback();
		var arr = data.toString('utf8').split('\n');
		for (var i = 0, length = arr.length; i < length; i++) {
			var item = arr[i];
			if (item) {
				try {
					item = JSON.parse(item.trim(), jsonparser);
					item && self.inmemory[view].push(item);
				} catch (e) {}
			}
		}

		callback();
	});

	return self;
};

TP.$meta = DP.$meta = function(write) {

	var self = this;

	if (write) {
		self.readonly && self.throwReadonly();
		Fs.writeFile(self.filenameMeta, JSON.stringify(self.metadata), F.errorcallback);
		return self;
	}

	if (self.readonly)
		return self;

	try {
		self.metadata = JSON.parse(Fs.readFileSync(self.filenameMeta).toString('utf8'), jsonparser);
	} catch (err) {}

	return self;
};

DP.$append = function() {
	var self = this;
	self.step = 1;

	if (!self.pending_append.length) {
		self.next(0);
		return;
	}

	self.$writting = true;

	self.pending_append.splice(0).limit(JSONBUFFER, function(items, next) {

		var json = '';
		for (var i = 0, length = items.length; i < length; i++) {
			json += items[i].doc + NEWLINE;
		}

		Fs.appendFile(self.filename, json, function(err) {

			err && F.error(err, 'NoSQL insert: ' + self.name);

			for (var i = 0, length = items.length; i < length; i++) {
				items[i].builder.$options.log && items[i].builder.log();
				var callback = items[i].builder.$callback;
				callback && callback(err, 1);
			}

			next();
		});

	}, () => setImmediate(next_append, self));
};

function next_append(self) {
	self.$writting = false;
	self.next(0);
	self.$events.change && self.emit('change', 'insert');
}

DP.$append_inmemory = function() {
	var self = this;
	self.step = 1;

	if (!self.pending_append.length) {
		self.next(0);
		return self;
	}

	var items = self.pending_append.splice(0);

	return self.$inmemory(function() {

		for (var i = 0, length = items.length; i < length; i++) {
			self.inmemory['#'].push(JSON.parse(items[i].doc, jsonparser));
			items[i].builder.$options.log && items[i].builder.log();
			var callback = items[i].builder.$callback;
			callback && callback(null, 1);
		}

		self.$save();
		setImmediate(next_append, self);
	});
};

DP.$update = function() {

	var self = this;
	self.step = 2;

	if (!self.pending_update.length) {
		self.next(0);
		return self;
	}

	self.$writting = true;

	var filter = self.pending_update.splice(0);
	var filters = new NoSQLReader();
	var fs = new NoSQLStream(self.filename);
	var change = false;

	for (var i = 0; i < filter.length; i++)
		filters.add(filter[i].builder, true);

	if (self.buffersize)
		fs.buffersize = self.buffersize;

	if (self.buffercount)
		fs.buffercount = self.buffercount;

	var update = function(docs, doc, dindex, f, findex) {

		var rec = fs.docsbuffer[dindex];
		var fil = filter[findex];
		var e = fil.keys ? 'modify' : 'update';
		var old = self.$events[e] ? CLONE(doc) : 0;

		if (f.first)
			f.canceled = true;

		if (fil.keys) {
			for (var j = 0; j < fil.keys.length; j++) {
				var key = fil.keys[j];
				var val = fil.doc[key];
				if (val !== undefined) {
					if (typeof(val) === 'function')
						doc[key] = val(doc[key], doc, f.filter.repository, f.filter.arg);
					else if (fil.inc && fil.inc[key]) {
						switch (fil.inc[key]) {
							case '!':
								doc[key] = !doc[key];
								break;
							case '+':
								doc[key] = (doc[key] || 0) + val;
								break;
							case '-':
								doc[key] = (doc[key] || 0) - val;
								break;
							case '*':
								doc[key] = (doc[key] || 0) + val;
								break;
							case '/':
								doc[key] = (doc[key] || 0) / val;
								break;
						}
					} else
						doc[key] = val;
				}
			}
		} else
			docs[dindex] = typeof(fil.doc) === 'function' ? (fil.doc(doc, f.filter.repository, f.filter.arg) || doc) : fil.doc;

		self.$events[e] && self.emit(e, doc, old);
		f.builder.$options.backup && f.builder.$backupdoc(rec.doc);
	};

	var updateflush = function(docs, doc, dindex) {

		doc = docs[dindex];

		var rec = fs.docsbuffer[dindex];
		var upd = JSON.stringify(doc).replace(REGBOOL, JSONBOOL);
		if (upd === rec.doc)
			return;

		!change && (change = true);
		var was = true;

		if (rec.doc.length === upd.length) {
			var b = Buffer.byteLength(upd);
			if (rec.length === b) {
				fs.write(upd + NEWLINE, rec.position);
				was = false;
			}
		}

		if (was) {
			var tmp = fs.remchar + rec.doc.substring(1) + NEWLINE;
			fs.write(tmp, rec.position);
			fs.write2(upd + NEWLINE);
		}
	};

	fs.ondocuments = function() {
		filters.compare2(JSON.parse('[' + fs.docs + ']', jsonparser), update, updateflush);
	};

	fs.$callback = function() {

		fs = null;
		self.$writting = false;
		self.next(0);

		for (var i = 0; i < filters.builders.length; i++) {
			var item = filters.builders[i];
			var fil = filter[i];
			if (fil.insert && !item.counter) {
				item.builder.$insertcallback && item.builder.$insertcallback(fil.insert, item.filter ? item.filter.repository : EMPTYOBJECT);
				var tmp = self.insert(fil.insert);
				tmp.$callback = item.builder.$callback;
				tmp.$options.log = item.builder.$options.log;
				item.builder.$callback = null;
			} else {
				item.builder.$options.log && item.builder.log();
				item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter ? item.filter.repository : EMPTYOBJECT);
			}
		}

		if (change) {
			self.$events.change && self.emit('change', 'update');
			!F.databasescleaner[self.name] && (F.databasescleaner[self.name] = 1);
		}
	};

	fs.openupdate();
	return self;
};

DP.$update_inmemory = function() {

	var self = this;
	self.step = 2;

	if (!self.pending_update.length) {
		self.next(0);
		return self;
	}

	var filter = self.pending_update.splice(0);
	var change = false;
	var filters = new NoSQLReader();

	for (var i = 0; i < filter.length; i++)
		filters.add(filter[i].builder, true);

	return self.$inmemory(function() {

		var old;

		var update = function(docs, doc, dindex, f, findex) {

			var fil = filter[findex];
			var e = fil.keys ? 'modify' : 'update';

			if (!old)
				old = self.$events[e] ? CLONE(doc) : 0;

			if (f.first)
				f.canceled = true;

			if (fil.keys) {
				for (var j = 0; j < fil.keys.length; j++) {
					var key = fil.keys[j];
					var val = fil.doc[key];
					if (val !== undefined) {
						if (typeof(val) === 'function')
							doc[key] = val(doc[key], doc);
						else if (fil.inc && fil.inc[key]) {
							switch (fil.inc[key]) {
								case '!':
									doc[key] = !doc[key];
									break;
								case '+':
									doc[key] = (doc[key] || 0) + val;
									break;
								case '-':
									doc[key] = (doc[key] || 0) - val;
									break;
								case '*':
									doc[key] = (doc[key] || 0) + val;
									break;
								case '/':
									doc[key] = (doc[key] || 0) / val;
									break;
							}
						} else
							doc[key] = val;
					}
				}
			} else
				docs[dindex] = typeof(fil.doc) === 'function' ? fil.doc(doc, f.filter.repository) : fil.doc;

			self.$events[e] && self.emit(e, doc, old);
			f.builder.$options.backup && f.builder.$backupdoc(old);
			return 1;
		};

		var updateflush = function(docs, doc) {
			!change && (change = true);
			self.$events.update && self.emit('update', doc, old);
			old = null;
		};

		filters.compare2(self.inmemory['#'], update, updateflush);

		change && self.$save();

		for (var i = 0; i < filters.builders.length; i++) {
			var item = filters.builders[i];
			if (item.insert && !item.counter) {
				item.builder.$insertcallback && item.builder.$insertcallback(item.insert, item.filter.repository || EMPTYOBJECT);
				var tmp = self.insert(item.insert);
				tmp.$callback = item.builder.$callback;
				tmp.$options.log = item.builder.$options.log;
				item.builder.$callback = null;
			} else {
				item.builder.$options.log && item.builder.log();
				item.builder.$callback && item.builder.$callback(errorhandling(null, item.builder, item.counter), item.counter, item.count, item.filter.repository);
			}
		}

		setImmediate(function() {
			self.next(0);
			change && self.$events.change && self.emit('change', 'update');
		});
	});
};

DP.$reader = function() {

	var self = this;
	self.step = 4;

	if (!self.pending_reader.length) {
		self.next(0);
		return self;
	}

	var list = self.pending_reader.splice(0);
	if (INMEMORY[self.name]) {
		self.$reader2_inmemory(list, () => self.next(0));
	} else {
		self.$reading = true;
		self.$reader2(self.filename, list, function() {
			self.$reading = false;
			self.next(0);
		});
	}

	return self;
};

function listing(builder, item) {
	var skip = builder.$options.skip || 0;
	var take = builder.$options.take || 0;
	return { page: ((skip / take) + 1), pages: item.count ? Math.ceil(item.count / take) : 0, limit: take, count: item.count, items: item.response || [] };
}

DP.$reader2 = function(filename, items, callback, reader) {

	var self = this;

	if (self.readonly) {
		if (reader === undefined) {
			U.download(filename, FLAGS_READ, function(err, response) {
				err && F.error(err, 'NoSQL database download: ' + self.name);
				self.$reader2(filename, items, callback, err ? null : response);
			});
			return self;
		}
	}

	var fs = new NoSQLStream(self.filename);
	var filters = new NoSQLReader(items);

	if (self.buffersize)
		fs.buffersize = self.buffersize;

	if (self.buffercount)
		fs.buffercount = self.buffercount;

	fs.ondocuments = function() {
		return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser));
	};

	fs.$callback = function() {
		filters.done();
		fs = null;
		callback();
	};

	if (reader)
		fs.openstream(reader);
	else
		fs.openread();

	return self;
};

DP.$reader3 = function() {

	var self = this;
	self.step = 11;

	if (!self.pending_reader2.length) {
		self.next(0);
		return self;
	}

	self.$reading = true;

	var fs = new NoSQLStream(self.filename);
	var filters = new NoSQLReader(self.pending_reader2.splice(0));

	if (self.buffersize)
		fs.buffersize = self.buffersize;

	if (self.buffercount)
		fs.buffercount = self.buffercount;

	fs.ondocuments = function() {
		return filters.compare(JSON.parse('[' + fs.docs + ']', jsonparser));
	};

	fs.$callback = function() {
		filters.done();
		self.$reading = false;
		fs = null;
		self.next(0);
	};

	fs.openreadreverse();
	return self;
};

DP.$streamer = function() {

	var self = this;
	self.step = 10;

	if (!self.pending_streamer.length) {
		self.next(0);
		return self;
	}

	self.$reading = true;

	var filter = self.pending_streamer.splice(0);
	var length = filter.length;
	var count = 0;
	var fs = new NoSQLStream(self.filename);

	if (self.buffersize)
		fs.buffersize = self.buffersize;

	if (self.buffercount)
		fs.buffercount = self.buffercount;

	fs.ondocuments = function() {
		var docs = JSON.parse('[' + fs.docs + ']', jsonparser);
		for (var j = 0; j < docs.length; j++) {
			var json = docs[j];
			count++;
			for (var i = 0; i < length; i++)
				filter[i].fn(json, filter[i].repository, count);
		}
	};

	fs.$callback = function() {
		for (var i = 0; i < length; i++)
			filter[i].callback && filter[i].callback(null, filter[i].repository, count);
		self.$reading = false;
		self.next(0);
		fs = null;
	};

	fs.openread();
	return self;
};

function nosqlinlinesorter(item, builder, doc) {

	if (!item.response) {
		item.response = [doc];
		return;
	}

	var length = item.response.length;
	if (length < builder.$limit) {
		item.response.push(doc);
		length + 1 >= builder.$limit && item.response.quicksort(builder.$options.sort.name, builder.$options.sort.asc);
	} else
		nosqlresort(item.response, builder, doc);
}

function nosqlsortvalue(a, b, sorter) {
	var type = typeof(a);
	if (type === 'number')
		return sorter.asc ? a > b : a < b;
	else if (type === 'string') {
		var c = COMPARER(a, b);
		return sorter.asc ? c === 1 : c === -1;
	} else if (type === 'boolean')
		// return sorter.asc ? a > b : a < b;
		return sorter.asc ? (a && !b) : (!a && b);
	else if (a instanceof Date)
		return sorter.asc ? a > b : a < b;
	return false;
}

function nosqlresort(arr, builder, doc) {
	var b = doc[builder.$options.sort.name];
	var beg = 0;
	var length = arr.length;
	var tmp = length - 1;

	var sort = nosqlsortvalue(arr[tmp][builder.$options.sort.name], b, builder.$options.sort);
	if (!sort)
		return;

	tmp = arr.length / 2 >> 0;
	sort = nosqlsortvalue(arr[tmp][builder.$options.sort.name], b, builder.$options.sort);
	if (!sort)
		beg = tmp + 1;

	for (var i = beg; i < length; i++) {
		var item = arr[i];
		var sort = nosqlsortvalue(item[builder.$options.sort.name], b, builder.$options.sort);
		if (!sort)
			continue;
		for (var j = length - 1; j > i; j--)
			arr[j] = arr[j - 1];
		arr[i] = doc;
		return;
	}
}

DP.$reader2_inmemory = function(items, callback) {
	var self = this;
	return self.$inmemory(function() {
		var filters = new NoSQLReader(items);
		filters.clone = true;
		filters.compare(self.inmemory['#']);
		filters.done();
		callback();
	});
};

DP.$remove = function() {

	var self = this;
	self.step = 3;

	if (!self.pending_remove.length) {
		self.next(0);
		return;
	}

	self.$writting = true;

	var fs = new NoSQLStream(self.filename);
	var filter = self.pending_remove.splice(0);
	var filters = new NoSQLReader(filter);
	var change = false;

	if (self.buffersize)
		fs.buffersize = self.buffersize;

	if (self.buffercount)
		fs.buffercount = self.buffercount;

	var remove = function(docs, d, dindex, f) {
		var rec = fs.docsbuffer[dindex];
		f.builder.$options.backup && f.builder.$backupdoc(rec.doc);
		return 1;
	};

	var removeflush = function(docs, d, dindex) {
		var rec = fs.docsbuffer[dindex];
		!change && (change = true);
		self.$events.remove && self.emit('remove', d);
		fs.write(fs.remchar + rec.doc.substring(1) + NEWLINE, rec.position);
	};

	fs.ondocuments = function() {
		filters.compare2(JSON.parse('[' + fs.docs + ']', jsonparser), remove, removeflush);
	};

	fs.$callback = function() {
		filters.done();
		fs = null;
		self.$writting = false;
		self.next(0);
		if (change) {
			self.$events.change && self.emit('change', 'remove');
			!F.databasescleaner[self.name] && (F.databasescleaner[self.name] = 1);
		}
	};

	fs.openupdate();
};

DP.$clear = function() {

	var self = this;
	self.step = 12;

	if (!self.pending_clear.length) {
		self.next(0);
		return;
	}

	var filter = self.pending_clear.splice(0);
	Fs.unlink(self.filename, function() {
		for (var i = 0; i < filter.length; i++)
			filter[i]();
		self.$events.change && self.emit('change', 'clear');
		self.next(0);
	});
};

DP.$clean = function() {

	var self = this;
	self.step = 13;

	if (!self.pending_clean.length) {
		self.next(0);
		return;
	}

	var filter = self.pending_clean.splice(0);
	var length = filter.length;
	var now = Date.now();

	F.databasescleaner[self.name] = undefined;
	CONF.nosql_logger && PRINTLN('NoSQL embedded "{0}" cleaning (beg)'.format(self.name));

	var fs = new NoSQLStream(self.filename);