level-filesystem/index.js

Full implementation of the fs module on top of leveldb

var fwd = require('fwd-stream');
var sublevel = require('level-sublevel');
var blobs = require('level-blobs');
var peek = require('level-peek');
var once = require('once');
var octal = require('octal')
var errno = require('./errno');
var paths = require('./paths');
var watchers = require('./watchers');

var nextTick = function(cb, err, val) {
	process.nextTick(function() {
		cb(err, val);
	});
};

var noop = function() {};

module.exports = function(db, opts) {
	var fs = {};

	db = sublevel(db);

	var bl = blobs(db.sublevel('blobs'), opts);
	var ps = paths(db.sublevel('stats'));
	var links = db.sublevel('links');

	var listeners = watchers();
	var fds = [];

	var now = Date.now();
	var inc = function() {
		return ++now;
	};

	fs.mkdir = function(key, mode, cb) {
		if (typeof mode === 'function') return fs.mkdir(key, null, mode);
		if (!mode) mode = octal(777);
		if (!cb) cb = noop;

		ps.follow(key, function(err, stat, key) {
			if (err && err.code !== 'ENOENT') return cb(err);
			if (stat) return cb(errno.EEXIST(key));

			ps.put(key, {
				type:'directory',
				mode: mode,
				size: 4096
			}, listeners.cb(key, cb));
		});
	};

	fs.rmdir = function(key, cb) {
		if (!cb) cb = noop;
		ps.follow(key, function(err, stat, key) {
			if (err) return cb(err);
			fs.readdir(key, function(err, files) {
				if (err) return cb(err);
				if (files.length) return cb(errno.ENOTEMPTY(key));
				ps.del(key, listeners.cb(key, cb));
			});
		});

	};

	fs.readdir = function(key, cb) {
		ps.follow(key, function(err, stat, key) {
			if (err) return cb(err);
			if (!stat) return cb(errno.ENOENT(key));
			if (!stat.isDirectory()) return cb(errno.ENOTDIR(key));
			ps.list(key, cb);
		});
	};

	var stat = function(key, lookup, cb) {
		lookup(key, function(err, stat, key) {
			if (err) return cb(err);
			if (!stat.isFile()) return cb(null, stat);
			var blob = stat && stat.blob || key;
			bl.size(blob, function(err, size) {
				if (err) return cb(err);
				stat.size = size;
				cb(null, stat);
			});
		});
	};

	fs.stat = function(key, cb) {
		stat(key, ps.follow, cb);
	};

	fs.lstat = function(key, cb) {
		stat(key, ps.get, cb);
	};

	fs.exists = function(key, cb) {
		ps.follow(key, function(err) {
			cb(!err);
		});
	};

	var chmod = function(key, lookup, mode, cb) {
		if (!cb) cb = noop;
		lookup(key, function(err, stat, key) {
			if (err) return cb(err);
			ps.update(key, {mode:mode}, listeners.cb(key, cb));
		});
	};

	fs.chmod = function(key, mode, cb) {
		chmod(key, ps.follow, mode, cb);
	};

	fs.lchmod = function(key, mode, cb) {
		chmod(key, ps.get, mode, cb);
	};

	var chown = function(key, lookup, uid, gid, cb) {
		if (!cb) cb = noop;
		lookup(key, function(err, stat, key) {
			if (err) return cb(err);
			ps.update(key, {uid:uid, gid:gid}, listeners.cb(key, cb));
		});
	};

	fs.chown = function(key, uid, gid, cb) {
		chown(key, ps.follow, uid, gid, cb);
	};

	fs.lchown = function(key, uid, gid, cb) {
		chown(key, ps.get, uid, gid, cb);
	};

	fs.utimes = function(key, atime, mtime, cb) {
		if (!cb) cb = noop;
		ps.follow(key, function(err, stat, key) {
			if (err) return cb(err);
			var upd = {};
			if (atime) upd.atime = atime;
			if (mtime) upd.mtime = mtime;
			ps.update(key, upd, listeners.cb(key, cb));
		});
	};

	fs.rename = function(from, to, cb) {
		if (!cb) cb = noop;

		ps.follow(from, function(err, statFrom, from) {
			if (err) return cb(err);

			var rename = function() {
				cb = listeners.cb(to, listeners.cb(from, cb));
				ps.put(to, statFrom, function(err) {
					if (err) return cb(err);
					ps.del(from, cb);
				});
			};

			ps.follow(to, function(err, statTo, to) {
				if (err && err.code !== 'ENOENT') return cb(err);
				if (!statTo) return rename();
				if (statFrom.isDirectory() !== statTo.isDirectory()) return cb(errno.EISDIR(from));

				if (statTo.isDirectory()) {
					fs.readdir(to, function(err, list) {
						if (err) return cb(err);
						if (list.length) return cb(errno.ENOTEMPTY(from));
						rename();
					});
					return;
				}

				rename();
			});
		});
	};

	fs.realpath = function(key, cache, cb) {
		if (typeof cache === 'function') return fs.realpath(key, null, cache);
		ps.follow(key, function(err, stat, key) {
			if (err) return cb(err);
			cb(null, key);
		});
	};

	fs.writeFile = function(key, data, opts, cb) {
		if (typeof opts === 'function') return fs.writeFile(key, data, null, opts);
		if (typeof opts === 'string') opts = {encoding:opts};
		if (!opts) opts = {};
		if (!cb) cb = noop;

		if (!Buffer.isBuffer(data)) data = new Buffer(data, opts.encoding || 'utf-8');

		var flags = opts.flags || 'w';
		opts.append = flags[0] !== 'w';

		ps.follow(key, function(err, stat, key) {
			if (err && err.code !== 'ENOENT') return cb(err);
			if (stat && stat.isDirectory()) return cb(errno.EISDIR(key));
			if (stat && flags[1] === 'x') return cb(errno.EEXIST(key));

			var blob = stat && stat.blob || key;
			ps.writable(key, function(err) {
				if (err) return cb(err);

				bl.write(blob, data, opts, function(err) {
					if (err) return cb(err);

					ps.put(key, {
						ctime: stat && stat.ctime,
						mtime: new Date(),
						mode: opts.mode || octal(666),
						type:'file'
					}, listeners.cb(key, cb));
				});
			});
		});
	};

	fs.appendFile = function(key, data, opts, cb) {
		if (typeof opts === 'function') return fs.appendFile(key, data, null, opts);
		if (typeof opts === 'string') opts = {encoding:opts};
		if (!opts) opts = {};

		opts.flags = 'a';
		fs.writeFile(key, data, opts, cb);
	};

	fs.unlink = function(key, cb) {
		if (!cb) cb = noop;

		ps.get(key, function(err, stat, key) {
			if (err) return cb(err);
			if (stat.isDirectory()) return cb(errno.EISDIR(key));

			var clean = function(target) {
				peek(links, {start:target+'\xff', end:target+'\xff\xff'}, function(err) {
					if (err) return bl.remove(target, cb); // no more links
					cb();
				});
			};

			var onlink = function() {
				var target = stat.link.slice(0, stat.link.indexOf('\xff'));
				links.del(stat.link, function(err) {
					if (err) return cb(err);
					clean(target);
				});
			};

			ps.del(key, listeners.cb(key, function(err) {
				if (err) return cb(err);
				if (stat.link) return onlink();
				links.del(key+'\xff', function(err) {
					if (err) return cb(err);
					clean(key);
				});
			}));
		});
	};

	fs.readFile = function(key, opts, cb) {
		if (typeof opts === 'function') return fs.readFile(key, null, opts);
		if (typeof opts === 'string') opts = {encoding:opts};
		if (!opts) opts = {};

		var encoding = opts.encoding || 'binary';
		var flag = opts.flag || 'r';

		ps.follow(key, function(err, stat, key) {
			if (err) return cb(err);
			if (stat.isDirectory()) return cb(errno.EISDIR(key));

			var blob = stat && stat.blob || key;
			bl.read(blob, function(err, data) {
				if (err) return cb(err);
				cb(null, opts.encoding ? data.toString(opts.encoding) : data);
			});
		});
	};

	fs.createReadStream = function(key, opts) {
		if (!opts) opts = {};

		var closed = false;
		var rs = fwd.readable(function(cb) {
			ps.follow(key, function(err, stat, key) {
				if (err) return cb(err);
				if (stat.isDirectory()) return cb(errno.EISDIR(key));

				var blob = stat && stat.blob || key;
				var r = bl.createReadStream(blob, opts);

				rs.emit('open');
				r.on('end', function() {
					process.nextTick(function() {
						if (!closed) rs.emit('close');
					});
				});

				cb(null, r);
			});
		});

		rs.on('close', function() {
			closed = true;
		});

		return rs;
	};

	fs.createWriteStream = function(key, opts) {
		if (!opts) opts = {};

		var flags = opts.flags || 'w';
		var closed = false;
		var mode = opts.mode || octal(666);

		opts.append = flags[0] === 'a';

		var ws = fwd.writable(function(cb) {
			ps.follow(key, function(err, stat, key) {
				if (err && err.code !== 'ENOENT') return cb(err);
				if (stat && stat.isDirectory()) return cb(errno.EISDIR(key));
				if (stat && flags[1] === 'x') return cb(errno.EEXIST(key));

				var blob = stat && stat.blob || key;
				ps.writable(blob, function(err) {
					if (err) return cb(err);

					var ctime = stat ? stat.ctime : new Date();
					var s = {
						ctime: ctime,
						mtime: new Date(),
						mode: mode,
						type:'file'
					};

					ps.put(key, s, function(err) {
						if (err) return cb(err);

						var w = bl.createWriteStream(blob, opts);

						ws.emit('open');
						w.on('finish', function() {
							s.mtime = new Date();
							ps.put(key, s, function() {
								listeners.change(key);
								if (!closed) ws.emit('close');
							});
						});

						cb(null, w);
					});
				});
			});
		});

		ws.on('close', function() {
			closed = true;
		});

		return ws;
	};

	fs.truncate = function(key, len, cb) {
		ps.follow(key, function(err, stat, key) {
			if (err) return cb(err);

			var blob = stat && stat.blob || key;
			bl.size(blob, function(err, size) {
				if (err) return cb(err);

				ps.writable(key, function(err) {
					if (err) return cb(err);

					cb = once(listeners.cb(key, cb));
					if (!len) return bl.remove(blob, cb);

					var ws = bl.createWriteStream(blob, {
						start:size < len ? len-1 : len
					});

					ws.on('error', cb);
					ws.on('finish', cb);

					if (size < len) ws.write(new Buffer([0]));
					ws.end();
				});
			});
		});
	};

	fs.watchFile = function(key, opts, cb) {
		if (typeof opts === 'function') return fs.watchFile(key, null, opts);
		return listeners.watch(ps.normalize(key), cb);
	};

	fs.unwatchFile = function(key, cb) {
		listeners.unwatch(ps.normalize(key), cb);
	};

	fs.watch = function(key, opts, cb) {
		if (typeof opts === 'function') return fs.watch(key, null, opts)
		return listeners.watcher(ps.normalize(key), cb);
	};

	fs.notify = function(cb) {
		listeners.on('change', cb)
	}

	fs.open = function(key, flags, mode, cb) {
		if (typeof mode === 'function') return fs.open(key, flags, null, mode);

		ps.follow(key, function(err, stat, key) {
			if (err && err.code !== 'ENOENT') return cb(err);

			var fl = flags[0];
			var plus = flags[1] === '+' || flags[2] === '+';
			var blob = stat && stat.blob || key;

			var f = {
				key: key,
				blob: blob,
				mode: mode || octal(666),
				readable: fl === 'r' || ((fl === 'w' || fl === 'a') && plus),
				writable: fl === 'w' || fl === 'a' || (fl === 'r' && plus),
				append: fl === 'a'
			};

			if (fl === 'r' && err) return cb(err);
			if (flags[1] === 'x' && stat) return cb(errno.EEXIST(key));
			if (stat && stat.isDirectory()) return cb(errno.EISDIR(key));

			bl.size(blob, function(err, size) {
				if (err) return cb(err);

				if (f.append) f.writePos = size;

				ps.writable(key, function(err) {
					if (err) return cb(err);

					var onready = function(err) {
						if (err) return cb(err);

						var i = fds.indexOf(null);
						if (i === -1) i = 10+fds.push(fds.length+10)-1;

						f.fd = i;
						fds[i] = f;
						listeners.change(key);

						cb(null, f.fd);
					};

					var ontruncate = function(err) {
						if (err) return cb(err);
						if (stat) return onready();
						ps.put(blob, {ctime:stat && stat.ctime, type:'file'}, onready);
					};

					if (!f.append && f.writable) return bl.remove(blob, ontruncate);
					ontruncate();
				});
			});
		});
	};

	fs.close = function(fd, cb) {
		var f = fds[fd];
		if (!f) return nextTick(cb, errno.EBADF());

		fds[fd] = null;
		nextTick(listeners.cb(f.key, cb));
	};

	fs.write = function(fd, buf, off, len, pos, cb) {
		var f = fds[fd];
		if (!cb) cb = noop;
		if (!f || !f.writable) return nextTick(cb, errno.EBADF());

		if (pos === null) pos = f.writePos || 0;

		var slice = buf.slice(off, off+len);
		f.writePos = pos + slice.length;

		bl.write(f.blob, slice, {start:pos, append:true}, function(err) {
			if (err) return cb(err);
			cb(null, len, buf);
		});
	};

	fs.read = function(fd, buf, off, len, pos, cb) {
		var f = fds[fd];
		if (!cb) cb = noop;
		if (!f || !f.readable) return nextTick(cb, errno.EBADF());

		if (pos === null) pos = fs.readPos || 0;

		bl.read(f.blob, {start:pos, end:pos+len-1}, function(err, read) {
			if (err) return cb(err);
			var slice = read.slice(0, len);
			slice.copy(buf, off);
			fs.readPos = pos+slice.length;
			cb(null, slice.length, buf);
		});
	};

	fs.fsync = function(fd, cb) {
		var f = fds[fd];
		if (!cb) cb = noop;
		if (!f || !f.writable) return nextTick(cb, errno.EBADF());

		nextTick(cb);
	};

	fs.ftruncate = function(fd, len, cb) {
		var f = fds[fd];
		if (!cb) cb = noop;
		if (!f) return nextTick(cb, errno.EBADF());

		fs.truncate(f.blob, len, cb);
	};

	fs.fchown = function(fd, uid, gid, cb) {
		var f = fds[fd];
		if (!cb) cb = noop;
		if (!f) return nextTick(cb, errno.EBADF());

		fs.chown(f.key, uid, gid, cb);
	};

	fs.fchmod = function(fd, mode, cb) {
		var f = fds[fd];
		if (!cb) cb = noop;
		if (!f) return nextTick(cb, errno.EBADF());

		fs.chmod(f.key, mode, cb);
	};

	fs.futimes = function(fd, atime, mtime, cb) {
		var f = fds[fd];
		if (!cb) cb = noop;
		if (!f) return nextTick(cb, errno.EBADF());

		fs.utimes(f.key, atime, mtime, cb);
	};

	fs.fstat = function(fd, cb) {
		var f = fds[fd];
		if (!f) return nextTick(cb, errno.EBADF());

		fs.stat(f.key, cb);
	};

	fs.symlink = function(target, name, cb) {
		if (!cb) cb = noop;
		ps.follow(target, function(err, stat, target) {
			if (err) return cb(err);
			ps.get(name, function(err, stat) {
				if (err && err.code !== 'ENOENT') return cb(err);
				if (stat) return cb(errno.EEXIST(name));
				ps.put(name, {type:'symlink', target:target, mode:octal(777)}, cb);
			});
		});
	};

	fs.readlink = function(key, cb) {
		ps.get(key, function(err, stat) {
			if (err) return cb(err);
			if (!stat.target) return cb(errno.EINVAL(key));
			cb(null, stat.target);
		});
	};

	fs.link = function(target, name, cb) {
		if (!cb) cb = noop;
		ps.follow(target, function(err, stat, target) {
			if (err) return cb(err);
			if (!stat.isFile()) return cb(errno.EINVAL(target));
			ps.get(name, function(err, st) {
				if (err && err.code !== 'ENOENT') return cb(err);
				if (st) return cb(errno.EEXIST(name));
				var link = target+'\xff'+inc();
				links.put(target+'\xff', target, function(err) {
					if (err) return cb(err);
					links.put(link, target, function(err) {
						if (err) return cb(err);
						ps.put(name, {type:'file', link:link, blob:target, mode:stat.mode}, cb);
					});
				});
			});
		});
	};

	return fs;
};