nodebb-plugin-import
Version:
Import your forum data to nodebb
823 lines (720 loc) • 23.6 kB
JavaScript
const nbbRequire = require('nodebb-plugin-require');
const async = require('async');
const _ = require('underscore');
const { EventEmitter2 } = require('eventemitter2');
const COUNT_BATCH_SIZE = 600000;
const DEFAULT_EXPORT_BATCH_SIZE = 600000;
// mysql is terrible
const MAX_MYSQL_INT = -1 >>> 1;
const noop = function () {};
const getModuleId = function (module) {
if (module.indexOf('github.com') > -1) {
return module.split('/').pop().split('#')[0];
}
return module.split('@')[0];
};
const searchModulesCache = function (moduleName, callback) {
let mod = safeRequire(getModuleId(moduleName));
if (mod && ((mod = require.cache[mod]) !== undefined)) {
(function run(mod) {
mod.children.forEach((child) => {
run(child);
});
callback(mod);
}(mod));
}
};
var safeRequire = function (moduleName) {
let m;
try {
m = require(require.resolve(moduleName));
} catch (e) {
m = nbbRequire(moduleName);
}
return m;
};
const reloadModule = function (moduleName) {
searchModulesCache(moduleName, (mod) => {
delete require.cache[mod.id];
});
return safeRequire(moduleName);
};
(function (Exporter) {
const utils = require('../../public/js/utils');
Exporter._exporter = null;
Exporter._dispatcher = new EventEmitter2({
wildcard: true,
});
Exporter.init = function (config, cb) {
Exporter.config = config.exporter || {};
async.series([
function (next) {
const opt = { force: true };
if (config.exporter.skipInstall) {
opt.skipInstall = true;
opt.force = false;
}
Exporter.install(config.exporter.module, opt, next);
},
Exporter.setup,
], _.isFunction(cb) ? cb() : noop);
};
Exporter.setup = function (cb) {
Exporter.augmentLogFunctions();
Exporter._exporter.setup(Exporter.config, (err) => {
if (err) {
Exporter.emit('exporter.error', { error: err });
return cb(err);
}
Exporter.emit('exporter.ready');
cb();
});
};
Exporter.countAll = function (cb) {
async.series([
Exporter.countUsers,
Exporter.countGroups,
Exporter.countCategories,
Exporter.countTopics,
Exporter.countPosts,
Exporter.countRooms,
Exporter.countMessages,
Exporter.countVotes,
Exporter.countBookmarks,
], (err, results) => {
if (err) return cb(err);
cb({
users: results[0],
groups: results[1],
categories: results[2],
topics: results[3],
posts: results[4],
rooms: results[5],
messages: results[6],
votes: results[7],
bookmarks: results[8],
});
});
};
Exporter.countUsers = function (cb) {
if (Exporter._exporter.countUsers) {
return Exporter._exporter.countUsers(cb);
}
let count = 0;
Exporter.exportUsers((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
Exporter.countGroups = function (cb) {
if (Exporter._exporter.countGroups) {
return Exporter._exporter.countGroups(cb);
}
let count = 0;
Exporter.exportGroups((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
Exporter.countRooms = function (cb) {
if (Exporter._exporter.countRooms) {
return Exporter._exporter.countRooms(cb);
}
let count = 0;
Exporter.exportRooms((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
}, {
batch: COUNT_BATCH_SIZE,
}, (err) => {
cb(err, count);
});
};
Exporter.countMessages = function (cb) {
if (Exporter._exporter.countMessages) {
return Exporter._exporter.countMessages(cb);
}
let count = 0;
Exporter.exportMessages((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
Exporter.countCategories = function (cb) {
if (Exporter._exporter.countCategories) {
return Exporter._exporter.countCategories(cb);
}
let count = 0;
Exporter.exportCategories((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
Exporter.countTopics = function (cb) {
if (Exporter._exporter.countTopics) {
return Exporter._exporter.countTopics(cb);
}
let count = 0;
Exporter.exportTopics((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
Exporter.countPosts = function (cb) {
if (Exporter._exporter.countPosts) {
return Exporter._exporter.countPosts(cb);
}
let count = 0;
Exporter.exportPosts((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
Exporter.countVotes = function (cb) {
if (Exporter._exporter.countVotes) {
return Exporter._exporter.countVotes(cb);
}
let count = 0;
Exporter.exportVotes((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
Exporter.countBookmarks = function (cb) {
if (Exporter._exporter.countBookmarks) {
return Exporter._exporter.countBookmarks(cb);
}
let count = 0;
Exporter.exportBookmarks((err, map, arr, nextBatch) => {
count += arr.length;
nextBatch();
},
{
batch: COUNT_BATCH_SIZE,
},
(err) => {
cb(err, count);
});
};
const onGroups = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_gid'), arg1);
}
};
Exporter.getGroups = function (cb) {
if (!Exporter._exporter.getGroups) {
return onGroups(null, {}, [], cb);
}
Exporter._exporter.getGroups((err, arg1, arg2) => {
onGroups(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedGroups = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedGroups) {
return Exporter.getGroups(cb);
}
Exporter._exporter.getPaginatedGroups(start, end, (err, arg1, arg2) => {
onUsers(err, arg1, arg2, cb);
});
};
var onUsers = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_uid'), arg1);
}
};
Exporter.getUsers = function (cb) {
Exporter._exporter.getUsers((err, arg1, arg2) => {
onUsers(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedUsers = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedUsers) {
return Exporter.getUsers(cb);
}
Exporter._exporter.getPaginatedUsers(start, end, (err, arg1, arg2) => {
onUsers(err, arg1, arg2, cb);
});
};
const onCategories = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_cid'), arg1);
}
};
Exporter.getCategories = function (cb) {
Exporter._exporter.getCategories((err, arg1, arg2) => {
onCategories(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedCategories = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedCategories) {
return Exporter.getCategories(cb);
}
Exporter._exporter.getPaginatedCategories(start, end, (err, arg1, arg2) => {
onCategories(err, arg1, arg2, cb);
});
};
const onTopics = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_tid'), arg1);
}
};
Exporter.getTopics = function (cb) {
Exporter._exporter.getTopics((err, arg1, arg2) => {
onTopics(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedTopics = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedTopics) {
return Exporter.getTopics(cb);
}
Exporter._exporter.getPaginatedTopics(start, end, (err, arg1, arg2) => {
onTopics(err, arg1, arg2, cb);
});
};
const onPosts = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_pid'), arg1);
}
};
Exporter.getPosts = function (cb) {
Exporter._exporter.getPosts((err, arg1, arg2) => {
onPosts(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedPosts = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedPosts) {
return Exporter.getPosts(cb);
}
Exporter._exporter.getPaginatedPosts(start, end, (err, arg1, arg2) => {
onPosts(err, arg1, arg2, cb);
});
};
const onRooms = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_mid'), arg1);
}
};
Exporter.getRooms = function (cb) {
if (!Exporter._exporter.getRooms) {
Exporter.emit('exporter.warn', { warn: 'Current selected exporter does not implement getRooms function, skipping...' });
return onRooms(null, {}, [], cb);
}
Exporter._exporter.getRooms((err, arg1, arg2) => {
onRooms(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedRooms = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedRooms) {
return Exporter.getRooms(cb);
}
Exporter._exporter.getPaginatedRooms(start, end, (err, arg1, arg2) => {
onRooms(err, arg1, arg2, cb);
});
};
const onMessages = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_mid'), arg1);
}
};
Exporter.getMessages = function (cb) {
if (!Exporter._exporter.getMessages) {
Exporter.emit('exporter.warn', { warn: 'Current selected exporter does not implement getMessages function, skipping...' });
return onMessages(null, {}, [], cb);
}
Exporter._exporter.getMessages((err, arg1, arg2) => {
onMessages(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedMessages = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedMessages) {
return Exporter.getMessages(cb);
}
Exporter._exporter.getPaginatedMessages(start, end, (err, arg1, arg2) => {
onMessages(err, arg1, arg2, cb);
});
};
// Votes getters
const onVotes = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_vid'), arg1);
}
};
Exporter.getVotes = function (cb) {
if (!Exporter._exporter.getVotes) { // votes is an optional feature
Exporter.emit('exporter.warn', { warn: 'Current selected exporter does not implement getVotes function, skipping...' });
return onVotes(null, {}, [], cb);
}
Exporter._exporter.getVotes((err, arg1, arg2) => {
onVotes(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedVotes = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedVotes) {
return Exporter.getVotes(cb);
}
Exporter._exporter.getPaginatedVotes(start, end, (err, arg1, arg2) => {
onVotes(err, arg1, arg2, cb);
});
};
// Bookmarks getters
const onBookmarks = function (err, arg1, arg2, cb) {
if (err) return cb(err);
if (_.isObject(arg1)) {
return cb(null, arg1, _.isArray(arg2) ? arg2 : _.toArray(arg1));
}
if (_.isArray(arg1)) {
return cb(null, _.isObject(arg2) ? arg2 : _.indexBy(arg1, '_bid'), arg1);
}
};
Exporter.getBookmarks = function (cb) {
if (!Exporter._exporter.getBookmarks) { // votes is an optional feature
Exporter.emit('exporter.warn', { warn: 'Current selected exporter does not implement getBookmarks function, skipping...' });
return onBookmarks(null, {}, [], cb);
}
Exporter._exporter.getBookmarks((err, arg1, arg2) => {
onBookmarks(err, arg1, arg2, cb);
});
};
Exporter.getPaginatedBookmarks = function (start, end, cb) {
if (!Exporter._exporter.getPaginatedBookmarks) {
return Exporter.getBookmarks(cb);
}
Exporter._exporter.getPaginatedBookmarks(start, end, (err, arg1, arg2) => {
onBookmarks(err, arg1, arg2, cb);
});
};
Exporter.teardown = function (cb) {
Exporter._exporter.teardown(cb);
};
Exporter.install = function (module, options, next) {
const npm = require('npm');
Exporter._exporter = null;
if (_.isFunction(options)) {
next = options;
options = {};
}
if (options.skipInstall) {
const mid = getModuleId(module);
Exporter._exporter = reloadModule(mid);
Exporter._module = module;
Exporter._moduleId = mid;
return next();
}
npm.load(options, (err) => {
if (err) {
next(err);
}
Exporter.emit('exporter.log', `installing: ${module}`);
npm.config.set('spin', false);
npm.config.set('force', !!options.force);
npm.config.set('verbose', true);
npm.commands.install([module], (err) => {
if (err) {
next(err);
}
const moduleId = getModuleId(module);
const exporter = reloadModule(moduleId);
if (!Exporter.isCompatible(exporter)) {
// no?
if (module.indexOf('github.com/akhoury') === -1) {
Exporter.emit('exporter.warn', { warn: `${module} is not compatible, trying github.com/akhoury's fork` });
npm.commands.uninstall([module], (err) => {
if (err) {
next(err);
}
Exporter.emit('exporter.log', `uninstalled: ${module}`);
// let's try my #master fork till the PRs close and get published
Exporter.install(`git://github.com/akhoury/${moduleId}#master`, { 'no-registry': true }, next);
});
} else {
Exporter.emit('exporter.error', { error: `${module} is not compatible.` });
next({ error: `${module} is not compatible.` });
}
} else {
if (!Exporter.supportsPagination(exporter)) {
Exporter.emit('exporter.warn', {
warn: `${module} does not support Pagination, `
+ `it will work, but if you run into memory issues, you might want to contact the developer of it or add support your self. `
+ `See https://github.com/akhoury/nodebb-plugin-import/blob/master/write-my-own-exporter.md`,
});
}
Exporter._exporter = exporter;
Exporter._module = module;
Exporter._moduleId = moduleId;
next();
}
});
});
};
Exporter.isCompatible = function (exporter) {
exporter = exporter || Exporter._exporter;
return exporter
&& _.isFunction(exporter.setup)
&& (
Exporter.supportsPagination(exporter)
|| (
_.isFunction(exporter.getUsers)
&& _.isFunction(exporter.getCategories)
&& _.isFunction(exporter.getTopics)
&& _.isFunction(exporter.getPosts)
)
)
&& _.isFunction(exporter.teardown);
};
Exporter.supportsPagination = function (exporter, type) {
exporter = exporter || Exporter._exporter;
return exporter
&& (function (type) {
switch (type) {
case 'users':
return _.isFunction(exporter.getPaginatedUsers);
break;
case 'categories':
return _.isFunction(exporter.getPaginatedCategories);
break;
case 'topics':
return _.isFunction(exporter.getPaginatedTopics);
break;
case 'posts':
return _.isFunction(exporter.getPaginatedPosts);
break;
// optional interfaces
case 'rooms':
return _.isFunction(exporter.getPaginatedRooms);
break;
case 'messages':
return _.isFunction(exporter.getPaginatedMessages);
break;
case 'votes':
return _.isFunction(exporter.getPaginatedVotes);
break;
case 'bookmarks':
return _.isFunction(exporter.getPaginatedBookmarks);
break;
// if just checking if in general pagination is supported, then don't check the optional ones
default:
return _.isFunction(exporter.getPaginatedUsers)
&& _.isFunction(exporter.getPaginatedCategories)
&& _.isFunction(exporter.getPaginatedTopics)
&& _.isFunction(exporter.getPaginatedPosts);
}
}(type));
};
Exporter.exportGroups = function (process, options, callback) {
return Exporter.exportType('groups', process, options, callback);
};
Exporter.exportUsers = function (process, options, callback) {
return Exporter.exportType('users', process, options, callback);
};
Exporter.exportRooms = function (process, options, callback) {
return Exporter.exportType('rooms', process, options, callback);
};
Exporter.exportMessages = function (process, options, callback) {
return Exporter.exportType('messages', process, options, callback);
};
Exporter.exportCategories = function (process, options, callback) {
return Exporter.exportType('categories', process, options, callback);
};
Exporter.exportTopics = function (process, options, callback) {
return Exporter.exportType('topics', process, options, callback);
};
Exporter.exportPosts = function (process, options, callback) {
return Exporter.exportType('posts', process, options, callback);
};
Exporter.exportVotes = function (process, options, callback) {
return Exporter.exportType('votes', process, options, callback);
};
Exporter.exportBookmarks = function (process, options, callback) {
return Exporter.exportType('bookmarks', process, options, callback);
};
Exporter.eachTypeImmediateProcess = function (type, obj, options, callback) {
var exporter = Exporter._exporter;
if (Exporter.supportsEachTypeImmediateProcess(type) && obj) {
return exporter[`each${type[0].toUpperCase()}${type.slice(1)}ImmediateProcess`](obj, options, callback);
}
callback();
};
Exporter.supportsEachTypeImmediateProcess = function (type) {
return Exporter._exporter && typeof Exporter._exporter[`each${type[0].toUpperCase()}${type.slice(1)}ImmediateProcess`] === 'function';
};
Exporter.exportType = function (type, process, options, callback) {
if (typeof options === 'function') {
callback = options;
options = {};
}
callback = typeof callback === 'function' ? callback : function () {};
options = options || {};
if (typeof process !== 'function') {
throw new Error(`${process} is not a function`);
}
// custom done condition
options.doneIf = typeof options.doneIf === 'function' ? options.doneIf : function () {};
// always start at, useful when deleting all records
// options.alwaysStartAt
// i.e. exporter.getPaginatedPosts
// will fallback to get[Type] is pagination is not supported
const Type = type[0].toUpperCase() + type.substr(1).toLowerCase();
const fnName = `getPaginated${Type}`;
const batch = Exporter.supportsPagination(null, type) ? options.batch || Exporter._exporter.DEFAULT_EXPORT_BATCH_SIZE || DEFAULT_EXPORT_BATCH_SIZE : MAX_MYSQL_INT;
let start = 0;
const limit = batch;
let done = false;
async.whilst(
(err) => {
if (err) {
return true;
}
return !done;
},
(next) => {
if (!Exporter.supportsPagination(null, type) && start > 0) {
done = true;
return next();
}
Exporter[fnName](start, limit, (err, map, arr) => {
if (err) {
return next(err);
}
if (!arr.length || options.doneIf(start, limit, map, arr)) {
done = true;
return next();
}
process(err, map, arr, (err) => {
if (err) {
return next(err);
}
start += utils.isNumber(options.alwaysStartAt) ? options.alwaysStartAt : batch + 1;
next();
});
});
},
callback,
);
};
Exporter.emit = function (type, b, c) {
const args = Array.prototype.slice.call(arguments, 0);
console.log.apply(console, args);
args.unshift(args[0]);
Exporter._dispatcher.emit.apply(Exporter._dispatcher, args);
};
Exporter.on = function () {
Exporter._dispatcher.on.apply(Exporter._dispatcher, arguments);
};
Exporter.once = function () {
Exporter._dispatcher.once.apply(Exporter._dispatcher, arguments);
};
Exporter.removeAllListeners = function () {
Exporter._dispatcher.removeAllListeners();
};
Exporter.augmentFn = function (base, extra) {
return (function () {
return function () {
base.apply(this, arguments);
extra.apply(this, arguments);
};
}());
};
Exporter.augmentLogFunctions = function () {
const { log } = Exporter._exporter;
if (_.isFunction(log)) {
Exporter._exporter.log = Exporter.augmentFn(log, function (a, b, c) {
const args = _.toArray(arguments);
args[0] = `[${(new Date()).toISOString()}] ${args[0]}`;
args.unshift('exporter.log');
Exporter.emit.apply(Exporter, args);
});
}
const { warn } = Exporter._exporter;
if (_.isFunction(warn)) {
Exporter._exporter.warn = Exporter.augmentFn(warn, function () {
const args = _.toArray(arguments);
args[0] = `[${(new Date()).toISOString()}] ${args[0]}`;
args.unshift('exporter.warn');
Exporter.emit.apply(Exporter, args);
});
}
const { error } = Exporter._exporter;
if (_.isFunction(error)) {
Exporter._exporter.error = Exporter.augmentFn(error, function () {
const args = _.toArray(arguments);
args[0] = `[${(new Date()).toISOString()}] ${args[0]}`;
args.unshift('exporter.error');
Exporter.emit.apply(Exporter, args);
});
}
};
}(module.exports));