cnpmjs.org
Version:
Private npm registry and web for Enterprise, base on MySQL and Simple Store Service
216 lines (196 loc) • 6.17 kB
JavaScript
const debug = require('debug')('cnpmjs.org:sync:index');
const co = require('co');
const ms = require('humanize-ms');
const util = require('util');
const config = require('../config');
const mail = require('../common/mail');
const logger = require('../common/logger');
const totalService = require('../services/total');
let sync = null;
switch (config.syncModel) {
case 'all':
sync = require('./sync_all');
break;
case 'exist':
sync = require('./sync_exist');
break;
}
if (!sync && config.enableCluster) {
console.log('[%s] [sync_worker:%s] no need to sync, exit now', Date(), process.pid);
process.exit(0);
}
console.log('[%s] [sync_worker:%s] syncing with %s mode, changesStreamingSync: %s',
Date(), process.pid, config.syncModel, config.changesStreamingSync);
function onerror(err) {
logger.error(err);
}
// set sync_status = 0 at first
co(function* () {
yield totalService.updateSyncStatus(0);
yield checkSyncStatus();
}).catch(onerror);
let syncInterval = ms(config.syncInterval);
const minSyncInterval = ms('5m');
if (!syncInterval || syncInterval < minSyncInterval) {
syncInterval = minSyncInterval;
}
if (sync) {
// the same time only sync once
let syncing = false;
const syncFn = co.wrap(function*() {
debug('mode: %s, syncing: %s', config.syncModel, syncing);
if (!syncing) {
syncing = true;
debug('start syncing');
let data;
let error;
try {
data = yield sync();
} catch (err) {
error = err;
error.message += ' (sync package error)';
logger.syncError(error);
}
data && logger.syncInfo(data);
if (!config.debug) {
sendMailToAdmin(error, data, new Date());
}
syncing = false;
}
// check last_sync_time and last_exist_sync_time
yield checkSyncStatus();
});
syncFn().catch(onerror);
setInterval(() => syncFn().catch(onerror), syncInterval);
}
/**
* sync popular modules
*/
if (config.syncPopular) {
const sync = require('./sync_popular');
let syncing = false;
const syncFn = co.wrap(function*() {
if (syncing) {
return;
}
syncing = true;
logger.syncInfo('Start syncing popular modules...');
let data;
let error;
try {
data = yield sync();
} catch (err) {
error = err;
error.message += ' (sync package error)';
logger.syncError(error);
}
if (data) {
logger.syncInfo(data);
}
if (!config.debug) {
sendMailToAdmin(error, data, new Date());
}
syncing = false;
});
syncFn().catch(onerror);
setInterval(() => syncFn().catch(onerror), ms(config.syncPopularInterval));
}
if (config.syncChangesStream) {
const sync = require('./changes_stream_syncer');
let syncing = false;
const syncFn = co.wrap(function*() {
if (syncing) {
return;
}
syncing = true;
logger.syncInfo('Start changes stream syncing...');
try {
yield sync();
} catch (err) {
err.message += ' (sync changes stream error)';
logger.syncError(err);
}
syncing = false;
});
syncFn().catch(onerror);
setInterval(() => syncFn().catch(onerror), ms('1m'));
}
function sendMailToAdmin(err, result, syncTime) {
result = result || {};
const to = [];
for (var name in config.admins) {
to.push(config.admins[name]);
}
debug('Send email to all admins: %j, with err message: %s, result: %j, start sync time: %s.',
to, err ? err.message : '', result, syncTime);
let subject;
let type;
let html;
if (err) {
// ignore 503, 504 error
// 504: Gateway Time-out
if (err.status === 503 || err.status === 504) {
return;
}
if (err.name === 'JSONResponseFormatError') {
return;
}
subject = 'Sync Error';
type = 'error';
html = util.format('Sync packages from official registry failed.\n' +
'Start sync time is %s.\nError message is %s: %s\n%s.', syncTime, err.name, err.message, err.stack);
} else if (result.fails && result.fails.length) {
subject = 'Sync Finished But Some Packages Failed';
type = 'warn';
html = util.format('Sync packages from official registry finished, but some packages sync failed.\n' +
'Start sync time is %s.\n %d packges sync failed: %j ...\n %d packages sync successes :%j ...',
syncTime, result.fails.length, result.fails.slice(0, 10),
result.successes.length, result.successes.slice(0, 10));
// skip email notice when fails items small then 3
if (result.fails.length < 3) {
type = 'log';
}
} else if (result.successes && result.successes.length) {
subject = 'Sync Finished';
type = 'log';
html = util.format('Sync packages from official registry finished.\n' +
'Start sync time is %s.\n %d packages sync successes :%j ...',
syncTime, result.successes.length, result.successes.slice(0, 10));
}
debug('send email with type: %s, subject: %s, html: %s', type, subject, html);
logger.syncInfo('send email with type: %s, subject: %s, html: %s', type, subject, html);
if (type && type !== 'log') {
mail[type](to, subject, html, err => {
if (err) {
logger.error(err);
}
});
}
}
function* checkSyncStatus() {
// if source registry not cnpm, ignore it. e.g.: cnpmjs.org source registry is npmjs.org
if (!config.sourceNpmRegistryIsCNpm) {
return;
}
const total = yield totalService.getTotalInfo();
let lastSyncTime;
if (config.syncModel === 'all') {
lastSyncTime = total.last_sync_time;
} else if (config.syncModel === 'exist') {
lastSyncTime = total.last_exist_sync_time;
}
debug('checkSyncStatus start, lastSyncTime: %s, syncInterval: %s', lastSyncTime, syncInterval);
if (!lastSyncTime) {
return;
}
const diff = Date.now() - lastSyncTime;
const oneDay = 3600000 * 24;
const maxTime = Math.max(oneDay, syncInterval * 2);
if (diff > maxTime) {
const err = new Error('Last sync time is expired in ' + diff + ' ms, lastSyncTime: ' +
new Date(lastSyncTime) + ', maxTime: ' + maxTime + ' ms');
err.name = 'SyncExpiredError';
sendMailToAdmin(err, null, new Date());
}
}
;