tooltwist
Version:
Tooltwist Command Line Interface
471 lines (384 loc) • 13 kB
JavaScript
/**
* Trigger generating Navpoints and other files in a ToolTwist server.
*/
var http = require('http'),
fs = require('fs'),
glob = require('glob'),
moment = require('moment'),
Log = require('./log'),
Config = require('./config'),
Templates = require('./templates'),
Mustache = require("mustache"),
mkdirp = require('mkdirp');
var HOST = 'localhost';
// If there's less than 'NOT_MANY_NAVPOINTS' navpoints to be generated, do the entire generate in a single thread.
var NOT_MANY_NAVPOINTS = 1000
// Try to process 'NAVPOINTS_PER_THREAD' navpoints in each thread, however if 'MAX_THREADS'
// would be exceeded, it may be necessary to handle more navpoints in each thread.
var NAVPOINTS_PER_THREAD = 30
var MAX_THREADS = 50
// Generation threads waiting to complete.
var _threadsRemainingToRun;
var _numThreadsWithErrors = 0;
var _logdir;
/**
* Generate navpoints and other files.
*/
exports.generate = function generate() {
findNavpoints();
};
/**
* Call the Tomcat server to get the names of all the navpoints.
*/
function findNavpoints() {
Log.debug();
Log.debug('generate.findNavpoints()');
var path = '/ttsvr/listNavpoints'
+ '?project=' + Config.PROJECT_NAME
+ '&shuffle=true';
var options = {
host: HOST,
port: Config.HTTP_PORT,
path: path
};
Log.info();
Log.info("Call server to get list of navpoints.");
Log.info("http://" + HOST + ':' + Config.HTTP_PORT + path)
// Call the server
http.get(options, function(res) {
Log.debug("Got response: " + res.statusCode);
Log.debug('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
var reply = '';
res.on('data', function (chunk) {
Log.debug('BODY: ' + chunk);
reply += chunk;
}).on('end', function(){
Log.debug('END\n' + reply)
var navpoints = []
while (reply != '') {
var i = reply.indexOf('\n');
if (i < 0)
break;
var np = reply.substring(0, i)
reply = reply.substring(i + 1)
if (np.indexOf('navpoint=') == 0) {
np = np.substring(9);
navpoints.push(np);
} else if (np == 'COMPLETION-MARKER') {
break;
} else {
// Unknown line
}
}
Log.log('Found ' + navpoints.length + ' navpoints.')
Log.debug('navpoints=', navpoints)
return generateNavpoints(navpoints);
}).on('close', function(){
Log.debug('CLOSE')
});
}).on('error', function(e) {
Log.error("Got error: " + e.message);
});
}
/**
* Call the Tomcat server to generate a list of navpoints.
*/
function generateNavpoints(navpoints) {
Log.debug();
Log.debug('generate.generateNavpoints()\n', navpoints);
// Decide how many threads to use.
var threads = [];
if (navpoints.length <= NOT_MANY_NAVPOINTS) {
// Use a single thread
Log.log('Will use a single thread.\n')
var xml = ''
for (var i = 0; i < navpoints.length; i++) {
xml += '\t\t<navpoint>' + navpoints[i] + '</navpoint>\n'
}
threads.push(xml);
} else {
// Use multiple threads
var numThreads = Math.floor(navpoints.length / NAVPOINTS_PER_THREAD) + 2
if (numThreads > MAX_THREADS) {
numThreads = MAX_THREADS
}
Log.log('Will use ' + numThreads + ' threads.')
// Prepare an array of XML to define the navpoints for each thread.
for (var i = 0; i < numThreads; i++) {
threads.push('')
}
// Add the navpoints to the XML for each thread
for (var i = 0; i < navpoints.length; i++) {
var xml = '\t\t<navpoint>' + navpoints[i] + '</navpoint>\n';
Log.debug(' xml=' + xml)
if (i == 0) {
// Put only a single navpoint in the first thread,
// since it has to load config files, etc.
threads[0] = xml
} else {
// Put the rest in the remaining threads
var thread = 1 + ((i-1) % (numThreads-1))
threads[thread] = threads[thread] + xml;
}
}
}
/* Display the navpoints for each filter.
for (var i = 0; i < threads.length; i++) {
Log.debug('\n\n#' + i + ':\n' + threads[i])
}
*/
// // Decide which filter template to use
// var templatePath = __dirname + '/../templates/generate/filters.xml'
// var path = Config.TEMPLATE_OVERRIDE_DIR + '/filters.xml'
// if (fs.existsSync(path)) {
// Log.debug(' - using custom template: ' + path)
// templatePath = path;
// }
// Log.debug('Use template ' + path)
mkdirp.sync(Config.HIDDEN_DIR + '/do_generate');
mkdirp.sync(Config.HIDDEN_DIR + '/do_generate/transformed-web-assets');
_threadsRemainingToRun = threads.length;
/**
* Start a generation thread. (Note threadNo starts at 1)
*/
function startGenerateThread(threadNo) {
if (threadNo > threads.length) {
return;
}
Log.debug('Start thread ' + threadNo)
// Create the filter file
var data = {
navpointXml: threads[threadNo-1] // Starts with i==1
};
var filterFile = Config.HIDDEN_DIR + '/do_generate/filter-' + threadNo + '.xml';
var defaultTemplateDir = Config.RESOURCES_DIR + '/templates';
Templates.copyTemplate(defaultTemplateDir, 'generate/filters.xml', filterFile, data);
// Start the generate thread.
// var filterFile = process.cwd()'/ControllerV8/launchpads/controllerTest/launchpads/myProject/filters/filter-1.xml';
var buildId = '20140720-1759';//ZZZZZZ
callServerToGenerate(buildId, filterFile, threads.length, threadNo)
// Wait a while before the next thread starts.
// Give the first thread a head start to load the ToolTwist metedata, etc
var delay = (threadNo == 1) ? 15000 : 100;
delay = 100;
setTimeout(function(){
startGenerateThread(threadNo + 1);
}, delay);
}
// Start generating.
Log.info();
Log.info("Call server to generate navpoints.");
startGenerateThread(1); // Thread numbers start at 1
}
function callServerToGenerate(buildId, filterFile, numThreads, threadId) {
var path = '/ttsvr/generator'
+ '?project=' + Config.PROJECT_NAME
+ '&webapp=ttsvr'
// + '&base=' + Config.PROJECT_ROOT
+ '&base=' + Config.HIDDEN_DIR + '/do_generate'
+ '&config=' + filterFile
// + '&deploy=' + Config.BUILD_DIR;
+ '&deploy=' + Config.HIDDEN_DIR + '/do_generate/transformed-web-assets'
// ZZZZZ
// Force the generate to use exactly what is seen in the Designer.
path += '&designerVersions=true';
// The second and subsequent thread should generate the required navpoints, but not install any other web assets.
if (threadId > 1) {
path += "&onlyInstallNavpoints"
}
// If we have multiple threads, use a different build ID for
// each thread and keep the progress messages to a minimum.
if (numThreads > 1) {
path += '&quiet'
path += '&buildId=' + buildId + '-thread-' + threadId
} else {
path += '&buildId=' + buildId
}
// Just display the first URL
if (threadId == 1) {
var url = 'http://' + HOST + ':' + Config.HTTP_PORT + path
Log.info('uri=' + url);
//console.log('uri=' + url);
}
var options = {
host: HOST,
port: Config.HTTP_PORT,
path: path
};
_logdir = Config.HIDDEN_DIR + '/do_generate/logs/' + buildId;
if (numThreads > 1) {
_logdir += '-thread-' + threadId;
}
var now = moment().format('LLLL');
var startTime = new Date().getTime();
Log.info(" " + url)
Log.log('+----------------------------------------------------------------------------------------+');
Log.log('| START: filter-' + threadId + '.xml');
Log.log('| ' + now);
Log.log('| logs in ' + _logdir);
Log.log('`-----------------------------------------------------------------------------------------');
//ZZZZ Check these
// Call the server
http.get(options, function(res) {
Log.debug("Got response: " + res.statusCode);
if (res.statusCode == 400) {
// HTTPBadRequest
Log.error('\n\n** Thread ' + threadId + ':');
Log.error('**');
Log.error('** ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR');
Log.error('**');
Log.error('** The generator has rejected the parameters we passed.');
Log.error('** URI=' + url);
Log.error('**');
_numThreadsWithErrors++;
threadFinished();
return;
} else if (res.statusCode != 200) {
// Some other error
Log.error('\n\n** Thread ' + threadId + ':');
Log.error('**');
Log.error('** ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR');
Log.error('**');
Log.error('** Error contacting server: ' + res + '\n\n\n\n');
Log.error('** URI=' + url);
Log.error('**');
_numThreadsWithErrors++;
threadFinished();
return;
}
Log.debug('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
var reply = '';
res.on('data', function (chunk) {
Log.debug('BODY: ' + chunk);
reply += chunk;
}).on('end', function(){
// Normal reply from the generate servlet.
Log.debug('END\n' + reply)
// Get the number of errors from the 'ERRORS=...' file in the log directory.
var prefix = _logdir + '/ERRORS='
var errorFiles = glob.sync(prefix + '*');
var threadErrors = 0;
if (errorFiles.length > 0) {
suffix = errorFiles[0].substring(prefix.length)
threadErrors = parseInt(suffix)
if (threadErrors > 0) {
_numThreadsWithErrors++;
}
}
// Check times and durations
var now = moment().format('LLLL');
var endTime = new Date().getTime();
var duration = endTime - startTime;
var seconds = Math.floor(duration / 1000) % 60;
var minutes = Math.floor(duration / (60 * 1000));
//ZZZZ Fix hard coded values
Log.log('*-------------------------------------------------------------------------+');
Log.log('* END: filter-' + threadId + '.xml');
Log.log('* Total of ' + threadErrors + ' errors in thread.');
Log.log('* ' + now);
Log.log('* ' + duration + ' ms = ' + minutes + ' minutes, ' + seconds + ' seconds.');
Log.log('`--------------------------------------------------------------------------');
Log.debug(reply);
/*
var navpoints = []
while (reply != '') {
var i = reply.indexOf('\n');
if (i < 0)
break;
var np = reply.substring(0, i)
reply = reply.substring(i + 1)
if (np.indexOf('navpoint=') == 0) {
np = np.substring(9);
navpoints.push(np);
} else if (np == 'COMPLETION-MARKER') {
break;
} else {
// Unknown line
}
}
Log.debug('navpoints=', navpoints)
return generateNavpoints(navpoints);
*/
threadFinished();
}).on('close', function(){
Log.debug('CLOSE')
threadFinished();
});
}).on('error', function(e) {
Log.error("Got error: " + e.message);
threadFinished();
});
}
/**
* A thread has finished. If there are none left, finish up.
*/
function threadFinished() {
Log.debug('threadFinished ' + _threadsRemainingToRun)
_threadsRemainingToRun--;
if (_threadsRemainingToRun <= 0) {
// All threads are finished
if (_numThreadsWithErrors > 0) {
// Display a large error message
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.error('Fatal errors in ' + _numThreadsWithErrors + ' threads.');
Log.error('Logs in ' + _logdir);
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.error(' _ _ _ _ _ _ ')
Log.error('|_ |_) |_) / \\ |_) ( ')
Log.error('|_ | \\ | \\ \\_/ | \\ _) ')
Log.error('')
Log.error(' _ _ _ _ _ _ ')
Log.error(' |_ |_) |_) / \\ |_) ( ')
Log.error(' |_ | \\ | \\ \\_/ | \\ _ ) ')
Log.error('')
Log.error(' _ _ _ _ _ _ ')
Log.error(' |_ |_) |_) / \\ |_) ( ')
Log.error(' |_ | \\ | \\ \\_/ | \\ _ ) ')
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.error('')
// We don't want to stop the generate, as the errors may not be significant.
process.exit(0);
} else {
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.error(' _ _ _ _ _ _ _ ')
Log.error(' |\\| / \\ |_ |_) |_) / \\ |_) ( ')
Log.error(' | | \\_/ |_ | \\ | \\ \\_/ | \\ _) ')
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.error('')
Log.log('All threads completed without errors.')
process.exit(0)
}
}
}
/*
//ZZZZ Duplicate from tooltwist-cli.js
function copy(from, to, data) {
//Log.debug('copy ' + from + ", " + to)
//Log.debug(from + ' --> ' + to)
var template = fs.readFileSync(from, "utf8");
var data = data ? data : { };
var output = Mustache.render(template, data);
//Log.debug("****************************************")
//Log.debug("output = " + output)
fs.writeFileSync(to, output, { encoding: 'utf8' });
}
*/