webgme-rosmod
Version:
This repository contains ROSMOD developed for WebGME. ROSMOD is a web-based, collaborative, modeling and execution environment for distributed embedded applications built using ROS http://rosmod.rcps.isis.vanderbilt.edu
1,210 lines (1,093 loc) • 54.2 kB
JavaScript
/*globals define*/
/*jshint node:true, browser:true*/
/**
* Generated by PluginGenerator 0.14.0 from webgme on Wed Mar 02 2016 22:16:42 GMT-0600 (Central Standard Time).
*/
define([
'plugin/PluginConfig',
'plugin/PluginBase',
'text!./metadata.json',
'ejs', // for ejs templates
'common/util/xmljsonconverter', // used to save model as json
'plugin/SoftwareGenerator/SoftwareGenerator/Templates/Templates', //
'remote-utils/remote-utils',
'webgme-to-json/webgme-to-json',
'rosmod/processor',
// hfsm
'hfsm-library/src/plugins/SoftwareGenerator/SoftwareGenerator',
// promises
'q'
], function (
PluginConfig,
PluginBase,
pluginMetadata,
ejs,
Converter,
TEMPLATES,
utils,
webgmeToJson,
processor,
// hfsm
hfsmSoftwareGenerator,
Q) {
'use strict';
pluginMetadata = JSON.parse(pluginMetadata);
/**
* Initializes a new instance of SoftwareGenerator.
* @class
* @augments {PluginBase}
* @classdesc This class represents the plugin SoftwareGenerator.
* @constructor
*/
var SoftwareGenerator = function () {
// Call base class' constructor.
PluginBase.call(this);
this.pluginMetadata = pluginMetadata;
this.FILES = {
'component_cpp': 'component.cpp.ejs',
'component_hpp': 'component.hpp.ejs',
'cmakelists': 'CMakeLists.txt.ejs',
'package_xml': 'package_xml.ejs',
'doxygen_config': 'doxygen_config.ejs'
};
};
SoftwareGenerator.metadata = pluginMetadata;
// Prototypal inheritance from PluginBase.
SoftwareGenerator.prototype = Object.create(PluginBase.prototype);
SoftwareGenerator.prototype.constructor = SoftwareGenerator;
SoftwareGenerator.prototype.notify = function(level, msg, obj) {
var self = this;
if (!obj)
obj = self.activeNode;
var prefix = self.projectId + '::' + self.projectName + '::' + level + '::';
var lines = msg.split('\n');
lines.map(function(line) {
if (level=='error')
self.logger.error(line);
else if (level=='debug')
self.logger.debug(line);
else if (level=='info')
self.logger.info(line);
else if (level=='warning')
self.logger.warn(line);
self.createMessage(obj, line, level);
self.sendNotification(prefix+line);
});
};
/**
* Main function for the plugin to execute. This will perform the execution.
* Notes:
* - Always log with the provided logger.[error,warning,info,debug].
* - Do NOT put any user interaction logic UI, etc. inside this method.
* - callback always has to be called even if error happened.
*
* @param {function(string, plugin.PluginResult)} callback - the result callback
*/
SoftwareGenerator.prototype.main = function (callback) {
// Use self to access core, project, result, logger etc from PluginBase.
// These are all instantiated at this point.
var self = this;
// Default fails
self.result.success = false;
// What did the user select for our configuration?
var currentConfig = self.getCurrentConfig();
self.compileCode = currentConfig.compile;
self.enforceConstraints = currentConfig.enforceConstraints;
self.forceIsolation = currentConfig.forceIsolation;
self.generateDocs = currentConfig.generate_docs;
self.returnZip = currentConfig.returnZip;
self.usePTY = currentConfig.usePTY;
// which architectures did the user ask us to compile on?
self.selectedArchitectures = [];
self.archPriorities = {};
self.archJobConfig = {};
Object.keys(currentConfig).map(function(k) {
if (k.indexOf("_COMPILATION_CONFIG") > -1) {
var arch = k.replace("_COMPILATION_CONFIG", "");
var archConfig = currentConfig[k];
// is the arch selected for compilation
var isSelected = archConfig.enabled;
if (isSelected) {
self.selectedArchitectures.push( arch );
}
// how many jobs should we spawn for this architecture?
var jobConfig = null;
try {
jobConfig = parseInt(archConfig.jobs);
} catch (err) {
}
if (jobConfig < 1 || !isFinite(jobConfig)) {
jobConfig = '\`nproc\`';
}
self.archJobConfig[ arch ] = jobConfig;
// how should we prioritize hosts?
var orderedHosts = archConfig.hostPriority;
if (Array.isArray(orderedHosts)) {
self.archPriorities[ arch ] = orderedHosts;
}
}
});
// make sure we don't try to compile if we don't have architectures
self.compileCode = self.selectedArchitectures.length && self.compileCode;
self.generateTypes = false;
self.runningOnClient = false;
if (typeof WebGMEGlobal !== 'undefined') {
self.runningOnClient = true;
if (self.compileCode)
self.notify('error', 'Cannot compile while running in client! Please re-run the plugin and enable server execution!');
if (self.generateDocs)
self.notify('error', 'Cannot generate documentation while running in client! Please re-run the plugin and enable server execution!');
self.compileCode = self.generateDocs = false;
}
// the active node for this plugin is software -> project
var projectNode = self.activeNode;
self.projectName = self.core.getAttribute(projectNode, 'name');
if (!self.runningOnClient) {
var path = require('path');
// Setting up variables that will be used by various functions of this plugin
self.gen_dir = path.join(process.cwd(),
'generated',
self.project.projectId,
self.branchName,
self.projectName);
}
self.projectModel = {}; // will be filled out by loadProjectModel (and associated functions)
self.artifacts = {}; // will be filled out and used by various parts of this plugin
self.artifactName = self.projectName + "+Software";
if (self.compileCode)
self.artifactName += '+Binaries';
if (self.generateDocs)
self.artifactName += '+Docs';
// set up libraries
webgmeToJson.notify = function(level, msg) {self.notify(level, msg);}
utils.notify = function(level, msg) {self.notify(level, msg);}
utils.trackedProcesses = ['catkin', 'rosmod_actor', 'roscore'];
webgmeToJson.loadModel(self.core, self.rootNode, projectNode, true, true) // resolve ptrs, keep webgme nodes
.then(function (projectModel) {
// hfsm stuff
self.generateHFSM(projectModel);
// regular stuff
processor.processModel(projectModel);
self.projectModel = projectModel.root;
self.projectObjects = projectModel.objects;
return self.generateArtifacts();
})
.then(function () {
return self.downloadLibraries();
})
.then(function () {
return self.copyDevelopmentLibraries();
})
.then(function () {
return self.runCompilation();
})
.then(function () {
return self.generateDocumentation();
})
.then(function () {
if (self.runningOnClient)
return self.returnSource();
else
return self.createZip();
})
.then(function () {
self.result.setSuccess(true);
callback(null, self.result);
})
.catch(function (err) {
if (typeof err === 'string')
self.notify('error', err);
self.result.setSuccess(false);
callback(err, self.result);
})
.done();
};
SoftwareGenerator.prototype.generateHFSM = function(projectModel) {
var self = this;
var hfsmSW = new hfsmSoftwareGenerator();
hfsmSW.notify = function(level, msg) {self.notify(level,msg);}
var originalRoot = projectModel.root;
hfsmSW.setProjectModel(projectModel);
function objToFilePrefix(obj) {
// object here will be a stateMachine
// get the package name from parent->parent (comp->package)
var filePrefix = null;
var comp = obj.parent;
var pkg = comp.parent;
if (comp.type == 'Component' && pkg.type == 'Package') {
var pkgName = pkg.name;
var compName = comp.name;
var prefix = 'src';
filePrefix = prefix + '/' + pkgName + '/include/' + pkgName + '/' + compName + '_HFSM/';
}
return filePrefix;
}
// add component includes to State Machines
Object.keys(projectModel.objects).map(function(k) {
var obj = projectModel.objects[k];
if (obj.type == 'State Machine') {
var comp = projectModel.objects[obj.parentPath];
var pkg = projectModel.objects[comp.parentPath];
if (comp.type == 'Component' && pkg.type == 'Package') {
var pkgName = pkg.name;
var compName = comp.name;
var includes = [
`#include "${pkgName}/${compName}.hpp"`,
`class ${compName};`,
''
].join('\n');
obj.Includes = includes + obj.Includes;
// make sure we have functions and pointers to the component itself
var declarations = [
`static ${compName}* this_component;`,
`void setComponentPtr( ${compName}* c ) { this_component = c; }`,
''
].join('\n');
obj.Declarations = declarations + obj.Declarations;
var definitions = [
`${compName}* StateMachine::${obj.sanitizedName}::this_component;`,
].join('\n');
obj.Definitions = definitions + obj.Definitions;
}
}
});
// now render the HFSM code
var hfsmArtifacts = hfsmSW.generateArtifacts(self.result, false, false, objToFilePrefix)
self.artifacts = Object.assign(self.artifacts, hfsmArtifacts);
projectModel.root = originalRoot;
};
SoftwareGenerator.prototype.generateArtifacts = function () {
var self = this,
prefix = 'src';
self.artifacts[self.projectModel.name + '_metadata.json'] = JSON.stringify({
projectID: self.project.projectId,
commitHash: self.commitHash,
branchName: self.branchName,
timeStamp: (new Date()).toISOString(),
pluginVersion: self.getVersion()
}, null, 2);
// render the doxygen template
var doxygenConfigName = 'doxygen_config',
doxygenTemplate = TEMPLATES[self.FILES['doxygen_config']];
self.artifacts[doxygenConfigName] = ejs.render(doxygenTemplate,
{'projectName': self.projectName});
var software_folder = self.projectModel.Software_list[0];
if (software_folder && software_folder.Package_list) {
software_folder.Package_list.map(function(pkgInfo) {
if (pkgInfo.Component_list) {
pkgInfo.generateComps = true;
pkgInfo.Component_list.map(function(compInfo) {
self.generateComponentFiles(prefix, pkgInfo, compInfo);
});
}
if (pkgInfo.Message_list) {
pkgInfo.generateTypes = true;
pkgInfo.Message_list.map(function(msgInfo) {
var msgFileName = [prefix,
pkgInfo.name,
'msg',
msgInfo.name + '.msg'].join('/');
self.artifacts[msgFileName] = msgInfo.Definition;
});
}
if (pkgInfo.Service_list) {
pkgInfo.generateTypes = true;
pkgInfo.Service_list.map(function(srvInfo) {
var srvFileName = [prefix,
pkgInfo.name,'srv',
srvInfo.name + '.srv'].join('/');
self.artifacts[srvFileName] = srvInfo.Definition;
});
}
if (pkgInfo.Action_list) {
pkgInfo.generateTypes = true;
pkgInfo.Action_list.map(function(actInfo) {
var actFileName = [prefix,
pkgInfo.name,'action',
actInfo.name + '.action'].join('/');
self.artifacts[actFileName] = actInfo.Definition;
});
}
var cmakeFileName = [prefix,
pkgInfo.name,
'CMakeLists.txt'].join('/'),
cmakeTemplate = TEMPLATES[self.FILES['cmakelists']];
self.artifacts[cmakeFileName] = ejs.render(cmakeTemplate, {
'pkgInfo':pkgInfo,
'model': self.projectModel,
'objects': self.projectObjects
});
var packageXMLFileName = [prefix,
pkgInfo.name,
'package.xml'].join('/'),
packageXMLTemplate = TEMPLATES[self.FILES['package_xml']];
self.artifacts[packageXMLFileName] = ejs.render(packageXMLTemplate, {
'pkgInfo': pkgInfo,
'model': self.projectModel,
'objects': self.projectObjects
});
});
}
var compile_script = [
'catkin config --extend ${HOST_WORKSPACE}',
'catkin config -i install',
'catkin config --install',
'catkin clean -b --yes',
'catkin build -j`nproc` --no-status',
'mkdir bin',
'cp devel/lib/*.so bin/.'
];
// save the compilation script
self.artifacts['compile_script.bash'] = compile_script.join('\n');
if ( self.runningOnClient ) {
self.notify('info', 'Generated code in client.');
return;
}
// put the files on the FS
var path = require('path'),
filendir = require('filendir'),
child_process = require('child_process');
// clear out any previous project files
self.notify('info','Clearing out previous binaries.');
self.selectedArchitectures.map(function(a) {
var binDir = utils.sanitizePath(path.join(self.gen_dir,'bin', a));
child_process.execSync('rm -rf ' + binDir);
});
var srcDir = utils.sanitizePath(path.join(self.gen_dir,'src'));
self.notify('info','Clearing out previous generated code.');
child_process.execSync('rm -rf ' + srcDir);
var fileNames = Object.keys(self.artifacts);
var tasks = fileNames.map(function(fileName) {
var deferred = Q.defer();
var data = self.artifacts[fileName];
filendir.writeFile(path.join(self.gen_dir, fileName), data, function(err) {
if (err) {
deferred.reject(err);
}
else {
deferred.resolve();
}
});
return deferred.promise;
});
return Q.all(tasks)
.then(function() {
var msg = 'Generated artifacts.';
self.notify('info', msg);
});
};
SoftwareGenerator.prototype.generateComponentFiles = function (prefix, pkgInfo, compInfo) {
var self = this;
var inclFileName = [prefix,
pkgInfo.name,
'include',
pkgInfo.name,
compInfo.name + '.hpp'].join('/'),
srcFileName = [prefix,
pkgInfo.name,
'src',
pkgInfo.name,
compInfo.name + '.cpp'].join('/'),
compCPPTemplate = TEMPLATES[this.FILES['component_cpp']],
compHPPTemplate = TEMPLATES[this.FILES['component_hpp']];
// render the header file
self.artifacts[inclFileName] = ejs.render(compHPPTemplate, {
'compInfo': compInfo,
'model': self.projectModel,
'objects': self.projectObjects
});
// re-render so any user-provided templates get rendered
self.artifacts[inclFileName] = ejs.render(self.artifacts[inclFileName], {
'compInfo': compInfo,
'model': self.projectModel,
'objects': self.projectObjects
});
// render the src file
self.artifacts[srcFileName] = ejs.render(compCPPTemplate, {
'compInfo': compInfo,
'model': self.projectModel,
'objects': self.projectObjects
});
// re-render so any user-provided templates get rendered
self.artifacts[srcFileName] = ejs.render(self.artifacts[srcFileName], {
'compInfo': compInfo,
'model': self.projectModel,
'objects': self.projectObjects
});
};
SoftwareGenerator.prototype.addMarkup = function(level, summary, details) {
var self = this,
Convert = require('ansi-to-html'),
convert = new Convert();
// ADD STDOUT / STDERR TO RESULTS AS HIDABLE TEXT
var msg = `<details><summary><b>${summary}</b></summary>` +
'<pre>' +
'<code>'+
convert.toHtml(details) +
'</code>'+
'</pre>' +
'</details>';
self.createMessage(self.activeNode, msg, level);
};
SoftwareGenerator.prototype.cloneLibrary = function(lib, dir) {
var self = this,
path = require('path'),
fs = require('fs'),
sanitizedDir = utils.sanitizePath(dir);
const { exec } = require('child_process');
self.notify('info', 'Cloning: ' + lib.name + ' from '+ lib.URL);
var options = {
shell: '/bin/bash',
};
var commands = [
`git clone --depth=1 --branch=${lib.Branch || 'master'} ${lib.URL} ${sanitizedDir}/${lib.name}`,
`rm -rf ${sanitizedDir}/${lib.name}/.git`
].join('\n');
var deferred = Q.defer();
var hasError = function(error, stdout, stderr) {
self.addMarkup('error', `STDERR cloning ${lib.name} from ${lib.URL}:`, stderr);
var errMsg = `Failed to clone ${lib.name} from ${lib.URL}: Make sure the URL is correct and the name does not conflict with any other libraries.`;
deferred.reject(errMsg);
};
exec(commands, options, function(error, stdout, stderr) {
if (error) {
hasError(error, stdout, stderr);
} else {
fs.access(path.join(dir, lib.name), (err) => {
if (err) {
hasError(error, stdout, stderr);
} else {
deferred.resolve();
}
});
}
});
return deferred.promise;
};
SoftwareGenerator.prototype.downloadLibraries = function ()
{
var self = this;
if (self.runningOnClient) {
self.notify('info', 'Skipping source library download in client mode.');
return;
}
var path = require('path'),
dir = path.join(self.gen_dir, 'src');
self.notify('info', 'Downloading Source Libraries');
var tasks = [];
if (self.projectModel.Software_list[0]['Source Library_list']) {
tasks = self.projectModel.Software_list[0]['Source Library_list'].map(function(lib) {
if (lib.Branch || lib.URL.indexOf(".zip") == -1) {
return self.cloneLibrary(lib, dir);
} else {
self.notify('info', 'Downloading: ' + lib.name + ' from '+ lib.URL);
return utils.wgetAndUnzipLibrary(lib.URL, dir);
}
});
}
return Q.all(tasks);
};
SoftwareGenerator.prototype.copyDevelopmentLibraries = function ()
{
var self = this;
if (self.runningOnClient) {
self.notify('info', 'Skipping development library copy in client mode.');
return;
}
var path = require('path'),
child_process = require('child_process'),
dir = utils.sanitizePath(path.join(self.gen_dir, 'src'));
var deferred = Q.defer();
if (self.projectModel.Software_list[0]['Development Library_list']) {
self.notify('info', 'Copying Development Libraries');
// make list of cp commands for the terminal
var commands = self.projectModel.Software_list[0]['Development Library_list'].map(function(lib) {
self.notify('info', 'Copying: ' + lib.name + ' from '+ lib.Directory);
var fromDir = utils.sanitizePath(lib.Directory);
return `cp -r ${fromDir} ${dir}/.`;
}).join("\n");
// set up stdout/stderr
var stdout = '',
stderr = '';
// execute commands
var terminal = child_process.spawn('bash', [], {cwd:self.gen_dir});
terminal.stdout.on('data', function (data) { stdout += data; });
terminal.stderr.on('data', function (data) { stderr += data; });
terminal.on('exit', function (code) {
if (code == 0) {
deferred.resolve(code);
}
else {
var files = {
'developmentlibraries.stdout.txt': stdout,
'developmentlibraries.stderr.txt': stderr
};
var fnames = Object.keys(files);
var tasks = fnames.map((fname) => {
return self.blobClient.putFile(fname, files[fname])
.then((hash) => {
self.result.addArtifact(hash);
});
});
return Q.all(tasks)
.then(() => {
deferred.reject('copying development libraries:: child process exited with code ' + code);
});
}
});
setTimeout(function() {
terminal.stdin.write(commands);
terminal.stdin.end();
}, 1000);
}
else {
deferred.resolve();
}
return deferred.promise;
};
SoftwareGenerator.prototype.generateDocumentation = function ()
{
var self = this;
if (!self.generateDocs || self.runningOnClient) {
var msg = 'Skipping documentation generation.'
self.notify('info', msg);
return;
}
var msg = 'Generating documentation.'
self.notify('info', msg);
var path = require('path'),
child_process = require('child_process');
var docPath = utils.sanitizePath(path.join(self.gen_dir, 'doc'));
// clear out any previous documentation
child_process.execSync('rm -rf ' + docPath);
var stdOut = '';
var stdErr = '';
var deferred = Q.defer();
var terminal = child_process.spawn('bash', [], {cwd:self.gen_dir});
terminal.stdout.on('data', function (data) { stdOut += data; });
terminal.stderr.on('data', function (data) { stdErr += data; });
terminal.on('exit', function (code) {
if (code == 0) {
deferred.resolve(code);
}
else {
var files = {
'documentation.stdout.txt': stdOut,
'documentation.stderr.txt': stdErr
};
var fnames = Object.keys(files);
var tasks = fnames.map((fname) => {
return self.blobClient.putFile(fname, files[fname])
.then((hash) => {
self.result.addArtifact(hash);
});
});
return Q.all(tasks)
.then(() => {
deferred.reject('document generation:: child process exited with code ' + code);
});
}
});
setTimeout(function() {
terminal.stdin.write('doxygen doxygen_config\n');
terminal.stdin.write('make -C ./doc/latex/ pdf\n');
terminal.stdin.write('mv ./doc/latex/refman.pdf ' +
utils.sanitizePath(self.projectName) + '.pdf');
terminal.stdin.end();
}, 1000);
return deferred.promise;
};
/* * * * * * * * * * * * * Constraint handling * * * * * * * * * * * */
SoftwareGenerator.prototype.mapPackagesToHosts = function (validHostList) {
var self = this;
var sw = self.projectModel.Software_list && self.projectModel.Software_list[0];
if (!sw) {
throw new String('Must have software!');
}
var package_list = sw.Package_list;
var host_list = validHostList; // is a dictionary of arch -> valid hosts
if (!package_list || !host_list) {
throw new String('Must have hosts and packages!');
}
var sortedPackages = [];
// figure out which packages have which constraints;
package_list.map(function(pkg) {
pkg.constraints = [];
// get all components in the package
if (pkg.Component_list) {
pkg.Component_list.map(function(comp) {
// get all constraints in the component
if (comp.Constraint_list) {
comp.Constraint_list.map(function(constraint) {
var deploymentOnly = constraint['Deployment Only'];
if (deploymentOnly === null || deploymentOnly === undefined) {
deploymentOnly = true;
}
if (!deploymentOnly && pkg.constraints.indexOf(constraint) == -1) {
pkg.constraints.push(constraint);
}
});
}
});
}
sortedPackages.push(pkg);
});
// Sort packages by decreasing number of constraints
// sort function : < 0 -> a < b ; = 0 -> a==b ; > 0 -> b < a
sortedPackages.sort(function(a,b) {
return b.constraints.length - a.constraints.length
});
// make sure they have compilePackages.
for (var arch in validHostList) {
var hl = validHostList[ arch ];
hl.map(function(h) {
h.compilePackages = sortedPackages;
});
};
if (!self.enforceConstraints) {
self.notify('info', 'User did not ask to enforce constraints at compile time.');
return;
}
self.notify('info', 'Mapping constraints in packages to hosts.');
// now figure out the hosts that meet the package's constraints
for (var arch in validHostList) {
var hl = validHostList[ arch ];
hl.map(function(h) {
var host = h.host;
var capabilities = host.Capability_list;
var validPackages = sortedPackages.filter(function(pkg) {
var constraints = pkg.constraints;
return self.capabilitiesMeetConstraints(capabilities, constraints);
});
h.compilePackages = validPackages;
});
};
};
SoftwareGenerator.prototype.capabilitiesMeetConstraints = function(capabilities, constraints) {
var self = this;
if (constraints.length == 0) {
return true;
}
if (constraints.length > 0 && capabilities == undefined) {
return false;
}
capabilities = capabilities.map((capability) => { return capability.name; });
for (var c in constraints) {
var constraint = constraints[c];
if (capabilities.indexOf(constraint.name) == -1) {
return false;
}
}
return true;
};
SoftwareGenerator.prototype.getValidArchitectures = function() {
var self = this,
validArchs = {};
var systems_folder = self.projectModel.Systems_list[0];
if (systems_folder && systems_folder.System_list) {
systems_folder.System_list.map(function(system) {
if (system.Host_list) {
system.Host_list.map(function(host) {
var devName = utils.getDeviceType(host);
if (self.selectedArchitectures.indexOf(devName) != -1) {
if (validArchs[devName] == undefined) {
validArchs[devName] = [];
}
validArchs[devName].push(host);
}
});
}
});
}
return validArchs;
};
SoftwareGenerator.prototype.selectCompilationArchitectures = function() {
var self = this;
var _ = require('underscore');
var validArchitectures = self.getValidArchitectures();
if (_.isEmpty(validArchitectures)) {
throw new String("No valid architectures found!");
}
var tasks = Object.keys(validArchitectures).map(function(index) {
return utils.getAvailableHosts(validArchitectures[index], self.forceIsolation)
.then(function(hostArr) {
var retObj = {};
retObj[index] = hostArr;
return retObj;
});
});
return Q.all(tasks)
.then(function (nestedArr) {
var validHosts = {};
var addresses = [];
nestedArr.forEach(function(subArr) {
var arch = Object.keys(subArr)[0];
var hostList = subArr[arch];
hostList.map((host) => {
if (addresses.indexOf(host.intf.IP) == -1) {
addresses.push(host.intf.IP);
}
else {
throw new String("Multiple hosts with different 'Architecture' and / or 'Device ID' have same IP and are reachable, IP: "+host.intf.IP);
}
});
validHosts[arch] = subArr[arch];
});
return validHosts;
});
};
SoftwareGenerator.prototype.getObjectAttributeFromBuild = function (fileName, fileLineNumber) {
var self = this;
// find correct file
var fileData = self.artifacts['src/'+fileName];
if (fileData) {
// split the file string into line string array
var fileLines = fileData.split("\n");
// use line number from error to start working our way back using regex to find obj.attr
var regex = /\/\/::::([a-zA-Z0-9\/]*)::::([^:\s]*)::::(?:end::::)?/gi;
var path, attr, attrLineNumber;
for (var l=fileLineNumber; l>0; l--) {
var line = fileLines[l];
var result = regex.exec(line);
if (result) {
path = result[1];
attr = result[2];
attrLineNumber = fileLineNumber - l -1;
break;
}
}
var node = self.projectObjects[path];
return {node: node, attr: attr, lineNumber: attrLineNumber};
}
else {
return {node: null, attr: null, lineNumber: null};
}
};
SoftwareGenerator.prototype.compileHasError = function(data) {
return data.indexOf('Errors << ') > -1 ||
data.indexOf('Traceback (most recent call last):') > -1 ||
data.indexOf('error:') > -1;
};
SoftwareGenerator.prototype.convertCompileMessage = function(host, data) {
var self = this;
var path = require('path');
var stripANSI = require('strip-ansi');
var base_compile_dir = path.join(host.user.Directory, 'compilation');
var compile_dir = path.join(base_compile_dir, self.project.projectId, self.branchName);
var removeDir = path.join(compile_dir, 'src/');
var strippedData = stripANSI(data);
function getParentByType(path, type) {
var o = self.projectObjects[path];
if (o) {
if (o.type != type)
o = getParentByType(o.parentPath, type);
} else {
return { name: null };
}
return o;
}
if (strippedData.indexOf('error:') > -1)
{
var compileErrors = utils.parseMakeErrorOutput(strippedData);
compileErrors.map(function(compileError) {
var baseName = compileError.fileName.replace(removeDir, '');
var nodeInfo = self.getObjectAttributeFromBuild(baseName, compileError.line);
var node = nodeInfo.node,
attr = nodeInfo.attr,
lineNum = nodeInfo.lineNumber;
if (node) {
var nodeName = node.name;
var packageName = getParentByType(node.path, 'Package').name;
var compName = getParentByType(node.path, 'Component').name;
var hfsmName = getParentByType(node.path, 'State Machine').name;
self.notify('error',
'Error in Package: ' + packageName +
', Component: ' + compName + (hfsmName ? ", HFSM: " + hfsmName : "") + ', attribute: ' + attr +
', at line: ' + lineNum,
node.node
);
var msg = '<details><summary><b>Build Error:: package: ' + packageName + ', component: ' +
compName + (hfsmName ? ", HFSM: " + hfsmName : "") +':</b></summary>' +
'<pre><code>'+
compileError.text +
'</code></pre>'+
'</details>';
self.createMessage(node.node, msg, 'error');
}
else {
var packageName = baseName.split('/')[0];
var compName = path.basename(compileError.fileName).split('.')[0];
var msg = '<details><summary><b>Build Error:: Library: ' + packageName +':</b></summary>' +
'<pre><code>'+
compileError.text +
'</code></pre>'+
'</details>';
self.createMessage(self.activeNode, msg, 'error');
}
});
}
else if (strippedData.indexOf('warning:') > -1)
{
var compileErrors = utils.parseMakeErrorOutput(strippedData);
compileErrors.map(function(compileError) {
var baseName = compileError.fileName.replace(removeDir, '');
var nodeInfo = self.getObjectAttributeFromBuild(baseName, compileError.line);
var node = nodeInfo.node,
attr = nodeInfo.attr,
lineNum = nodeInfo.lineNumber;
if (node) {
var nodeName = node.name;
var packageName = getParentByType(node.path, 'Package').name;
var compName = getParentByType(node.path, 'Component').name;
var hfsmName = getParentByType(node.path, 'State Machine').name;
self.notify('error',
'Warning in Package: ' + packageName +
', Component: ' + compName + (hfsmName ? ", HFSM: " + hfsmName : "") + ', attribute: ' + attr +
', at line: ' + lineNum,
node.node
);
var msg = '<details><summary><b>Build Warning:: package: ' + packageName + ', component: ' +
compName + (hfsmName ? ", HFSM: " + hfsmName : "") +':</b></summary>' +
'<pre><code>'+
compileError.text +
'</code></pre>'+
'</details>';
self.createMessage(node.node, msg, 'error');
}
else {
var packageName = baseName.split('/')[0];
var compName = path.basename(compileError.fileName).split('.')[0];
var msg = '<details><summary><b>Build Warning:: Library: ' + packageName +':</b></summary>' +
'<pre><code>'+
compileError.text +
'</code></pre>'+
'</details>';
self.createMessage(self.activeNode, msg, 'error');
}
});
}
};
SoftwareGenerator.prototype.parseCompileError = function(host, data) {
var self = this;
var path = require('path');
var stripANSI = require('strip-ansi');
var base_compile_dir = path.join(host.user.Directory, 'compilation');
var compile_dir = path.join(base_compile_dir, self.project.projectId, self.branchName);
var removeDir = path.join(compile_dir, 'src/')
var strippedData = stripANSI(data);
host.stdErr += data;
if (host.hasError || self.compileHasError(strippedData)) {
host.hasError = true;
// see if it's a make error
if ( strippedData.indexOf('error:') > -1 ) {
self.convertCompileMessage(host, data);
//return true;
}
else if ( strippedData.indexOf('Traceback (most recent call last):') > -1) {
//return true;
}
}
return false;
};
SoftwareGenerator.prototype.parseCompileData = function(host, data) {
var self = this;
var stripANSI = require('strip-ansi');
var strippedData = stripANSI(data);
if (host.hasError || self.compileHasError(strippedData)) {
return self.parseCompileError(host, data);
}
else {
host.stdOut += data;
// see if it's a make warning
if ( strippedData.indexOf('warning:') > -1 ) {
self.convertCompileMessage(host, data);
}
else {
// see if we can parse percent from it
self.parseCompilePercent(host, strippedData);
}
}
return false;
};
SoftwareGenerator.prototype.parseCompilePercent = function (host, data) {
var self = this;
var percent = utils.parseMakePercentOutput(data);
if (host.__lastPercent === undefined)
host.__lastPercent = 0;
if (percent.length && percent[0] > 0 && (percent[0] - host.__lastPercent) > 4 ) {
var msg = 'Build on ' + utils.getDeviceType(host.host) + ': ' + percent[0] + '%';
host.__lastPercent = percent[0];
self.notify('info', msg);
}
};
SoftwareGenerator.prototype.compileOnHost = function (host) {
var self = this;
var path = require('path');
var mkdirp = require('mkdirp');
var child_process = require('child_process');
var base_compile_dir = path.join(host.user.Directory, 'compilation');
var compile_dir = path.join(base_compile_dir, self.project.projectId, self.branchName);
var archBinPath = path.join(self.gen_dir, 'bin' , utils.getDeviceType(host.host));
// install folder for storing package.xmls (generated above) and the msg/srv deserialization
var installPath = path.join(self.gen_dir, 'install');
var buildCommand = 'catkin build -j'+ self.archJobConfig[utils.getDeviceType(host.host)] +' --no-status '+
host.compilePackages.map(function(p) { return p.name; }).join(' ');
var compile_commands = [
'cd ' + utils.sanitizePath(compile_dir),
'rm -rf bin',
'rm -rf install',
'catkin config --extend ' + host.host['Build Workspace'],
'catkin config -i install',
'catkin config --install',
'catkin clean -b --yes',
'mkdir bin',
buildCommand
];
var hasComps = host.compilePackages.filter(function(p) { return p.generateComps == true; }).length > 0;
if (hasComps) {
// only do this if we have generated components (and will therefore generate binaries
compile_commands.push('cp devel/lib/*.so bin/.');
}
// save the compilation script
self.artifacts['compile-'+host.intf.IP+'.bash'] = compile_commands.join('\n');
var compilationFailed = false;
child_process.execSync('rm -rf ' + utils.sanitizePath(archBinPath));
// make the compile dir
return new Promise(function(resolve,reject) {
self.notify('info', 'making compilation directory on: ' + host.intf.IP);
utils.mkdirRemote(compile_dir, host.intf.IP, host.user)
.then(function() {
resolve();
})
.catch(function() {
reject("Couldn't make remote compilation dir!");
});
})
.then(function() {
// copy the sources to remote
self.notify('info', 'copying compilation sources to: ' + host.intf.IP);
var srcDir = path.join(self.gen_dir,'src');
return utils.copyToHost(srcDir, compile_dir +'/.', host.intf.IP, host.user);
})
.then(function() {
// run the compile step
self.notify('info', 'compiling on: ' + host.intf.IP + ' into '+compile_dir);
self.notify('info', ' : ' + buildCommand);
host.hasError = false;
host.stdErr = '';
host.stdOut = '';
var compileDataCallback = function(host) {
return function(data) {
return self.parseCompileData(host, data);
};
}(host);
var compileErrorCallback = function(host) {
return function(data) {
return self.parseCompileError(host, data);
};
}(host);
return utils.executeOnHost(compile_commands,
host.intf.IP,
host.user,
compileErrorCallback,
compileDataCallback,
self.usePTY)
.catch(function(err) {
compilationFailed = true;
})
.finally(function() {
self.addMarkup('error', `Compile STDOUT from ${host.intf.IP}:`, host.stdOut);
self.addMarkup('error', `Compile STDERR from ${host.intf.IP}:`, host.stdErr);
// ADD STDOUT / STDERR TO RESULTS AS BLOBS
var files = {};
var stripANSI = require('strip-ansi');
files[ host.intf.IP + '.compile.stdout.txt' ] = stripANSI(host.stdOut);
files[ host.intf.IP + '.compile.stderr.txt' ] = stripANSI(host.stdErr);
var fnames = Object.keys(files);
var tasks = fnames.map((fname) => {
return self.blobClient.putFile(fname, files[fname])
.then((hash) => {
self.result.addArtifact(hash);
});
});
return Q.all(tasks);
})
})
.then(function(output) {
if (host.hasError || output == undefined || output.returnCode != 0)
compilationFailed = true;
if (compilationFailed) {
throw new String('Compilation failed on ' + host.intf.IP);
}
})
.then(function() {
if (hasComps) {
// make the local binary folder for the architecture
mkdirp.sync(archBinPath);
// copy the compiled binaries from remote into the local bin folder
self.notify('info', 'copying binaries from ' + host.intf.IP + ' into local storage.');
return utils.copyFromHost(path.join(compile_dir, 'bin') + '/*',
archBinPath + '/.',
host.intf.IP,
host.user);
}
})
.then(function() {
// copy the message/service deserialization generated as part of the build
self.notify('info', 'copying definitions from ' + host.intf.IP + ' into local storage.');
return utils.copyFromHost(path.join(compile_dir, 'install') + '/*',
installPath + '/.',
host.intf.IP,
host.user);
})
.then(function() {
// remove the remote folders
return self.cleanHost(host);
});
};
SoftwareGenerator.prototype.cleanHost = function(host) {
var self = this;
var path = require('path');
var base_compile_dir = utils.sanitizePath(path.join(host.user.Directory, 'compilation'));
self.notify('info', 'removing compilation artifacts off: ' + host.intf.IP);
return utils.executeOnHost([
'pkill catkin',
'rm -rf ' + base_compile_dir,
], host.intf.IP, host.user);
};
SoftwareGenerator.prototype.runCompilation = function ()
{
var self = this;
if (!self.compileCode || self.runningOnClient) {
var msg = 'Skipping compilation.';
self.notify('info', msg);
return;
}
return self.selectCompilationArchitectures()
.then(function(validHostList) {
return self.compileBinaries(validHostList);
});
};
SoftwareGenerator.prototype.compileBinaries = function (validHostList)
{
var self = this;
var selectedHosts = [];
var path = require('path');
var mkdirp = require('mkdirp');
var child_process = require('child_process');
// install folder for storing package.xmls (generated above) and the msg/srv deserialization
var installPath = path.join(self.gen_dir, 'install');
child_process.execSync('rm -rf ' + utils.sanitizePath(installPath));
mkdirp.sync(installPath);
self.mapPackagesToHosts( validHostList );
for (var arch in validHostList) {
var hosts = validHostList[arch];
if (hosts.length) {
// sort hosts based on:
// * which host can compile the most packages
// * user-provided compilation priority
hosts.sort(function(a,b) {
var pkgCmp = (b.compilePackages.length || 0) - (a.compilePackages.length || 0);
var akey = `${a.host.path}::${a.host.name}`;
var bkey = `${b.host.path}::${b.host.name}`;
var priCmp = (