jsharmony-cms
Version:
1,002 lines (896 loc) • 157 kB
JavaScript
/*
Copyright 2019 apHarmony
This file is part of jsHarmony.
jsHarmony is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
jsHarmony is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this package. If not, see <http://www.gnu.org/licenses/>.
*/
var Helper = require('jsharmony/Helper');
var HelperFS = require('jsharmony/HelperFS');
var _ = require('lodash');
var urlparser = require('url');
var path = require('path');
var ejs = require('ejs');
var fs = require('fs');
var async = require('async');
var crypto = require('crypto');
var wclib = require('jsharmony/WebConnect');
var yazl = require('yazl');
var baseModule = module;
module.exports = exports = function(module, funcs){
var exports = {};
var _t = module._t;
funcs.deploymentQueue = async.queue(function (op, done){
var jsh = module.jsh;
if(!op) return done();
if(op.exec == 'deployment'){
//{ exec: 'deployment', deployment_id: deployment_id }
funcs.deploy_exec(op.deployment_id, done);
}
else if(op.exec == 'deployment_download'){
//{ exec: 'deployment_download', deployment_id: deployment_id, dstStream: res, onStart: function(){} }
funcs.deployment_download(op.deployment_id, op.dstStream, op.onStart, op.options, done);
}
else{
var err = new Error('Invalid deployment exec operation: '+op.exec);
jsh.Log.error(err);
return done(err);
}
}, 1);
exports.deployment_getLogFileName = function (deployment_id) {
return path.join(module.jsh.Config.datadir, 'publish_log', deployment_id + '.log');
};
exports.deployment_getChangeLogFileName = function (deployment_id) {
return path.join(module.jsh.Config.datadir, 'publish_log', deployment_id + '.changes.log');
};
exports.deploy_log = function (deployment_id, txt, logtype){
var jsh = module.jsh;
jsh.Log[logtype](txt);
var logfile = funcs.deployment_getLogFileName(deployment_id);
jsh.Log[logtype](logtype.toUpperCase() + ' ' + txt, { logfile: logfile });
};
exports.deploy_log_error = function (deployment_id, txt){
funcs.deploy_log(deployment_id, 'Publish Failed: '+(txt||'').toString(), 'error');
};
exports.deploy_log_info = function (deployment_id, txt){
funcs.deploy_log(deployment_id, txt, 'info');
};
exports.deploy_log_change = function (deployment_id, txt){
var jsh = module.jsh;
if(!jsh.Config.logdir) return;
var logfile = funcs.deployment_getChangeLogFileName(deployment_id);
jsh.Log.info(txt, { logfile: logfile });
};
exports.deploy = function (deployment_id, onComplete) {
funcs.deploymentQueue.push({ exec: 'deployment', deployment_id: deployment_id }, function(err){
if(onComplete) return onComplete();
});
};
exports.deploy_waitForLog = function(deployment_id, onComplete){
var jsh = module.jsh;
var logQueue = jsh.Log.getQueue();
var logfile = funcs.deployment_getLogFileName(deployment_id);
var waiting = false;
for(var i=0;i<logQueue.length;i++){
var log = logQueue[i];
if(log && logfile && (log.logfile == logfile)){ waiting = true; break; }
}
if(waiting){
if(jsh.Config.interactive) jsh.Log.flush();
setTimeout(function(){ exports.deploy_waitForLog(deployment_id, onComplete); }, 50);
return;
}
if(onComplete) onComplete();
};
exports.getPageRelativePath = function(page, publish_params){
var page_fpath = page.page_path||'';
if(!page_fpath) return '';
while(page_fpath.substr(0,1)=='/') page_fpath = page_fpath.substr(1);
var is_folder = (page_fpath[page_fpath.length-1]=='/');
if(is_folder) page_fpath += publish_params.site_default_page_filename;
if(path.isAbsolute(page_fpath)) throw new Error('Page path:'+page.page_path+' cannot be absolute');
if(page_fpath.indexOf('..') >= 0) throw new Error('Page path:'+page.page_path+' cannot contain directory traversals');
return page_fpath;
};
exports.getMediaRelativePath = function(media, publish_params, thumbnail_id, thumbnail_config){
var media_fpath = media.media_path||'';
if(!media_fpath) return '';
while(media_fpath.substr(0,1)=='/') media_fpath = media_fpath.substr(1);
if(thumbnail_id){
var lastDot = media_fpath.lastIndexOf('.');
var lastSlash = Math.max(media_fpath.lastIndexOf('/'),media_fpath.lastIndexOf('\\'));
if((lastDot < 0) || (lastDot < lastSlash)) media_fpath += '.'+thumbnail_id;
else {
media_fpath = media_fpath.substr(0, lastDot) +
'.' + thumbnail_id +
((thumbnail_config && thumbnail_config.format) ? '.' + thumbnail_config.format : media_fpath.substr(lastDot));
}
}
var is_folder = (media_fpath[media_fpath.length-1]=='/');
if(is_folder) throw new Error('Media path:'+media.media_path+' cannot be a folder');
if(path.isAbsolute(media_fpath)) throw new Error('Media path:'+media.media_path+' cannot be absolute');
if(media_fpath.indexOf('..') >= 0) throw new Error('Media path:'+media.media_path+' cannot contain directory traversals');
if(media_fpath.indexOf('./') >= 0) throw new Error('Media path:'+media.media_path+' cannot contain directory traversals');
return media_fpath;
};
exports.getMediaThumbnails = function(url, branchData){
if(!branchData || !branchData.media_items || !branchData.site_config || !branchData.site_config.media_thumbnails) return {};
if(!url || (url.indexOf('#@JSHCMS') < 0)) return {};
var urlparts = null;
try{
urlparts = urlparser.parse(url, true);
}
catch(ex){
return {};
}
if(!urlparts.pathname) return {};
var patharr = (urlparts.pathname||'').split('/');
var rslt = {};
if((urlparts.pathname.indexOf('/_funcs/media/')==0) && (patharr.length>=4)){
var media_key = patharr[3];
if(!(media_key in branchData.media_items)) return {};
var media = branchData.media_items[media_key];
for(var thumbnail_id in branchData.site_config.media_thumbnails){
var thumbnail_config = branchData.site_config.media_thumbnails[thumbnail_id];
if(!thumbnail_config || !thumbnail_config.export) continue;
if(!_.includes(['.jpg','.jpeg','.tif','.tiff','.png','.gif','.svg'], media.media_ext)) continue;
if((patharr.length >= 5) && patharr[4]) patharr[4] = thumbnail_id;
else patharr.splice(4,0,thumbnail_id);
rslt[thumbnail_id] = urlparts.protocol + '//' + urlparts.host + patharr.join('/') + (urlparts.search||'') + (urlparts.hash||'');
}
}
return rslt;
};
exports.downloadLocalTemplates = function(branchData, templates, template_html, options, download_cb){
options = _.extend({ templateType: 'PAGE', exportTemplates: {} }, options);
var jsh = module.jsh;
var sitePath = path.join(path.join(jsh.Config.datadir,'site'),(branchData.site_id||'').toString());
async.eachOf(templates, function(template, template_name, template_cb){
if(template.location != 'LOCAL') return template_cb();
if(!template.path) return template_cb();
async.waterfall([
function(template_action_cb){
var templatePath = path.join(sitePath, template.path);
fs.readFile(templatePath, 'utf8', function(err, templateContent){
if (HelperFS.fileNotFound(err)) return template_action_cb(new Error('Error downloading template - local template file not found: '+template.path));
if(err) return template_action_cb(new Error('Error downloading template: '+err.toString()));
//Parse and merge template config
var templateConfig = null;
try{
if(options.templateType == 'PAGE') templateConfig = funcs.readPageTemplateConfig(templateContent, 'local page template "'+template.path+'"', { templateName: template_name });
else if(options.templateType == 'COMPONENT') templateConfig = funcs.readComponentTemplateConfig(templateContent, 'local component template "'+template.path+'"');
else throw new Error('Invalid Template Type: ' + options.templateType);
}
catch(ex){
return template_action_cb(ex);
}
async.parallel([
//Download publish template, if necessary
function(template_publish_cb){
async.waterfall([
//Check publish template
function(template_process_cb){
if(templateConfig && templateConfig.remote_templates && templateConfig.remote_templates.publish){
templateConfig.remote_templates.publish = funcs.parseDeploymentUrl(templateConfig.remote_templates.publish, branchData.template_variables);
//If path is remote
if(templateConfig.remote_templates.publish.indexOf('//') >= 0) return template_process_cb();
//If path is local
var publishTemplatePath = path.normalize(path.join(path.dirname(templatePath), templateConfig.remote_templates.publish));
if(publishTemplatePath.indexOf(path.normalize(sitePath)+path.sep) != 0) return template_process_cb(new Error('Invalid remote_templates.publish path: '+templateConfig.remote_templates.publish));
//Download local publish template
fs.readFile(publishTemplatePath, 'utf8', function(err, publishTemplateContent){
if (HelperFS.fileNotFound(err)) return template_process_cb(new Error('Error downloading publish template - publish template file not found: '+templateConfig.remote_templates.publish));
if(err) return template_process_cb(new Error('Error downloading publish template: '+err.toString()));
//Add publish template to template_html
template_html[template_name] = publishTemplateContent;
return template_process_cb();
});
}
else return template_process_cb();
},
//If no publish template, add to template_html
function(template_process_cb){
//Components already merged the config and post-processed it in getComponentTemplates
if(options.templateType != 'COMPONENT') _.merge(template, templateConfig);
if(!(template.remote_templates && template.remote_templates.publish)){
try{
if(options.templateType == 'PAGE') templateContent = funcs.generateDeploymentTemplate(template, templateContent, { template_variables: branchData.template_variables, template_name: template_name });
}
catch(ex){
return template_publish_cb(new Error('Could not parse "'+template_name+'" '+options.templateType.toLowerCase()+' template: '+ex.toString()));
}
template_html[template_name] = templateContent;
}
return template_process_cb();
}
], template_publish_cb);
},
//Download export templates
function(template_publish_cb){
if(options.templateType != 'COMPONENT') return template_publish_cb();
async.eachOf(templateConfig.export, function(exportItem, exportIndex, export_cb){
var exportDesc = '#'+(exportIndex+1).toString();
if(exportItem.export_path) exportDesc = '"' + exportItem.export_path + '"';
var exportErrorPrefix = 'Error in "'+template_name+'" '+options.templateType.toLowerCase()+', export '+exportDesc+' - ';
if(!(template_name in options.exportTemplates)) options.exportTemplates[template_name] = {};
Helper.execif(exportItem.remote_template,
//Initialize remote template
function(done){
exportItem.remote_template = funcs.parseDeploymentUrl(exportItem.remote_template, branchData.template_variables);
//If path is remote
if(exportItem.remote_template.indexOf('//') >= 0) return done();
//If path is local
var exportTemplatePath = path.normalize(path.join(path.dirname(templatePath), exportItem.remote_template));
if(exportTemplatePath.indexOf(path.normalize(sitePath)+path.sep) != 0) return export_cb(new Error(exportErrorPrefix + 'Invalid remote_template path: '+exportItem.remote_template));
//Download local export template
fs.readFile(exportTemplatePath, 'utf8', function(err, exportTemplateContent){
if (HelperFS.fileNotFound(err)) return export_cb(new Error(exportErrorPrefix + 'File not found: '+exportItem.remote_template));
if(err) return export_cb(new Error(exportErrorPrefix + 'Could not download remote_template: '+err.toString()));
//Add export template to exportTemplates
options.exportTemplates[template_name][exportIndex] = exportTemplateContent;
return done();
});
},
//If no publish template, add current template to exportTemplates
function(){
if(!(exportItem.remote_template)){
options.exportTemplates[template_name][exportIndex] = templateContent;
}
return export_cb();
}
);
}, template_publish_cb);
},
], template_action_cb);
});
},
], template_cb);
}, download_cb);
};
exports.downloadRemoteTemplates = function(branchData, templates, template_html, options, download_cb){
options = _.extend({ templateType: 'PAGE', exportTemplates: {}, addWebRequest: null }, options);
var jsh = module.jsh;
async.eachOf(templates, function(template, template_name, template_cb){
if(template.location != 'REMOTE') return template_cb();
if(!template.remote_templates) return template_cb();
async.waterfall([
//Download template.remote_templates.publish or template.remote_templates.editor
function(template_action_cb){
var url = '';
var isPublishTemplate = false;
if(template.remote_templates.publish){
url = funcs.parseDeploymentUrl(template.remote_templates.publish, branchData.template_variables);
isPublishTemplate = true;
}
else if(template.remote_templates.editor){
url = funcs.parseDeploymentUrl(template.remote_templates.editor, branchData.template_variables);
}
else return template_action_cb();
//Add cache-busting timestamp and page_template_id to URL
try{
var parsedUrl = new urlparser.URL(url);
if((options.templateType=='PAGE') && !parsedUrl.searchParams.has('page_template_id')){
parsedUrl.searchParams.set('page_template_id', template_name);
url = parsedUrl.toString();
}
if(!parsedUrl.searchParams.has('_')){
parsedUrl.searchParams.set('_', (Date.now()).toString());
url = parsedUrl.toString();
}
}
catch(ex){
if(branchData.publish_params) funcs.deploy_log_info(branchData.publish_params.deployment_id, 'Downloading remote template: '+url);
else jsh.Log.info('Downloading remote template: '+url);
return template_action_cb(ex);
}
if(branchData.publish_params) funcs.deploy_log_info(branchData.publish_params.deployment_id, 'Downloading remote template: '+url);
else jsh.Log.info('Downloading remote template: '+url);
options.addWebRequest(url, function(err, res, templateContent, req_cb){
if(err) return req_cb(err);
if(res && res.statusCode){
if(res.statusCode > 500) return req_cb(new Error(res.statusCode+' Error downloading template '+url));
if(res.statusCode > 400) return req_cb(new Error(res.statusCode+' Error downloading template '+url));
}
//Parse and merge template config
var templateConfig = null;
try{
if(options.templateType == 'PAGE') templateConfig = funcs.readPageTemplateConfig(templateContent, 'remote page template "'+url + '"', { templateName: template_name });
else if(options.templateType == 'COMPONENT') templateConfig = funcs.readComponentTemplateConfig(templateContent, 'remote component template "'+url+'"');
else throw new Error('Invalid Template Type: ' + options.templateType);
}
catch(ex){
return req_cb(ex);
}
if(templateConfig && templateConfig.remote_templates && templateConfig.remote_templates.publish){
templateConfig.remote_templates.publish = funcs.parseDeploymentUrl(templateConfig.remote_templates.publish, branchData.template_variables, url);
}
_.merge(template, templateConfig);
if(isPublishTemplate){
template_html[template_name] = templateContent;
}
else if(template.templates && ('publish' in template.templates)){
template_html[template_name] = '';
}
else if(!template.remote_templates.publish){
try{
if(options.templateType == 'PAGE') templateContent = funcs.generateDeploymentTemplate(template, templateContent, { template_variables: branchData.template_variables, template_name: template_name });
}
catch(ex){
return req_cb(new Error('Could not parse "'+template_name+'" '+options.templateType.toLowerCase()+' template: '+ex.toString()));
}
template_html[template_name] = templateContent;
}
//Parse URLs for export templates
_.each((options.templateType == 'COMPONENT') && template.export, function(exportItem, exportIndex){
if(!(template_name in options.exportTemplates)) options.exportTemplates[template_name] = {};
if(exportItem.remote_template){
exportItem.remote_template = funcs.parseDeploymentUrl(exportItem.remote_template, branchData.template_variables, url);
}
if(!exportItem.remote_template){
options.exportTemplates[template_name][exportIndex] = templateContent;
}
});
return req_cb();
});
return template_action_cb();
},
], template_cb);
}, download_cb);
};
exports.downloadPublishTemplates = function(branchData, templates, template_html, options, download_cb){
options = _.extend({ templateType: 'PAGE', exportTemplates: {}, addWebRequest: null }, options);
async.eachOf(templates, function(template, template_name, template_cb){
async.parallel([
//Download standard templates
function(template_download_cb){
async.waterfall([
//Download template.remote_templates.publish (for page, component)
function(template_action_cb){
if(template_name in template_html) return template_action_cb(); //Already downloaded
if(!template.remote_templates || !template.remote_templates.publish) return template_action_cb();
var url = funcs.parseDeploymentUrl(template.remote_templates.publish, _.extend({ page_template_id: template_name }, branchData.template_variables));
funcs.deploy_log_info(branchData.publish_params.deployment_id, 'Downloading template: '+url);
options.addWebRequest(url, function(err, res, rslt, req_cb){
if(err) return req_cb(err);
if(res && res.statusCode){
if(res.statusCode > 500) return req_cb(new Error(res.statusCode+' Error downloading template '+url));
if(res.statusCode > 400) return req_cb(new Error(res.statusCode+' Error downloading template '+url));
}
template_html[template_name] = rslt;
return req_cb();
});
return template_action_cb();
},
//Add hard-coded templates to result
function(template_action_cb){
if(template.templates && ('publish' in template.templates)){
//Clear editor templates, if they were used
if(!(template.remote_templates && template.remote_templates.publish)) template_html[template_name] = '';
//Prepend hard-coded template
template_html[template_name] = template.templates.publish + (template_html[template_name]||'');
}
return template_action_cb();
}
], template_download_cb);
},
//Download component export templates
function(template_download_cb){
if(options.templateType != 'COMPONENT') return template_download_cb();
async.eachOf(template.export, function(exportItem, exportIndex, export_cb){
if(!(template_name in options.exportTemplates)) options.exportTemplates[template_name] = {};
async.waterfall([
//Download remote_template
function(template_action_cb){
if(exportIndex in options.exportTemplates[template_name]) return template_action_cb(); //Already downloaded
if(!exportItem.remote_template) return template_action_cb();
var url = funcs.parseDeploymentUrl(exportItem.remote_template, branchData.template_variables);
funcs.deploy_log_info(branchData.publish_params.deployment_id, 'Downloading template: '+url);
options.addWebRequest(url, function(err, res, rslt, req_cb){
if(err) return req_cb(err);
if(res && res.statusCode){
if(res.statusCode > 500) return req_cb(new Error(res.statusCode+' Error downloading template '+url));
if(res.statusCode > 400) return req_cb(new Error(res.statusCode+' Error downloading template '+url));
}
options.exportTemplates[template_name][exportIndex] = rslt;
return req_cb();
});
return template_action_cb();
},
//Add hard-coded templates to result
function(template_action_cb){
if(exportItem.template){
//Clear editor templates, if they were used
if(!exportItem.remote_template) options.exportTemplates[template_name][exportIndex] = '';
//Prepend hard-coded template
options.exportTemplates[template_name][exportIndex] = exportItem.template + (options.exportTemplates[template_name][exportIndex]||'');
}
return template_action_cb();
}
], export_cb);
}, template_download_cb);
},
], template_cb);
}, download_cb);
};
exports.getDeploymentSortedBranchItemTypes = function(){
var cms = module;
var branchItemTypes = _.keys(cms.BranchItems);
branchItemTypes.sort(function(a,b){
var aseq = (cms.BranchItems[a] && cms.BranchItems[a].deploy && cms.BranchItems[a].deploy.onDeploy_seq) || 0;
var bseq = (cms.BranchItems[b] && cms.BranchItems[b].deploy && cms.BranchItems[b].deploy.onDeploy_seq) || 0;
if(aseq > bseq) return 1;
if(bseq > aseq) return -1;
return 0;
});
return branchItemTypes;
};
//Shell Commands
exports.shellExec = function(cmd, default_cwd, params, cb, exec_options){
var rslt = '';
var stderr = '';
var returned = false;
var orig_cb = cb;
cb = function(err, rslt, stderr){
if(returned) return;
returned = true;
return orig_cb(err, rslt, stderr);
};
exec_options = _.extend({ cwd: default_cwd }, exec_options);
wclib.xlib.exec(cmd, params, function(err){ //cb
if(err) return cb(err, rslt.trim(), stderr.trim());
return cb(null,rslt.trim(), stderr.trim());
}, function(data){ //stdout
rslt += data.toString();
}, function(data){ //stderr
stderr += data.toString();
}, function(err){ //onError
return cb(err, rslt.trim(), stderr.trim());
}, exec_options);
};
//Git Commands
exports.gitExec = function(repo_path, params, cb, exec_options){
var cms = module;
var git_path = cms.Config.git.bin_path || 'git';
return funcs.shellExec(git_path, repo_path, params, function(err, rslt, stderr){
if(err){
if(err && (err.code == 'ENOENT')) err = new Error('Git executable not found using command: "'+git_path+'". Please install git, or set the git.bin_path config parameter');
return cb(new Error('Git Error: ' + err.toString() + ' ' + (stderr||'')), rslt);
}
return cb(err, rslt);
}, exec_options);
};
exports.gitExecVerbose = function(deployment_id, repo_path, params, cb, exec_options){
funcs.deploy_log_info(deployment_id, 'git '+(params||[]).join(' '));
return exports.gitExec(repo_path, params, cb, exec_options);
};
exports.deploy_exec = function (deployment_id, onComplete) {
if(!onComplete) onComplete = function(){};
var jsh = module.jsh;
var appsrv = jsh.AppSrv;
var dbtypes = appsrv.DB.types;
var cms = module;
//Update deployment to running status
var sql = "select \
deployment_id, dt.site_id, deployment_tag, deployment_params, deployment_target_name, deployment_target_publish_path, deployment_target_template_variables, deployment_target_publish_config, deployment_target_sts, deployment_git_revision, \
d.deployment_target_id, \
(select param_cur_val from jsharmony.v_param_cur where param_cur_process='CMS' and param_cur_attrib='PUBLISH_TGT') publish_tgt, \
site.site_default_page_filename site_default_page_filename \
from "+(module.schema?module.schema+'.':'')+'deployment d \
inner join '+(module.schema?module.schema+'.':'')+'deployment_target dt on d.deployment_target_id = dt.deployment_target_id \
inner join '+(module.schema?module.schema+'.':'')+"site site on site.site_id = dt.site_id\
where deployment_sts='PENDING' and deployment_id=@deployment_id\
";
appsrv.ExecRow('deployment', sql, [dbtypes.BigInt], { deployment_id: deployment_id }, function (err, rslt) {
if (err != null) { err.sql = sql; funcs.deploy_log_error(deployment_id, err); return onComplete(err); }
var publish_tgt = '';
var deployment = (rslt ? rslt[0] : null);
var template_variables = {};
var publish_path = '';
var publish_params = {};
if(!deployment) { let err = 'Invalid Deployment ID'; funcs.deploy_log_error(deployment_id, err); return onComplete(err); }
async.waterfall([
function(deploy_cb){
if(!module.Config.onDeploy_Init) return deploy_cb();
module.Config.onDeploy_Init(jsh, deployment_id, deployment, function(err, stopDeployment){
if(err){ funcs.deploy_log_error(deployment_id, err); return onComplete(err); }
if(stopDeployment) return onComplete();
return deploy_cb();
});
},
//Change status to RUNNING
function(deploy_cb){
var sql = 'update '+(module.schema?module.schema+'.':'')+"deployment set deployment_sts='RUNNING' where deployment_id=@deployment_id;";
var sql_ptypes = [dbtypes.BigInt];
var sql_params = { deployment_id: deployment_id };
appsrv.ExecCommand('deployment', sql, sql_ptypes, sql_params, function (err, rslt) {
if (err) return deploy_cb(err);
return deploy_cb();
});
},
//Initialize deployment
function(deploy_cb){
publish_tgt = deployment.publish_tgt;
deployment_id = rslt[0].deployment_id;
funcs.deploy_log_info(deployment_id, 'Deploying: '+(deployment.deployment_tag||''));
if(!publish_tgt){ return deploy_cb('Publish Target system parameter is not defined'); }
if(deployment.deployment_target_sts.toUpperCase() != 'ACTIVE'){ return deploy_cb('Deployment Target is not ACTIVE'); }
return deploy_cb();
},
//Get deployment target publish params
function(deploy_cb){
publish_path = path.isAbsolute(publish_tgt) ? publish_tgt : path.join(jsh.Config.datadir,publish_tgt);
publish_path = path.normalize(publish_path);
//Deployment Target Publish Params
try{
publish_params = funcs.parseDeploymentTargetPublishConfig(deployment.site_id, deployment.deployment_target_publish_config, 'publish');
}
catch(ex){
return deploy_cb(ex);
}
publish_params.site_default_page_filename = deployment.site_default_page_filename;
publish_params.publish_path = publish_path;
publish_params.deployment_id = deployment_id;
publish_params.deployment_target_id = deployment.deployment_target_id;
return deploy_cb();
},
//Get template variables
function(deploy_cb){
template_variables = {};
try{
if(deployment.deployment_target_template_variables) template_variables = _.extend(template_variables, JSON.parse(deployment.deployment_target_template_variables));
}
catch(ex){
return deploy_cb('Publish Target has invalid deployment_target_template_variables: '+deployment.deployment_target_template_variables);
}
funcs.parseTemplateVariables('publish', 'deployment', deployment.site_id, undefined, template_variables, publish_params, function(err, parsed_template_variables){
if(err) return deploy_cb(err);
template_variables = parsed_template_variables;
return deploy_cb();
});
},
//Initialize private key if not configured
function (deploy_cb){
funcs.generate_deployment_target_key(deployment.deployment_target_id, function(err){
return deploy_cb(err);
});
},
//Execute deployment
function(deploy_cb){
publish_params = _.extend(JSON.parse(JSON.stringify(template_variables)), publish_params);
deployment.publish_params = publish_params;
//Branch Data
var component_maxUniqueId = 0;
var component_maxUniqueIdSalt = crypto.randomBytes(16).toString('hex');
var branchData = {
publish_params: publish_params,
template_variables: template_variables,
deployment: deployment,
site_id: deployment.site_id,
site_config: {},
site_files: {},
site_redirects: [],
page_keys: {},
page_templates: null,
page_template_html: {},
page_redirects: {},
page_base_paths: {},
page_files: {},
page_data: {},
component_templates: null,
component_template_html: {},
component_export_template_html: {},
component_getUniqueId: function(){ return crypto.createHash('md5').update(component_maxUniqueIdSalt+'.'+(++component_maxUniqueId).toString()).digest('hex'); },
media_keys: {},
media_items: {},
menus: {},
menu_template_html: {},
sitemaps: {},
pageIncludes: {},
};
var git_branch = Helper.ReplaceAll(publish_params.git_branch, '%%%SITE_ID%%%', deployment.site_id);
var deployment_git_revision = (deployment.deployment_git_revision||'');
if(deployment_git_revision){
//--------------------------
//Single Revision Deployment
//--------------------------
async.waterfall([
//Create output folder if it does not exist
function (cb){
return HelperFS.createFolderRecursive(publish_path, cb);
},
//Git - Initialize and Select Branch
function (cb) {
if(!cms.Config.git || !cms.Config.git.enabled){
return cb(new Error('GIT setup required for redeployment. Please configure in app.config.js - jsHarmonyCMS.Config.git'));
}
async.waterfall([
//Initialize Git, if not initialized in publish folder
function(git_cb){
funcs.gitExec(publish_path, ['rev-parse','--show-toplevel'], function(err, rslt){
if(!err && rslt && (path.normalize(rslt)==publish_path)) return git_cb();
return git_cb(new Error('GIT not initialized in publish path. Cannot find revision for deployment.'));
});
},
//Check if branch exists
function(git_cb){
funcs.gitExec(publish_path, ['show-ref','--verify','--quiet','refs/heads/'+git_branch], function(err, rslt){
if(!err && !rslt) return git_cb();
return git_cb(new Error('Target revision not found in publish folder. Cannot find revision for deployment'));
});
},
//Check if deployment commit
function(git_cb){
funcs.gitExec(publish_path, ['cat-file','-e',deployment_git_revision], function(err, rslt){
if(!err && !rslt) return git_cb();
return git_cb(new Error('Target deployment commit not found in publish folder. Cannot find revision for deployment'));
});
},
//Checking out target revision
function(git_cb){
funcs.deploy_log_info(deployment_id, 'Checking out git revision: '+deployment_git_revision);
funcs.gitExec(publish_path, ['checkout','-f',deployment_git_revision], function(err, rslt){
return git_cb(err);
});
}
], cb);
},
//Get list of all site files
function (cb){
var folders = {};
HelperFS.funcRecursive(publish_path, function (filepath, relativepath, file_cb) { //filefunc
var parentpath = path.dirname(relativepath);
var webpath = '';
if(parentpath=='.') webpath = relativepath;
else webpath = folders[parentpath] + '/' + path.basename(relativepath);
fs.readFile(filepath, null, function(err, filecontent){
if(err) return cb(err);
branchData.site_files[webpath] = {
md5: crypto.createHash('md5').update(filecontent).digest('hex')
};
return file_cb();
});
}, function (dirpath, relativepath, dir_cb) {
if(relativepath=='.git') return dir_cb(false);
var parentpath = path.dirname(relativepath);
if(parentpath=='.') folders[relativepath] = relativepath;
else {
folders[relativepath] = folders[parentpath] + '/' + path.basename(relativepath);
}
return dir_cb();
}, {
file_before_dir: false,
preview_dir: function(dirpath, relativepath, dir_cb){
if(relativepath=='.git') return dir_cb(false);
return dir_cb();
}
}, cb);
},
//Deploy to target
function (cb) {
var deploy_path = (deployment.deployment_target_publish_path||'').toString();
if(!deploy_path){ return cb(); }
else if(deploy_path.indexOf('file://')==0){
//File Deployment
return funcs.deploy_fs(deployment, publish_path, deploy_path.substr(7), branchData.site_files, cb);
}
else if(deploy_path.indexOf('s3://')==0){
//Amazon S3 Deployment
return funcs.deploy_s3(deployment, publish_path, deploy_path, branchData.site_files, cb);
}
else if(deploy_path.indexOf('cmshost://')==0) {
return funcs.deploy_cmshost(deployment, publish_path, deploy_path.substr(10), branchData.site_files, cb);
}
else if((deploy_path.indexOf('ftps://')==0)||(deploy_path.indexOf('ftp://')==0)||(deploy_path.indexOf('sftp://')==0)) {
return funcs.deploy_ftp(deployment, publish_path, deploy_path, branchData.site_files, cb);
}
else if((deploy_path.indexOf('git_ssh://')==0)||(deploy_path.indexOf('git_https://')==0)) {
return funcs.deploy_git(deployment, publish_path, deploy_path, branchData.site_files, cb);
}
else return cb(new Error('Deployment Target path not supported'));
},
//Exec Post-Deployment Shell Command
function (cb) {
if(!publish_params.exec_post_deployment) return cb();
funcs.shellExec(
publish_params.exec_post_deployment.cmd,
publish_path,
_.extend({ shell: true }, publish_params.exec_post_deployment.params), function(err, rslt){
if(err) return cb(err);
if(rslt) rslt = rslt.trim();
funcs.deploy_log_info(deployment_id, rslt);
return cb();
});
},
//Log success
function (cb) {
var deploy_path = (deployment.deployment_target_publish_path||'').toString();
var msg = 'Deployment successful';
if(deploy_path.indexOf('cmshost://')==0) msg = 'CMS Deployment Host notified';
funcs.deploy_log_info(deployment_id, msg);
return cb();
},
], function (err, rslt) {
if (err) return deploy_cb(err.toString() + '\n' + (err.stack?err.stack:(new Error()).stack));
return deploy_cb();
});
}
else{
//-------------------
//Standard Deployment
//-------------------
async.waterfall([
//Create output folder if it does not exist
function (cb){
return HelperFS.createFolderRecursive(publish_path, cb);
},
//Get site config
function(cb){
funcs.getSiteConfig('deployment', branchData.site_id, { }, function(err, siteConfig){
if(err) return cb(err);
branchData.site_config = siteConfig;
return cb();
});
},
//Git - Initialize and Select Branch
function (cb) {
if(!cms.Config.git || !cms.Config.git.enabled){
if(deployment.deployment_git_revision) return cb(new Error('GIT setup required for redeployment. Please configure in app.config.js - jsHarmonyCMS.Config.git'));
return cb();
}
async.waterfall([
//Initialize Git, if not initialized in publish folder
function(git_cb){
funcs.gitExec(publish_path, ['rev-parse','--show-toplevel'], function(err, rslt){
if(!err && rslt && (path.normalize(rslt)==publish_path)) return git_cb();
//Initialize git for the first time
funcs.deploy_log_info(deployment_id, 'Initializing git in publish path: '+publish_path);
funcs.gitExec(publish_path, ['init','-q'], function(err, rslt){
if(err) return git_cb(err);
//Set git email
funcs.deploy_log_info(deployment_id, 'Setting git email');
funcs.gitExec(publish_path, ['config','user.email','cms@localhost'], function(err, rslt){
if(err) return git_cb(err);
//Disable CRLF Warnings
funcs.deploy_log_info(deployment_id, 'Disable git CRLF warnings');
funcs.gitExec(publish_path, ['config','core.safecrlf','false'], function(err, rslt){
if(err) return git_cb(err);
//Set git user
funcs.deploy_log_info(deployment_id, 'Setting git user');
funcs.gitExec(publish_path, ['config','user.name','CMS'], function(err, rslt){
if(err) return git_cb(err);
return git_cb();
});
});
});
});
});
},
//Check if branch exists, create if it does not
function(git_cb){
funcs.gitExec(publish_path, ['show-ref','--verify','--quiet','refs/heads/'+git_branch], function(err, rslt){
if(!err && !rslt) return git_cb();
//Initialize git for the first time
funcs.deploy_log_info(deployment_id, 'Initializing git branch: '+git_branch);
funcs.gitExec(publish_path, ['checkout','-f','--orphan',git_branch], function(err, rslt){
if(err) return git_cb(err);
funcs.deploy_log_info(deployment_id, 'Saving initial commit');
funcs.gitExec(publish_path, ['commit','--allow-empty','-m','Initial commit'], function(err, rslt){
if(err) return git_cb(err);
return git_cb();
});
});
});
},
//Checking out target branch
function(git_cb){
funcs.deploy_log_info(deployment_id, 'Checking out git branch: '+git_branch);
funcs.gitExec(publish_path, ['checkout','-f',git_branch], function(err, rslt){
return git_cb(err);
});
}
], cb);
},
//Clear output folder
function (cb){
//rmdirRecursive
HelperFS.funcRecursive(publish_path, function (filepath, relativepath, file_cb) { //filefunc
fs.unlink(filepath, file_cb);
}, function (dirpath, relativepath, dir_cb) { //dirfunc
if(relativepath) HelperFS.rmdirRecursive(dirpath, dir_cb);
else return dir_cb();
}, {
file_before_dir: true,
preview_dir: function(dirpath, relativepath, dir_cb){
if(relativepath=='.git') return dir_cb(false);
return dir_cb();
}
}, cb);
},
//Copy static files to publish folder
function (cb){
if(!publish_params.copy_folders) return cb();
async.eachSeries(publish_params.copy_folders, function(fpath, file_cb){
if(!path.isAbsolute(fpath)) fpath = path.join(jsh.Config.datadir, fpath);
fs.lstat(fpath, function(err, stats){
if(err) return file_cb(err);
if(!stats.isDirectory()) return file_cb(new Error('"copy_folder" parameter is not a folder: ' + fpath));
//Add copied files to "site_files" array
HelperFS.copyRecursive(fpath, publish_path,
{
forEachDir: function(dirpath, targetpath, relativepath, copy_cb){
if(relativepath=='.git') return cb(false);
return copy_cb(true);
},
forEachFile: function(filepath, targetpath, relativepath, copy_cb){
var fhash = crypto.createHash('md5');
HelperFS.copyFile(filepath, targetpath, copy_cb, {
onData: function(data) { fhash.update(data); },
onClose: function(){ branchData.site_files[HelperFS.convertWindowsToPosix(relativepath)] = { md5: fhash.digest('hex') }; },
});
},
},
file_cb
);
});
}, cb);
},
//Copy site files to publish folder
function (cb){
var sitePath = path.join(path.join(jsh.Config.datadir,'site'),(branchData.site_id||'').toString());
fs.lstat(sitePath, function(err, stats){
if(err){
if (HelperFS.fileNotFound(err)) return cb(); //Not found
return cb(err); //Other FS Error
}
HelperFS.copyRecursive(sitePath, publish_path,
{
forEachDir: function(dirpath, targetpath, relativepath, dir_cb){
if((relativepath=='templates') && !publish_params.publish_local_templates) return dir_cb(false);
if(relativepath=='.git') return dir_cb(false);
return dir_cb(true);
},
forEachFile: function(filepath, targetpath, relativepath, copy_cb){
var fhash = crypto.createHash('md5');
HelperFS.copyFile(filepath, targetpath, copy_cb, {
onData: function(data) { fhash.update(data); },
onClose: function(){ branchData.site_files[HelperFS.convertWindowsToPosix(relativepath)] = { md5: fhash.digest('hex') }; },
});
},
},
cb
);
});
},
//Run onBeforeDeploy functions
function(cb){
async.eachOfSeries(cms.BranchItems, function(branch_item, branch_item_type, branch_item_cb){
if(!branch_item.deploy) return branch_item_cb();
if(!publish_params.generate || !publish_params.generate.onBeforeDeploy) return branch_item_cb();
if(_.isArray(publish_params.generate.onBeforeDeploy) && !_.includes(publish_params.generate.onBeforeDeploy, branch_item_type)) return branch_item_cb();
Helper.execif(branch_item.deploy.onBeforeDeploy,
function(f){
branch_item.deploy.onBeforeDeploy(jsh, branchData, publish_params, f);
},
branch_item_cb
);
}, cb);
},
//Load Custom Branch Data
function (cb){
if(!module.Config.onDeploy_LoadData) return cb();
return module.Config.onDeploy_LoadData(jsh, branchData, publish_params, cb);
},
//Run onDeploy functions
function(cb){
var branchItemTypes = funcs.getDeploymentSortedBranchItemTypes();
async.eachSeries(branchItemTypes, function(branch_item_type, branch_item_cb){
funcs.deploy_log_info(deployment_id, 'Generating: '+branch_item_type.toUpperCase()+' items');
var branch_item = cms.BranchItems[branch_item_type];
if(!branch_item.deploy) r