sfcc-schemas
Version:
Salesforce Commerce Cloud import and export schemas validator
481 lines (480 loc) • 21.7 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const cli_progress_1 = __importDefault(require("cli-progress"));
const colorette_1 = require("colorette");
const es6_promise_pool_1 = __importDefault(require("es6-promise-pool"));
const fs_1 = __importDefault(require("fs"));
const glob_promise_1 = __importDefault(require("glob-promise"));
const lodash_1 = __importDefault(require("lodash"));
const moment_1 = __importDefault(require("moment"));
const path_1 = __importDefault(require("path"));
const promise_timeout_1 = require("promise-timeout");
const recursive_readdir_1 = __importDefault(require("recursive-readdir"));
const validate_with_xmllint_1 = require("validate-with-xmllint");
const xml2js_1 = __importDefault(require("xml2js"));
const yargs_1 = __importDefault(require("yargs"));
const { log } = console;
let options = {
projectpath: 'cartridges/app_project/cartridge',
sfrapath: 'cartridges/app_storefront_base/cartridge',
timeout: 5000
};
async function xsdfy() {
let files = await findXmlFiles();
let xsdmap = buildXsdMapping();
for (let j = 0; j < files.length; j++) {
let xml = files[j];
let xmllocal = path_1.default.relative(process.cwd(), xml);
let xmlcontent = fs_1.default.readFileSync(xml, { encoding: 'utf8' });
let ns = getNamespace(xmlcontent);
let schemaLocation = getSchemaLocation(xmlcontent);
if (schemaLocation) {
log((0, colorette_1.green)(`File ${xmllocal} already mapped to ${schemaLocation}`));
}
else {
let xsdfile = xsdmap.get(ns);
if (xsdfile) {
let xsdrelative = path_1.default.relative(xml, xsdfile);
log((0, colorette_1.yellow)(`Adding xsd to ${xml} -> ${xsdrelative}`));
xmlcontent = xmlcontent.replace(`${ns}"`, `${ns}" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="${ns} ${xsdrelative}"`);
fs_1.default.writeFileSync(xml, xmlcontent);
}
else {
log((0, colorette_1.yellow)(`Unmapped: ${xml}`));
}
}
}
}
async function validate(failsonerror) {
let files = await findXmlFiles();
let xsdmap = buildXsdMapping();
let results = [];
let message = `Validating ${files.length} xml files using sfcc schemas`;
const progress = new cli_progress_1.default.Bar({
format: `${(0, colorette_1.green)(message)} [${(0, colorette_1.cyan)('{bar}')}] {percentage}% | {value}/{total}`
}, cli_progress_1.default.Presets.rect);
progress.start(files.length, 0);
let count = 0;
let promisecount = 0;
let pool = new es6_promise_pool_1.default(() => {
if (promisecount < files.length) {
let xml = files[promisecount];
promisecount++;
return new Promise(async (resolve) => {
let xmlcontent = fs_1.default.readFileSync(xml, { encoding: 'utf8' });
let filename = path_1.default.basename(xml);
progress.update(++count);
let ns = getNamespace(xmlcontent);
if (!ns) {
(0, colorette_1.redBright)(`Namespace not found for ${filename}`);
}
let xsd = xsdmap.get(ns);
if (!xsd) {
if (ns !== 'http://www.demandware.com/xml/impex/accessrole/2007-09-05') { // exclude known missing ns
log((0, colorette_1.yellow)(`No xsd found for namespace ${ns}`));
}
resolve();
}
else {
let res = {};
try {
res = await (0, promise_timeout_1.timeout)(validateXml(xml, xsd), options.timeout);
}
catch (err) {
if (err instanceof promise_timeout_1.TimeoutError) {
res = {
xml: xml,
valid: false,
processerror: 'true',
messages: [`Validation timeout after ${options.timeout}ms`]
};
}
}
// console.log(chalk.green(`Done with ${xml}`));
results.push(res);
resolve();
}
});
}
return null;
}, 20);
await pool.start();
progress.stop();
let successcount = results.filter(i => i.valid).length;
let errorcount = results.filter(i => !i.valid && !i.processerror).length;
let notvalidated = results.filter(i => i.processerror).length;
if (errorcount > 0) {
log(`Validated ${results.length} files: ${(0, colorette_1.green)(successcount + ' valid')} and ${(0, colorette_1.redBright)(errorcount + ' with errors')}\n`);
}
else {
log(`Validated ${results.length} files: ${(0, colorette_1.green)('all good')} 🍺\n`);
}
if (notvalidated > 0) {
log((0, colorette_1.yellow)(`${notvalidated} files cannot be validated (environment problems or timeout)\n`));
}
results.forEach((result) => {
if (!result.valid) {
log((0, colorette_1.redBright)(`File ${result.xml} invalid:`));
result.messages.forEach((i) => {
let msg = i;
if (msg && msg.indexOf && msg.indexOf('cvc-complex-type') > -1 && msg.indexOf(': ') > -1) {
msg = msg.substr(msg.indexOf(': ') + 2);
}
log(`\n❌ ` + msg);
});
if (result.messages.length === 0) {
log((0, colorette_1.redBright)(`\n${JSON.stringify(result)}`));
}
log('\n');
}
});
if (failsonerror && errorcount > 0) {
log((0, colorette_1.redBright)(`${errorcount} xml files failed validation\n`));
process.exit(2); // fail build
throw new Error(`${errorcount} xml files failed validation`);
}
}
async function findXmlFiles() {
return (0, glob_promise_1.default)(`${path_1.default.join(process.cwd(), 'sites')}/**/*.xml`);
}
function buildXsdMapping() {
let xsdfolder = path_1.default.join(__dirname, '../xsd/');
let xsdmap = new Map();
fs_1.default.readdirSync(xsdfolder).forEach((file) => {
let fullpath = path_1.default.join(xsdfolder, file);
let ns = getNamespace(fs_1.default.readFileSync(fullpath, 'utf-8'));
if (ns) {
xsdmap.set(ns, fullpath);
}
else {
(0, colorette_1.redBright)(`Namespace not found in xsd ${fullpath}`);
}
});
return xsdmap;
}
function getNamespace(xmlcontent) {
let match = xmlcontent.match(new RegExp('xmlns(?::loc)? *= *"([a-z0-9/:.-]*)"'));
if (match) {
return match[1];
}
return null;
}
function getSchemaLocation(xmlcontent) {
let match = xmlcontent.match(/xsi:schemaLocation="(.*)"/);
// let match = xmlcontent.match(new RegExp('xsi:schemaLocation="([.|\n]*)"'));
if (match) {
return match[1];
}
return null;
}
async function validateXml(xml, xsd) {
return new Promise((resolve) => {
(0, validate_with_xmllint_1.validateXMLWithXSD)(fs_1.default.readFileSync(xml), xsd).then((res) => {
resolve({
xml: xml,
valid: true
});
})
.catch((err) => {
let result = {};
result.xml = xml;
result.xsd = xsd;
let filename = xml.replace('\\', '/');
filename = filename.substring(filename.lastIndexOf('/') + 1);
let errors = err.toString().split('\n-:')
.map((s) => s.trimEnd())
.filter((s) => !s.startsWith('Error: xmllint exited'))
.filter((s) => s != '' && s != '- fails to validate')
.map((s) => s.replace('\- fails to validate', ''))
.map((s) => s.replace(/Element '{\S*}/, 'Element \''))
.map((s) => s.replace('Schemas validity error : ', ''));
result.messages = errors;
resolve(result);
});
});
}
async function parseMeta(source) {
var parser = new xml2js_1.default.Parser({
trim: true,
normalizeTags: true,
mergeAttrs: true,
explicitArray: false,
attrNameProcessors: [function (name) { return lodash_1.default.replace(name, /-/g, ''); }],
tagNameProcessors: [function (name) { return lodash_1.default.replace(name, /-/g, ''); }]
});
let exts = await parser.parseStringPromise(fs_1.default.readFileSync(source, 'utf-8'));
if (exts.metadata && exts.metadata.typeextension) {
ensureArray(exts.metadata, 'typeextension');
exts = exts.metadata.typeextension.map((i) => cleanupEntry(i)).filter((i) => i.attributedefinitions);
}
else if (exts.metadata && exts.metadata.customtype) {
ensureArray(exts.metadata, 'customtype');
exts = exts.metadata.customtype.map((i) => cleanupEntry(i));
}
ensureArray(exts.urlrules, 'pipelinealiases');
cleanI18n(exts);
fs_1.default.writeFileSync(path_1.default.join(process.cwd(), 'output/config/', `${path_1.default.basename(source)}.json`), JSON.stringify(exts, null, 2));
// date parsing utils
exts.moment = moment_1.default;
return exts;
}
function ensureArray(object, field) {
if (object && object[field] && !object[field].length) {
object[field] = [object[field]];
}
}
function cleanupEntry(i) {
let res = i;
// normalize
if (res.customattributedefinitions) {
res.attributedefinitions = res.customattributedefinitions;
delete res.customattributedefinitions;
}
delete res.systemattributedefinitions;
// cleanup single attributes without array
if (res.attributedefinitions && res.attributedefinitions.attributedefinition && res.attributedefinitions.attributedefinition.attributeid) {
res.attributedefinitions.attributedefinition = [res.attributedefinitions.attributedefinition];
}
return res;
}
function cleanI18n(obj) {
Object
.entries(obj)
.forEach(entry => {
let [k, v] = entry;
if (v !== null && typeof v === 'object' && !v.escape) {
if (v._ && v['xml:lang'] && Object.keys(v).length === 2) {
obj[k] = v._;
// log(`-> replaced ${obj[k]}`);
}
cleanI18n(v);
}
});
}
async function entrypoints() {
const regex = /server\.(get|post)\(['" \n]*([a-zA-Z0-9]*)['" ]*/gm;
let inputpath = path_1.default.join(process.cwd(), 'output/code');
if (!fs_1.default.existsSync(inputpath)) {
return;
}
let files = await (0, recursive_readdir_1.default)(inputpath, [(i, stats) => !stats.isDirectory() && path_1.default.basename(path_1.default.dirname(i)) !== "controllers" && path_1.default.extname(i) !== "js"]);
let mapping = {};
for (let j = 0; j < files.length; j++) {
let file = files[j];
let controllername = path_1.default.basename(file);
controllername = controllername.substr(0, controllername.lastIndexOf('.'));
let dirname = path_1.default.basename(path_1.default.dirname(path_1.default.dirname(path_1.default.dirname(file))));
let filecontent = fs_1.default.readFileSync(file, 'utf8');
let m;
// eslint-disable-next-line no-cond-assign
while ((m = regex.exec(filecontent)) !== null) {
if (m.index === regex.lastIndex) {
regex.lastIndex++;
}
let pipeline = `${controllername}-${m[2]}`;
let method = lodash_1.default.upperCase(m[1]);
if (mapping[pipeline]) {
mapping[pipeline].cartridges.push(dirname);
}
else {
mapping[pipeline] = {
pipeline: pipeline,
method: method,
controller: controllername,
cartridges: [dirname]
};
}
}
}
return mapping;
}
async function listcontrollers() {
let projectbase = path_1.default.join(process.cwd(), options.projectpath);
if (!fs_1.default.existsSync(projectbase)) {
log(`Skipping controller docs, folder ${projectbase} not available`);
return;
}
let files = await (0, recursive_readdir_1.default)(path_1.default.join(projectbase, 'controllers'));
files = files.map(i => path_1.default.relative(projectbase, i));
let sfrabase = path_1.default.join(process.cwd(), options.projectpath);
sfrabase = path_1.default.join(process.cwd(), options.sfrapath);
let sfrafiles = await (0, recursive_readdir_1.default)(path_1.default.join(sfrabase, 'controllers'));
sfrafiles = sfrafiles.map(i => path_1.default.relative(sfrabase, i));
let controllers = files.map(i => ({
name: i,
project: true,
sfra: sfrafiles.includes(i)
}));
let sfracontrollers = sfrafiles.filter(i => !files.includes(i)).map(i => ({
name: i,
project: false,
sfra: true
}));
controllers = controllers.concat(sfracontrollers);
let templates = await (0, recursive_readdir_1.default)(path_1.default.join(projectbase, 'templates/default'));
templates = templates.map(i => path_1.default.relative(projectbase, i));
let sfratemplates = await (0, recursive_readdir_1.default)(path_1.default.join(sfrabase, 'templates/default'));
sfratemplates = sfratemplates.map(i => path_1.default.relative(sfrabase, i));
let templatesprj = templates.map(i => ({
name: i,
project: true,
sfra: sfratemplates.includes(i)
}));
let context = {
controllers: controllers,
templates: templatesprj
};
let output = path_1.default.join(process.cwd(), 'output/config/', 'controllers.html');
fs_1.default.writeFileSync(output, lodash_1.default.template(fs_1.default.readFileSync(path_1.default.resolve(__dirname, `templates/controllers.html`), 'utf-8'))(context));
log((0, colorette_1.green)(`Generated documentation at ${output}`));
}
async function metacheatsheet() {
const argv = yargs_1.default.argv;
options.projectpath = argv.projectpath || options.projectpath;
options.sfrapath = argv.sfrapath || options.sfrapath;
await buildMeta();
await buildFromXml('sites/site_template/services.xml', 'services.html');
await buildFromXml('sites/site_template/jobs.xml', 'jobs.html');
await buildSeo('url-rules.xml', 'seo.html');
await buildFromXml('sites/site_template/pagemetatag.xml', 'pagemetatag.html');
await listcontrollers();
await buildAssetDoc();
}
async function buildMeta() {
let definitionspath = path_1.default.join(process.cwd(), 'sites/site_template/meta/custom-objecttype-definitions.xml');
let extensionspath = path_1.default.join(process.cwd(), 'sites/site_template/meta/system-objecttype-extensions.xml');
let exts = await parseMeta(extensionspath);
let defs = await parseMeta(definitionspath);
let context = {
extensions: exts,
definitions: defs
};
let output = path_1.default.join(process.cwd(), 'output/config/', 'metacheatsheet.html');
fs_1.default.writeFileSync(output, lodash_1.default.template(fs_1.default.readFileSync(path_1.default.resolve(__dirname, `../templates/meta.html`), 'utf-8'))(context));
log((0, colorette_1.green)(`Generated documentation at ${output}`));
}
async function buildFromXml(input, html) {
let inputpath = path_1.default.join(process.cwd(), input);
if (!fs_1.default.existsSync(inputpath)) {
return;
}
let output = path_1.default.join(process.cwd(), 'output/config/', html);
let filepath = path_1.default.resolve(__dirname, `../templates/${html}`);
fs_1.default.writeFileSync(output, lodash_1.default.template(fs_1.default.readFileSync(filepath, 'utf-8'))(await parseMeta(inputpath)));
log((0, colorette_1.green)(`Generated documentation at ${output}`));
}
async function buildSeo(xml, html) {
let mappings = await entrypoints();
let context = await parseXmlSites(xml, 'seo.html');
if (!context) {
return;
}
context.sites.forEach((site) => {
let siteentrypoints = JSON.parse(JSON.stringify(mappings));
if (site.urlrules && site.urlrules.pipelinealiases && site.urlrules.pipelinealiases[0]) {
site.urlrules.pipelinealiases[0].pipelinealias.forEach((alias) => {
if (siteentrypoints[alias.pipeline]) {
siteentrypoints[alias.pipeline].alias = alias._;
}
else {
// console.log(`Not existing remapping: ${alias._}=${alias.pipeline}`);
siteentrypoints[alias.pipeline] = {
alias: alias._,
pipeline: alias.pipeline,
controller: alias.pipeline.substr(0, alias.pipeline.indexOf('-')),
cartridges: []
};
}
});
}
let entrypointsarray = Object.keys(siteentrypoints).map(i => siteentrypoints[i]).sort((a, b) => a.pipeline.localeCompare(b.pipeline));
site.entrypoints = entrypointsarray;
});
let output = path_1.default.join(process.cwd(), 'output/config/', html);
fs_1.default.writeFileSync(output, lodash_1.default.template(fs_1.default.readFileSync(path_1.default.resolve(__dirname, `../templates/${html}`), 'utf-8'))(context));
log((0, colorette_1.green)(`Generated documentation at ${output}`));
}
async function isml() {
let inputpath = path_1.default.join(process.cwd(), `${options.projectpath}/templates/default`);
if (!fs_1.default.existsSync(inputpath)) {
return;
}
let sfrapath = path_1.default.join(process.cwd(), `${options.sfrapath}/templates/default`);
let sfrafiles = await (0, recursive_readdir_1.default)(sfrapath);
let mapping = sfrafiles.filter(i => path_1.default.extname(i) === '.isml').map(i => ({
path: i,
template: path_1.default.relative(sfrapath, i),
type: 'sfra'
}));
let files = await (0, recursive_readdir_1.default)(inputpath);
let projectmapping = files.filter(i => path_1.default.extname(i) === '.isml').map(i => ({
path: i,
template: path_1.default.relative(inputpath, i),
type: 'project'
}));
projectmapping.forEach(i => {
let idx = mapping.findIndex(p => p.template === i.template);
if (idx) {
mapping.splice(idx, 1);
}
});
return mapping.concat(projectmapping).sort((a, b) => a.template.localeCompare(b.template));
}
async function buildAssetDoc() {
let html = 'assets.html';
let ismls = await isml();
if (!ismls) {
return;
}
const assetsregexp = /iscontentasset['" a-zA-Z0-9-/\n]* aid="([a-zA-Z0-9-_/]*)/gm;
const slotregexp = /isslot['" a-zA-Z0-9-/\n]* id="([a-zA-Z0-9-_/]*)/gm;
const includesregexp = /isinclude['" a-zA-Z0-9-/\n]* template="([a-zA-Z0-9-_/]*)/gm;
ismls.forEach(i => {
let filecontent = fs_1.default.readFileSync(i.path, { encoding: 'utf8' });
i.assets = regexpmatch(assetsregexp, filecontent);
i.slots = regexpmatch(slotregexp, filecontent);
i.includes = regexpmatch(includesregexp, filecontent);
});
ismls = ismls.filter(i => i.assets.length !== 0 || i.slots.length !== 0 || i.includes.length !== 0);
// log('isml:', JSON.stringify(ismls, null, 2));
let contentonly = ismls.filter(i => i.assets.length !== 0 || i.slots.length !== 0);
let output = path_1.default.join(process.cwd(), 'output/config/', html);
fs_1.default.writeFileSync(output, lodash_1.default.template(fs_1.default.readFileSync(path_1.default.resolve(__dirname, `templates/${html}`), 'utf-8'))({ templates: ismls, content: contentonly }));
log((0, colorette_1.green)(`Generated documentation at ${output}`));
}
function regexpmatch(regex, filecontent) {
let matches = [];
let m;
// eslint-disable-next-line no-cond-assign
while ((m = regex.exec(filecontent)) !== null) {
if (m.index === regex.lastIndex) {
regex.lastIndex++;
}
matches.push(m[1]);
}
return matches;
}
async function parseXmlSites(filename, html) {
let files = await (0, recursive_readdir_1.default)(path_1.default.join(process.cwd(), 'sites/site_template/sites/'), [(i, stats) => !stats.isDirectory() && path_1.default.basename(i) !== "url-rules.xml"]);
if (!files || files.length === 0) {
return;
}
let context = { sites: [] };
for (let j = 0; j < files.length; j++) {
let single = await parseMeta(files[j]);
single.id = path_1.default.basename(path_1.default.dirname(files[j]));
context.sites.push(single);
}
return context;
}
// eslint-disable-next-line no-unused-vars
async function buildFromXmlSites(filename, html) {
let context = await parseXmlSites(filename, html);
let output = path_1.default.join(process.cwd(), 'output/config/', html);
fs_1.default.writeFileSync(output, lodash_1.default.template(fs_1.default.readFileSync(path_1.default.resolve(__dirname, `templates/${html}`), 'utf-8'))(context));
log((0, colorette_1.green)(`Generated documentation at ${output}`));
}
exports.default = { validate, xsdfy, metacheatsheet };