langium-cli
Version:
CLI for Langium - the language engineering tool
443 lines • 19.8 kB
JavaScript
/******************************************************************************
* Copyright 2021 TypeFox GmbH
* This program and the accompanying materials are made available under the
* terms of the MIT License, which is available in the project root.
******************************************************************************/
import { URI } from 'langium';
import { loadConfig } from './package.js';
import { AstUtils, GrammarAST } from 'langium';
import { createLangiumGrammarServices, resolveImport, resolveImportUri, resolveTransitiveImports } from 'langium/grammar';
import { NodeFileSystem } from 'langium/node';
import { generateAst } from './generator/ast-generator.js';
import { serializeGrammar } from './generator/grammar-serializer.js';
import { generateModule } from './generator/module-generator.js';
import { generateTextMate } from './generator/highlighting/textmate-generator.js';
import { generateMonarch } from './generator/highlighting/monarch-generator.js';
import { generatePrismHighlighting } from './generator/highlighting/prism-generator.js';
import { getTime, log } from './generator/langium-util.js';
import { elapsedTime, getUserChoice, schema } from './generator/node-util.js';
import { RelativePath } from './package-types.js';
import { getFilePath } from './package.js';
import { validateParser } from './parser-validation.js';
import { generateTypesFile } from './generator/types-generator.js';
import { createGrammarDiagramHtml, createGrammarDiagramSvg } from 'langium-railroad';
import { validate } from 'jsonschema';
import chalk from 'chalk';
import * as path from 'path';
import fs from 'fs-extra';
export async function generate(options) {
const config = await loadConfig(options);
const validation = validate(config, await schema, {
nestedErrors: true
});
if (!validation.valid) {
log('error', options, chalk.red('Error: Your Langium configuration is invalid.'));
const errors = validation.errors.filter(error => error.path.length > 0);
errors.forEach(error => {
log('error', options, `--> ${error.stack}`);
});
return false;
}
const result = await runGenerator(config, options);
if (options.watch) {
printSuccess(result);
console.log(getTime() + 'Langium generator will continue running in watch mode.');
await runWatcher(config, options, await allGeneratorFiles(result));
}
// Outside of watch mode, report elapsed time for successful generation.
printSuccess(result);
return result.success;
}
async function allGeneratorFiles(results) {
const files = Array.from(new Set(results.files));
const filesExist = await Promise.all(files.map(e => fs.exists(e)));
return files.filter((_, i) => filesExist[i]);
}
async function runWatcher(config, options, files) {
if (files.length === 0) {
return;
}
const watchers = [];
for (const grammarFile of files) {
const watcher = fs.watch(grammarFile, undefined, watch);
watchers.push(watcher);
}
// The watch might be triggered multiple times
// We only want to execute once
let watcherTriggered = false;
async function watch() {
if (watcherTriggered) {
return;
}
watcherTriggered = true;
// Delay the generation a bit in case multiple files are changed at once
await delay(20);
console.log(getTime() + 'File change detected. Starting compilation...');
const results = await runGenerator(config, options);
for (const watcher of watchers) {
watcher.close();
}
printSuccess(results);
runWatcher(config, options, await allGeneratorFiles(results));
}
await new Promise(() => { });
}
function printSuccess(results) {
if (results.success) {
console.log(`${getTime()}Langium generator finished ${chalk.green.bold('successfully')} in ${elapsedTime()}ms`);
}
}
async function delay(ms) {
return new Promise(resolve => {
setTimeout(() => resolve(), ms);
});
}
const { shared: sharedServices, grammar: grammarServices } = createLangiumGrammarServices(NodeFileSystem);
const documents = sharedServices.workspace.LangiumDocuments;
async function eagerLoad(document, uris = new Set()) {
const uriString = document.uri.toString();
if (!uris.has(uriString)) {
uris.add(uriString);
const grammar = document.parseResult.value;
if (GrammarAST.isGrammar(grammar)) {
for (const imp of grammar.imports) {
const importUri = resolveImportUri(imp);
if (importUri) {
const document = await sharedServices.workspace.LangiumDocuments.getOrCreateDocument(importUri);
await eagerLoad(document, uris);
}
}
}
}
return Array.from(uris).map(e => URI.parse(e));
}
/**
* Creates a map that contains elements of all grammars.
* This includes both input grammars and their transitive dependencies.
*/
function mapGrammarElements(grammars, visited = new Set(), map = new Map()) {
for (const grammar of grammars) {
const doc = AstUtils.getDocument(grammar);
const uriString = doc.uri.toString();
if (!visited.has(uriString)) {
visited.add(uriString);
map.set(grammar, grammar.rules
.concat(grammar.types)
.concat(grammar.interfaces));
const importedGrammars = grammar.imports.map(e => resolveImport(documents, e)).filter((e) => e !== undefined);
mapGrammarElements(importedGrammars, visited, map);
}
}
return map;
}
function embedReferencedGrammar(grammar, map) {
var _a;
const allGrammars = resolveTransitiveImports(documents, grammar);
const linker = grammarServices.references.Linker;
const buildReference = linker.buildReference.bind(linker);
for (const importedGrammar of allGrammars) {
const grammarElements = (_a = map.get(importedGrammar)) !== null && _a !== void 0 ? _a : [];
for (const element of grammarElements) {
const copy = AstUtils.copyAstNode(element, buildReference);
// Deactivate copied entry rule
if (GrammarAST.isParserRule(copy)) {
copy.entry = false;
}
if (GrammarAST.isAbstractRule(copy)) {
grammar.rules.push(copy);
}
else if (GrammarAST.isType(copy)) {
grammar.types.push(copy);
}
else if (GrammarAST.isInterface(copy)) {
grammar.interfaces.push(copy);
}
else {
throw new Error('Received invalid grammar element while generating project with multiple languages');
}
}
}
// Remove all imports, as their contents are now available in the grammar
const grammarCopy = Object.assign(Object.assign({}, grammar), { imports: [] });
// Link newly added elements to grammar
AstUtils.linkContentToContainer(grammarCopy);
return grammarCopy;
}
async function relinkGrammars(grammars) {
const linker = grammarServices.references.Linker;
const documentBuilder = sharedServices.workspace.DocumentBuilder;
const documentFactory = sharedServices.workspace.LangiumDocumentFactory;
const langiumDocuments = sharedServices.workspace.LangiumDocuments;
const documents = langiumDocuments.all.toArray();
// Unlink and delete all document data
for (const document of documents) {
linker.unlink(document);
}
await documentBuilder.update([], documents.map(e => e.uri));
// Create and build new documents
const newDocuments = grammars.map(e => {
const uri = AstUtils.getDocument(e).uri;
const newDoc = documentFactory.fromModel(e, uri);
e.$document = newDoc;
return newDoc;
});
newDocuments.forEach(e => langiumDocuments.addDocument(e));
await documentBuilder.build(newDocuments, { validation: false });
}
async function buildAll(config) {
for (const doc of documents.all) {
documents.deleteDocument(doc.uri);
}
const map = new Map();
const relPath = config[RelativePath];
const uris = new Set();
for (const languageConfig of config.languages) {
const absGrammarPath = URI.file(path.resolve(relPath, languageConfig.grammar));
const document = await documents.getOrCreateDocument(absGrammarPath);
await eagerLoad(document, uris);
}
for (const doc of documents.all) {
map.set(doc.uri.fsPath, doc);
}
await sharedServices.workspace.DocumentBuilder.build(documents.all.toArray(), {
validation: true
});
return map;
}
export async function runGenerator(config, options) {
var _a, _b;
if (!config.languages || config.languages.length === 0) {
log('error', options, 'No languages specified in config.');
return {
success: false,
files: []
};
}
if (options.mode) {
config.mode = options.mode;
}
const all = await buildAll(config);
const buildResult = (success) => ({
success,
files: Array.from(all.keys())
});
let hasErrors = false;
for (const [path, document] of all) {
const diagnostics = Array.from((_a = document.diagnostics) !== null && _a !== void 0 ? _a : []);
diagnostics.sort((a, b) => a.range.start.line - b.range.start.line);
for (const diagnostic of diagnostics) {
const message = `${getFilePath(path, config)}:${diagnostic.range.start.line + 1}:${diagnostic.range.start.character + 1} - ${diagnostic.message}`;
if (diagnostic.severity === 1) {
log('error', options, chalk.red(message));
}
else if (diagnostic.severity === 2) {
log('warn', options, chalk.yellow(message));
}
else {
log('log', options, message);
}
}
if (!hasErrors) {
hasErrors = diagnostics.length > 0 && diagnostics.some(e => e.severity === 1);
}
}
if (hasErrors) {
log('error', options, `Langium generator ${chalk.red.bold('failed')}.`);
return buildResult(false);
}
const grammars = [];
const configMap = new Map();
const relPath = config[RelativePath];
for (const languageConfig of config.languages) {
const absGrammarPath = URI.file(path.resolve(relPath, languageConfig.grammar)).fsPath;
const document = all.get(absGrammarPath);
if (document) {
const grammar = document.parseResult.value;
if (!grammar.isDeclared) {
log('error', options, chalk.red(`${absGrammarPath}: The entry grammar must start with the 'grammar' keyword.`));
return buildResult(false);
}
grammars.push(grammar);
configMap.set(grammar, languageConfig);
}
}
const grammarElements = mapGrammarElements(grammars);
const embeddedGrammars = [];
for (const grammar of grammars) {
const embeddedGrammar = embedReferencedGrammar(grammar, grammarElements);
embeddedGrammars.push(embeddedGrammar);
configMap.set(embeddedGrammar, configMap.get(grammar));
}
// We need to rescope the grammars again
// They need to pick up on the embedded references
await relinkGrammars(embeddedGrammars);
for (const grammar of embeddedGrammars) {
// Create and validate the in-memory parser
const parserAnalysis = await validateParser(grammar, config, configMap, grammarServices);
if (parserAnalysis instanceof Error) {
log('error', options, chalk.red(parserAnalysis.toString()));
return buildResult(false);
}
}
// Generate the output files
const output = path.resolve(relPath, (_b = config.out) !== null && _b !== void 0 ? _b : 'src/generated');
log('log', options, `Writing generated files to ${chalk.white.bold(output)}`);
if (await rmdirWithFail(output, ['ast.ts', 'grammar.ts', 'module.ts'], options)) {
return buildResult(false);
}
if (await mkdirWithFail(output, options)) {
return buildResult(false);
}
const genAst = generateAst(grammarServices, embeddedGrammars, config);
await writeWithFail(path.resolve(updateLangiumInternalAstPath(output, config), 'ast.ts'), genAst, options);
const serializedGrammar = serializeGrammar(grammarServices, embeddedGrammars, config);
await writeWithFail(path.resolve(output, 'grammar.ts'), serializedGrammar, options);
const genModule = generateModule(embeddedGrammars, config, configMap);
await writeWithFail(path.resolve(output, 'module.ts'), genModule, options);
for (const grammar of embeddedGrammars) {
const languageConfig = configMap.get(grammar);
if (languageConfig === null || languageConfig === void 0 ? void 0 : languageConfig.textMate) {
const genTmGrammar = generateTextMate(grammar, languageConfig);
const textMatePath = path.resolve(relPath, languageConfig.textMate.out);
log('log', options, `Writing textmate grammar to ${chalk.white.bold(textMatePath)}`);
await writeWithFail(textMatePath, genTmGrammar, options);
}
if (languageConfig === null || languageConfig === void 0 ? void 0 : languageConfig.monarch) {
const genMonarchGrammar = generateMonarch(grammar, languageConfig);
const monarchPath = path.resolve(relPath, languageConfig.monarch.out);
log('log', options, `Writing monarch grammar to ${chalk.white.bold(monarchPath)}`);
await writeWithFail(monarchPath, genMonarchGrammar, options);
}
if (languageConfig === null || languageConfig === void 0 ? void 0 : languageConfig.prism) {
const genPrismGrammar = generatePrismHighlighting(grammar, languageConfig);
const prismPath = path.resolve(relPath, languageConfig.prism.out);
log('log', options, `Writing prism grammar to ${chalk.white.bold(prismPath)}`);
await writeWithFail(prismPath, genPrismGrammar, options);
}
if (languageConfig === null || languageConfig === void 0 ? void 0 : languageConfig.railroad) {
let css;
if (languageConfig.railroad.css) {
const cssPath = path.resolve(relPath, languageConfig.railroad.css);
css = await readFileWithFail(cssPath, options);
}
const railroadOptions = { css };
const rules = grammar.rules.filter(GrammarAST.isParserRule);
const diagramPath = path.resolve(relPath, languageConfig.railroad.out);
if (languageConfig.railroad.mode !== 'svg') {
// Single File or no info -> write to HTML.
const diagram = createGrammarDiagramHtml(rules, railroadOptions);
log('log', options, `Writing railroad syntax diagram to ${chalk.white.bold(diagramPath)}`);
await writeWithFail(diagramPath, diagram, options);
}
else {
// Svg files requested -> make dir and write into it.
const diagrams = createGrammarDiagramSvg(rules, railroadOptions);
log('log', options, `Writing railroad syntax diagrams to ${chalk.white.bold(diagramPath)}`);
for (const [name, diagram] of diagrams) {
const filePath = path.join(diagramPath, name);
await writeWithFail(`${filePath}.svg`, diagram, options);
}
}
}
}
return buildResult(true);
}
function updateLangiumInternalAstPath(output, config) {
if (config.langiumInternal) {
// The Langium internal ast is generated to the languages package.
// This is done to prevent internal access to the `langium/grammar` export.
return path.join(output, '..', '..', 'languages', 'generated');
}
else {
return output;
}
}
export async function generateTypes(options) {
var _a;
const grammarPath = path.isAbsolute(options.grammar) ? options.grammar : path.resolve('.', options.grammar);
if (!fs.existsSync(grammarPath) || !fs.lstatSync(grammarPath).isFile()) {
log('error', { watch: false }, chalk.red(`Grammar file '${grammarPath}' doesn't exist or is not a file.`));
return;
}
const outputPath = (_a = options.output) !== null && _a !== void 0 ? _a : path.join(path.resolve(grammarPath, '..'), 'types.langium');
const typesFilePath = path.isAbsolute(outputPath) ? outputPath : path.join('.', outputPath);
if (!options.force && fs.existsSync(typesFilePath)) {
const overwriteTypesFile = await getUserChoice(`Target file '${path.relative('.', typesFilePath)}' already exists. Overwrite?`, ['yes', 'no'], 'yes') === 'yes';
if (!overwriteTypesFile) {
log('log', { watch: false }, 'Generation canceled.');
return;
}
}
const grammarDoc = await doLoadAndUpdate(await documents.getOrCreateDocument(URI.file(grammarPath)));
const genTypes = generateTypesFile(grammarServices, [grammarDoc.parseResult.value]);
await writeWithFail(typesFilePath, genTypes, { watch: false });
log('log', { watch: false }, `Generated type definitions to: ${chalk.white.bold(typesFilePath)}`);
return;
}
/**
* Builds the given grammar document and all imported grammars.
*/
async function doLoadAndUpdate(grammarDoc) {
const allUris = await eagerLoad(grammarDoc);
await sharedServices.workspace.DocumentBuilder.update(allUris, []);
for (const doc of documents.all) {
await sharedServices.workspace.DocumentBuilder.build([doc]);
if (doc.uri === grammarDoc.uri) {
// update grammar doc after rebuild
grammarDoc = doc;
}
}
return grammarDoc;
}
async function rmdirWithFail(dirPath, expectedFiles, options) {
try {
let deleteDir = true;
const dirExists = await fs.pathExists(dirPath);
if (dirExists) {
const existingFiles = await fs.readdir(dirPath);
const unexpectedFiles = existingFiles.filter(file => !expectedFiles.includes(path.basename(file)));
if (unexpectedFiles.length > 0) {
log('log', options, `Found unexpected files in the generated directory: ${unexpectedFiles.map(e => chalk.yellow(e)).join(', ')}`);
deleteDir = await getUserChoice('Do you want to delete the files?', ['yes', 'no'], 'yes') === 'yes';
}
if (deleteDir) {
await fs.remove(dirPath);
}
}
return false;
}
catch (e) {
log('error', options, `Failed to delete directory ${chalk.red.bold(dirPath)}`, e);
return true;
}
}
async function mkdirWithFail(path, options) {
try {
await fs.mkdirs(path);
return false;
}
catch (e) {
log('error', options, `Failed to create directory ${chalk.red.bold(path)}`, e);
return true;
}
}
async function writeWithFail(filePath, content, options) {
try {
const parentDir = path.dirname(filePath);
await mkdirWithFail(parentDir, options);
await fs.writeFile(filePath, content);
}
catch (e) {
log('error', options, `Failed to write file to ${chalk.red.bold(filePath)}`, e);
}
}
async function readFileWithFail(path, options) {
try {
return await fs.readFile(path, { encoding: 'utf8' });
}
catch (e) {
log('error', options, `Failed to read file from ${chalk.red.bold(path)}`, e);
return undefined;
}
}
//# sourceMappingURL=generate.js.map