bs-platform
Version:
bucklescript compiler, ocaml standard libary by bucklescript and its required runtime support
1,358 lines (1,236 loc) • 40.6 kB
JavaScript
//@ts-check
var fs = require('fs')
var path = require('path')
var cp = require('child_process')
var jscompDir = path.join(__dirname,'..','jscomp')
var runtimeDir = path.join(jscompDir, 'runtime')
var othersDir = path.join(jscompDir,'others')
var testDir = path.join(jscompDir,'test')
var jsDir = path.join(__dirname, '..', 'lib', 'js')
var runtimeFiles = fs.readdirSync(runtimeDir, 'ascii')
var runtimeMlFiles = runtimeFiles.filter(x=>!x.startsWith("bs_stdlib_mini") && x.endsWith('.ml') && x !== "js.ml")
var runtimeMliFiles = runtimeFiles.filter(x=>!x.startsWith("bs_stdlib_mini") && x.endsWith('.mli') && x !== "js.mli")
var runtimeSourceFiles = runtimeMlFiles.concat(runtimeMliFiles)
var runtimeJsFiles = [...new Set(runtimeSourceFiles.map(baseName))]
var js_package = pseudoTarget('js_pkg')
var runtimeTarget = pseudoTarget('runtime')
var othersTarget = pseudoTarget('others')
var stdlibTarget = pseudoTarget('$stdlib')
/**
* By default we use vendored,
* we produce two ninja files which won't overlap
* one is build.ninja which use vendored config
* the other is env.ninja which use binaries from environment
*
* In dev mode, files generated for vendor config
*
* build.ninja
* compiler.ninja
* snapshot.ninja
* runtime/build.ninja
* others/build.ninja
* $stdlib/build.ninja
* test/build.ninja
*
* files generated for env config
*
* env.ninja
* compilerEnv.ninja (no snapshot since env can not provide snapshot)
* runtime/env.ninja
* others/env.ninja
* $stdlib/env.ninja
* test/env.ninja
*
* In release mode:
*
* release.ninja
* runtime/release.ninja
* others/release.ninja
* $stdlib/release.ninja
*
* Like that our snapshot is so robust that
* we don't do snapshot in CI, we don't
* need do test build in CI either
*
*/
var useEnv = false
/**
* Note this file is not used in ninja file
* It is used to generate ninja file
* @returns {string}
* Note ocamldep.opt has built-in macro handling OCAML_VERSION
*/
var getOcamldepFile = ()=>{
if(useEnv){
return `ocamldep.opt`
} else{
return path.join(__dirname,'..','native',require('./buildocaml.js').getVersionPrefix(), 'bin','ocamldep.opt')
}
}
/**
* @type {string}
*/
var versionString = undefined
var getVersionString = () =>{
if (versionString === undefined) {
var searcher = 'version'
var output = cp.execSync(`${getOcamldepFile()} -version`, { encoding: 'ascii' })
versionString = output.substring(output.indexOf(searcher) + searcher.length).trim()
}
return versionString
}
/**
* @returns {boolean}
*/
var version6 = ()=>{
return !(getVersionString().includes('4.02'))
}
/**
* Fixed since it is already vendored
*/
var cppoMonoFile = `../vendor/cppo/cppo_bin.ml`
/**
*
* @param {string} name
* @param {string} content
*/
function writeFile(name,content){
fs.writeFile(name,content,'ascii',throwIfError)
}
/**
*
* @param {NodeJS.ErrnoException} err
*/
function throwIfError(err){
if(err!==null){
throw err
}
}
/**
*
* @typedef { {kind : "file" , name : string} | {kind : "pseudo" , name : string}} Target
* @typedef {{key : string, value : string}} Override
* @typedef { Target[]} Targets
* @typedef {Map<string,TargetSet>} DepsMap
*/
class TargetSet {
/**
*
* @param {Targets} xs
*/
constructor(xs=[]){
this.data = xs
}
/**
*
* @param {Target} x
*/
add (x){
var data = this.data
var found = false
for(var i = 0; i < data.length; ++i){
var cur = data[i]
if(cur.kind===x.kind && cur.name === x.name){
found = true
break
}
}
if(!found){
this.data.push(x)
}
return this
}
/**
* @returns {Targets} a copy
*
*/
toSortedArray(){
var newData = this.data.concat()
newData.sort((x,y)=>{
var kindx = x.kind
var kindy = y.kind
if(kindx > kindy){
return 1
} else if(kindx < kindy){
return -1
} else {
if(x.name > y.name){
return 1
} else if(x.name < y.name) {
return -1
} else {
return 0
}
}
})
return newData
}
/**
*
* @param {(item:Target)=>void} callback
*/
forEach(callback){
this.data.forEach(callback)
}
}
/**
*
* @param {string} target
* @param {string} dependency
* @param {DepsMap} depsMap
*/
function updateDepsKVByFile(target, dependency, depsMap) {
var singleTon = fileTarget(dependency)
if (depsMap.has(target)) {
depsMap.get(target).add(singleTon)
} else {
depsMap.set(target, new TargetSet([ singleTon]))
}
}
/**
*
* @param {string} target
* @param {string[]} dependencies
* @param {DepsMap} depsMap
*/
function updateDepsKVsByFile(target, dependencies, depsMap) {
var targets = fileTargets(dependencies)
if (depsMap.has(target)) {
var s = depsMap.get(target)
for (var i = 0; i < targets.length; ++i) {
s.add(targets[i])
}
} else {
depsMap.set(target, new TargetSet(targets))
}
}
/**
*
* @param {Target} file
* @param {string} cwd
*/
function targetToString(file,cwd){
switch(file.kind){
case "file":
return path.join(cwd,file.name)
case "pseudo":
return file.name
default:
throw Error
}
}
/**
*
* @param {Targets} files
* @param {string} cwd
*
* @returns {string} return a string separated with whitespace
*/
function targetsToString(files,cwd){
return files.map(x=>targetToString(x,cwd)).join(' ')
}
/**
*
* @param {Targets} outputs
* @param {Targets} inputs
* @param {Targets} deps
* @param {Override[]} overrides
* @param {string} rule
* @param {string} cwd
* @return {string}
*/
function ninjaBuild(outputs, inputs, rule, deps, cwd, overrides){
var fileOutputs = targetsToString(outputs,cwd)
var fileInputs = targetsToString(inputs,cwd)
var stmt = `build ${fileOutputs} : ${rule} ${fileInputs}`
// deps.push(pseudoTarget('../lib/bsc.exe'))
if (deps.length > 0) {
var fileDeps = targetsToString(deps,cwd)
stmt += ` | ${fileDeps}`
}
if (overrides.length > 0) {
stmt += `\n` + overrides.map(x=>{
return ` ${x.key} = ${x.value}`
}).join('\n')
}
return stmt
}
/**
*
* @param {Target} outputs
* @param {Targets} inputs
* @param {string} cwd
*/
function phony(outputs,inputs,cwd){
return ninjaBuild([outputs],inputs,'phony',[],cwd,[])
}
/**
*
* @param {string | string[]} outputs
* @param {string | string[]} inputs
* @param {string | string[]} fileDeps
* @param {string} rule
* @param {string} cwd
* @param {[string,string][]} overrides
* @param {Target | Targets} extraDeps
*/
function ninjaQuickBuild(outputs,inputs,rule,cwd, overrides,fileDeps,extraDeps){
var os =
Array.isArray(outputs)?
fileTargets(outputs) :
[fileTarget(outputs)]
var is =
Array.isArray(inputs) ?
fileTargets(inputs) :
[fileTarget(inputs)]
var ds =
Array.isArray(fileDeps) ?
fileTargets(fileDeps) :
[fileTarget(fileDeps)]
var dds =
Array.isArray(extraDeps) ?
extraDeps : [extraDeps]
return ninjaBuild(os, is, rule, ds.concat(dds), cwd, overrides.map(x=>{
return {key : x[0], value : x[1]}
}))
}
/**
* @typedef { (string | string []) } Strings
* @typedef { [string,string]} KV
* @typedef { [Strings, Strings, string, string, KV[], Strings, (Target|Targets)] } BuildList
* @param {BuildList[]} xs
* @returns {string}
*/
function ninjaQuickBuidList(xs){
return xs.map(x => ninjaQuickBuild(x[0],x[1],x[2],x[3],x[4],x[5],x[6])).join('\n')
}
/**
* @typedef { [string,string,string?]} CppoInput
* @param {CppoInput[]} xs
* @param {string} cwd
* @returns {string}
*/
function cppoList(cwd,xs){
return xs.map(x=>{
/**
* @type {KV[]}
*/
var variables;
if (x[2]){
variables = [['type',`-D ${x[2]}`]]
} else {
variables = []
}
var extraDeps = pseudoTarget(cppoFile)
return ninjaQuickBuild(x[0],x[1],cppoRuleName,cwd,variables,[],extraDeps)
}).join('\n')
}
/**
*
* @param {string} cwd
* @param {string[]} xs
* @returns {string}
*/
function mllList(cwd, xs){
return xs.map(x=>{
var output = baseName(x)+'.ml'
return ninjaQuickBuild(output,x,mllRuleName,cwd,[],[],[])
}).join('\n')
}
/**
*
* @param {string} name
* @returns {Target}
*/
function fileTarget(name){
return {kind:"file", name}
}
/**
*
* @param {string} name
* @returns {Target}
*/
function pseudoTarget(name){
return {kind : "pseudo", name}
}
/**
*
* @param {string[]} args
* @returns {Targets}
*/
function fileTargets(args){
return args.map(name=> fileTarget(name))
}
/**
*
* @param {string[]} outputs
* @param {string[]} inputs
* @param {DepsMap} depsMap
* @param {Override[]} overrides
* @param {Targets} extraDeps
* @param {string} rule
* @param {string} cwd
*/
function buildStmt(outputs, inputs, rule, depsMap, cwd, overrides,extraDeps){
var os = outputs.map(fileTarget)
var is = inputs.map(fileTarget)
var deps = new TargetSet()
for (var i = 0 ; i < outputs.length ; ++i ){
var curDeps = depsMap.get(outputs[i])
if(curDeps !== undefined){
curDeps.forEach(x=>deps.add(x))
}
}
extraDeps.forEach(x=>deps.add(x))
return ninjaBuild(os,is,rule,deps.toSortedArray(),cwd,overrides)
}
/**
*
* @param {string} x
*/
function replaceCmj(x) {
return x.trim().replace('cmx', 'cmj')
}
/**
*
* @param {string[]} files
* @param {string} dir
* @param {DepsMap} depsMap
* @return {Promise<DepsMap>}
* Note `bsdep.exe` does not need post processing and -one-line flag
* By default `ocamldep.opt` only list dependencies in its args
*/
function ocamlDepForBscAsync(files,dir, depsMap) {
return new Promise((resolve,reject) =>{
cp.exec(`${getOcamldepFile()} -one-line -native ${files.join(' ')}`, {
cwd: dir,
encoding: 'ascii'
},function(error,stdout,stderr){
if(error !== null){
return reject(error)
} else {
var pairs = stdout.split('\n').map(x => x.split(':'))
pairs.forEach(x => {
var deps;
if (x[1] !== undefined && (deps = x[1].trim())) {
deps = deps.split(' ');
updateDepsKVsByFile(replaceCmj(x[0]), deps.map(x => replaceCmj(x)), depsMap)
}
}
)
return resolve(depsMap)
}
})
})
}
/**
* @typedef {('HAS_ML' | 'HAS_MLI' | 'HAS_BOTH')} FileInfo
* @param {string[]} sourceFiles
* @returns {Map<string, FileInfo>}
* We make a set to ensure that `sourceFiles` are not duplicated
*/
function collectTarget(sourceFiles){
/**
* @type {Map<string,FileInfo>}
*/
var allTargets = new Map()
sourceFiles.forEach(x => {
var { ext, name } = path.parse(x)
var existExt = allTargets.get(name)
if (existExt === undefined) {
if(ext === '.ml') {
allTargets.set(name,'HAS_ML')
} else {
allTargets.set(name,'HAS_MLI')
}
} else {
switch (existExt) {
case 'HAS_ML':
if (ext === '.mli') {
allTargets.set(name, 'HAS_BOTH')
}
break
case 'HAS_MLI':
if (ext === '.ml') {
allTargets.set(name, 'HAS_BOTH')
}
case 'HAS_BOTH': break
}
}
})
return allTargets
}
/**
*
* @param {Map<string, FileInfo>} allTargets
* @param {string[]} collIn
* @returns {string[]} A new copy which is
*
*/
function scanFileTargets(allTargets,collIn){
var coll = collIn.concat()
allTargets.forEach((ext,mod)=>{
switch(ext){
case 'HAS_MLI':
coll.push(`${mod}.cmi`)
break
case 'HAS_BOTH':
coll.push(`${mod}.cmi`,`${mod}.cmj`)
break;
case 'HAS_ML':
coll.push(`${mod}.cmi`,`${mod}.cmj`)
break;
}
})
return coll
}
/**
*
* @param {DepsMap} depsMap
* @param {Map<string,string>} allTargets
* @param {string} cwd
* @param {Targets} extraDeps
* @return {string[]}
*/
function generateNinja(depsMap,allTargets, cwd,extraDeps=[]){
/**
* @type {string[]}
*/
var build_stmts = []
allTargets.forEach((x,mod)=>{
var ouptput_cmj = mod + ".cmj"
var output_cmi = mod + ".cmi"
var input_ml = mod + ".ml"
var input_mli = mod + ".mli"
/**
* @type {Override[]}
*/
var overrides = []
if(mod.endsWith('Labels')){
overrides.push({key:'bsc_flags',value : '$bsc_flags -nolabels'})
}
/**
*
* @param {string[]} outputs
* @param {string[]} inputs
*/
var mk = (outputs,inputs) => {
return build_stmts.push(buildStmt(outputs,inputs,'cc',depsMap,cwd,overrides,extraDeps))
}
switch (x) {
case 'HAS_BOTH':
mk([ouptput_cmj],[input_ml])
mk([output_cmi], [input_mli])
break;
case 'HAS_ML':
mk([output_cmi, ouptput_cmj], [input_ml])
break;
case 'HAS_MLI':
mk([output_cmi], [input_mli])
break;
}
})
return build_stmts
}
var COMPILIER= '../lib/bsc.exe'
var BSC_COMPILER = `bsc = ${COMPILIER}`
var compilerTarget = pseudoTarget(COMPILIER)
async function runtimeNinja(devmode=true){
var ninjaCwd = "runtime"
var externalDeps = devmode ? [compilerTarget] : []
var ninjaOutput = devmode ? (useEnv ? 'env.ninja' : 'build.ninja') : 'release.ninja'
var templateRuntimeRules = `
${BSC_COMPILER}
bsc_no_open_flags = -absname -no-alias-deps -bs-no-version-header -bs-diagnose -bs-no-check-div-by-zero -bs-cross-module-opt -bs-package-name bs-platform -bs-package-output commonjs:lib/js -bs-package-output es6:lib/es6 -nostdlib -nopervasives -unsafe -warn-error A -w -40-49-103 -bin-annot
bsc_flags = $bsc_no_open_flags -open Bs_stdlib_mini
rule cc
command = $bsc -bs-cmi -bs-cmj $bsc_flags -bs-no-implicit-include -I ${ninjaCwd} -c $in
description = $in -> $out
${ninjaQuickBuidList([
['bs_stdlib_mini.cmi', 'bs_stdlib_mini.mli',
'cc', ninjaCwd, [["bsc_flags", "-nostdlib -nopervasives"]], [],externalDeps],
[['js.cmj', 'js.cmi'], 'js.ml',
'cc', ninjaCwd,[["bsc_flags", "$bsc_no_open_flags"]], [], externalDeps]
])}
`
/**
* @type {DepsMap}
*/
var depsMap = new Map
var allTargets = collectTarget([...runtimeMliFiles, ...runtimeMlFiles])
var manualDeps = ['bs_stdlib_mini.cmi','js.cmj','js.cmi']
var allFileTargetsInRuntime = scanFileTargets(allTargets,manualDeps)
allTargets.forEach((ext,mod)=>{
switch(ext){
case 'HAS_MLI':
case 'HAS_BOTH':
updateDepsKVsByFile(mod+".cmi", manualDeps,depsMap)
break;
case 'HAS_ML':
updateDepsKVsByFile(mod+".cmj",manualDeps,depsMap)
break;
}
})
// FIXME: in dev mode, it should not rely on reading js file
// since it may cause a bootstrapping issues
try{
await Promise.all([ runJSCheckAsync(depsMap),
ocamlDepForBscAsync(runtimeSourceFiles,runtimeDir, depsMap)])
var stmts = generateNinja(depsMap,allTargets,ninjaCwd,externalDeps)
stmts.push(phony(runtimeTarget,fileTargets(allFileTargetsInRuntime),ninjaCwd))
writeFile(
path.join(runtimeDir, ninjaOutput),
templateRuntimeRules + stmts.join('\n') + '\n'
)
}catch(e){
console.log(e)
}
}
var dTypeString = 'TYPE_STRING'
var dTypeInt = 'TYPE_INT'
var dTypeFunctor = 'TYPE_FUNCTOR'
var dTypeLocalIdent = 'TYPE_LOCAL_IDENT'
var dTypeIdent = 'TYPE_IDENT'
var dTypePoly = 'TYPE_POLY'
var cppoRuleName = `cppo`
var cppoFile = `./bin/cppo.exe`
var cppoRule = () => `
rule ${cppoRuleName}
command = ${cppoFile} -V OCAML:${getVersionString()} $type $in -o $out
generator = true
`
var mllRuleName = `mll`
var mllRule = `
rule ${mllRuleName}
command = $ocamllex $in
generator = true
`
async function othersNinja(devmode=true) {
var externalDeps = [runtimeTarget]
var ninjaOutput = devmode ? (useEnv ?'env.ninja' : 'build.ninja') : 'release.ninja'
var ninjaCwd = 'others'
var templateOthersRules = `
${BSC_COMPILER}
bsc_flags = -absname -no-alias-deps -bs-no-version-header -bs-diagnose -bs-no-check-div-by-zero -bs-cross-module-opt -bs-package-name bs-platform -bs-package-output commonjs:lib/js -bs-package-output es6:lib/es6 -nostdlib -nopervasives -unsafe -warn-error A -w -40-49-103 -bin-annot -bs-noassertfalse -open Bs_stdlib_mini -I ./runtime
rule cc
command = $bsc -bs-cmi -bs-cmj $bsc_flags -bs-no-implicit-include -I ${ninjaCwd} -c $in
description = $in -> $out
${ devmode ?
`${cppoRule()}
${cppoList(ninjaCwd, [
['belt_HashSetString.ml', 'hashset.cppo.ml', dTypeString],
['belt_HashSetString.mli', 'hashset.cppo.mli', dTypeString ],
['belt_HashSetInt.ml', 'hashset.cppo.ml', dTypeInt ],
['belt_HashSetInt.mli', 'hashset.cppo.mli', dTypeInt ],
['belt_HashMapString.ml', 'hashmap.cppo.ml', dTypeString ],
['belt_HashMapString.mli', 'hashmap.cppo.mli', dTypeString ],
['belt_HashMapInt.ml', 'hashmap.cppo.ml', dTypeInt ],
['belt_HashMapInt.mli', 'hashmap.cppo.mli', dTypeInt, ],
['belt_MapString.ml', 'map.cppo.ml', dTypeString ],
['belt_MapString.mli', 'map.cppo.mli', dTypeString ],
['belt_MapInt.ml', 'map.cppo.ml', dTypeInt ],
['belt_MapInt.mli', 'map.cppo.mli', dTypeInt ],
['belt_SetString.ml', 'belt_Set.cppo.ml', dTypeString ],
['belt_SetString.mli', 'belt_Set.cppo.mli', dTypeString ],
['belt_SetInt.ml', 'belt_Set.cppo.ml', dTypeInt ],
['belt_SetInt.mli', 'belt_Set.cppo.mli', dTypeInt ],
['belt_MutableMapString.ml', 'mapm.cppo.ml', dTypeString ],
['belt_MutableMapString.mli', 'mapm.cppo.mli', dTypeString ],
['belt_MutableMapInt.ml', 'mapm.cppo.ml', dTypeInt ],
['belt_MutableMapInt.mli', 'mapm.cppo.mli', dTypeInt ],
['belt_MutableSetString.ml', 'setm.cppo.ml',dTypeString ],
['belt_MutableSetString.mli', 'setm.cppo.mli', dTypeString ],
['belt_MutableSetInt.ml', 'setm.cppo.ml', dTypeInt ],
['belt_MutableSetInt.mli', 'setm.cppo.mli', dTypeInt ],
['belt_SortArrayString.ml', 'sort.cppo.ml', dTypeString ],
['belt_SortArrayString.mli', 'sort.cppo.mli', dTypeString ],
['belt_SortArrayInt.ml', 'sort.cppo.ml', dTypeInt ],
['belt_SortArrayInt.mli', 'sort.cppo.mli', dTypeInt ],
['belt_internalMapString.ml', 'internal_map.cppo.ml', dTypeString ],
['belt_internalMapInt.ml', 'internal_map.cppo.ml', dTypeInt ],
['belt_internalSetString.ml', 'internal_set.cppo.ml', dTypeString ],
['belt_internalSetInt.ml', 'internal_set.cppo.ml', dTypeInt ],
['js_typed_array.ml', 'js_typed_array.cppo.ml',''],
['js_typed_array2.ml', 'js_typed_array2.cppo.ml',''],
])}
`
:
`
`
}
${ninjaQuickBuidList([
[['belt.cmj','belt.cmi'],'belt.ml',
'cc',ninjaCwd,[], [],externalDeps],
[['node.cmj','node.cmi'],'node.ml',
'cc',ninjaCwd,[], [],externalDeps],
])}
`
var othersDirFiles = fs.readdirSync(othersDir, 'ascii')
var jsPrefixSourceFiles = othersDirFiles.filter(
x => x.startsWith('js') && (x.endsWith('.ml') || x.endsWith(".mli")) && !(x.includes('.cppo'))
)
var othersFiles = othersDirFiles.filter(
x => !x.startsWith('js') && (x !== 'belt.ml') && (x!=='node.ml') && (x.endsWith('.ml') || x.endsWith('.mli')) && !(x.includes('.cppo')) // we have node ..
)
var jsTargets = collectTarget(jsPrefixSourceFiles)
var allJsTargets = scanFileTargets(jsTargets,[])
var [jsDepsMap, depsMap] = await Promise.all([ocamlDepForBscAsync(jsPrefixSourceFiles,
othersDir,
new Map
),
ocamlDepForBscAsync(othersFiles, othersDir, new Map())])
var jsOutput = generateNinja(jsDepsMap, jsTargets,ninjaCwd,externalDeps)
jsOutput.push(phony(js_package,fileTargets(allJsTargets),ninjaCwd))
// Note compiling belt.ml still try to read
// belt_xx.cmi we need enforce the order to
// avoid data race issues
var beltPackage = fileTarget('belt.cmi')
var nodePackage = fileTarget('node.cmi')
var beltTargets = collectTarget(othersFiles)
depsMap.forEach((s,k)=>{
if(k.startsWith('belt')){
s.add(beltPackage)
} else if(k.startsWith('node')){
s.add(nodePackage)
}
s.add(js_package)
})
var allOthersTarget = scanFileTargets(beltTargets,[])
var beltOutput = generateNinja(depsMap, beltTargets,ninjaCwd,externalDeps)
beltOutput.push(phony(othersTarget,fileTargets(allOthersTarget),ninjaCwd))
// ninjaBuild([`belt_HashSetString.ml`,])
writeFile(
path.join(othersDir, ninjaOutput),
templateOthersRules + jsOutput.join('\n') + '\n' + beltOutput.join('\n') + '\n'
)
}
/**
*
* @param {boolean} devmode
* generate build.ninja/release.ninja for stdlib-402
*/
async function stdlibNinja(devmode=true){
var stdlibVersion = version6 ()? 'stdlib-406' : 'stdlib-402'
var ninjaCwd = stdlibVersion
var stdlibDir = path.join(jscompDir,stdlibVersion)
var externalDeps = [othersTarget]
var ninjaOutput = devmode? (useEnv ? 'env.ninja' : 'build.ninja') : 'release.ninja'
var bsc_flags = 'bsc_flags'
/**
* @type [string,string][]
*/
var bsc_builtin_overrides = [[bsc_flags,`$${bsc_flags} -nopervasives`]]
// It is interesting `-w -a` would generate not great code sometimes
var warnings = devmode ? '-w -40-49-103' : '-w -40-49-103-3'
var templateStdlibRules = `
${BSC_COMPILER}
${bsc_flags} = -absname -no-alias-deps -bs-no-version-header -bs-diagnose -bs-no-check-div-by-zero -bs-cross-module-opt -bs-package-name bs-platform -bs-package-output commonjs:lib/js -bs-package-output es6:lib/es6 -nostdlib ${warnings} -bin-annot -bs-no-warn-unimplemented-external -I runtime -I others
rule cc
command = $bsc -bs-cmi -bs-cmj $${bsc_flags} -bs-no-implicit-include -I ${ninjaCwd} -c $in
description = $in -> $out
${ninjaQuickBuidList([
['camlinternalFormatBasics.cmi', 'camlinternalFormatBasics.mli',
'cc', ninjaCwd, bsc_builtin_overrides, [], externalDeps],
// we make it still depends on external
// to enjoy free ride on dev config for compiler-deps
['camlinternalFormatBasics.cmj', 'camlinternalFormatBasics.ml',
'cc', ninjaCwd, bsc_builtin_overrides, 'camlinternalFormatBasics.cmi',externalDeps],
['pervasives.cmj', 'pervasives.ml',
'cc',ninjaCwd, bsc_builtin_overrides,'pervasives.cmi', externalDeps],
[ 'pervasives.cmi', 'pervasives.mli',
'cc', ninjaCwd, bsc_builtin_overrides, 'camlinternalFormatBasics.cmj', externalDeps]
])}
`
var stdlibDirFiles = fs.readdirSync(stdlibDir,'ascii')
var sources = stdlibDirFiles.filter(x=>{
return !(x.startsWith('camlinternalFormatBasics')) &&
!(x.startsWith('pervasives')) &&
(x.endsWith('.ml') || x.endsWith('.mli'))
})
var depsMap = await ocamlDepForBscAsync(sources, stdlibDir, new Map)
var targets = collectTarget(sources)
var allTargets = scanFileTargets(targets,
['camlinternalFormatBasics.cmi','camlinternalFormatBasics.cmj',
'pervasives.cmi', 'pervasives.cmj'
])
targets.forEach((ext,mod)=>{
switch(ext){
case 'HAS_MLI':
case 'HAS_BOTH':
updateDepsKVByFile(mod+".cmi", 'pervasives.cmj',depsMap)
break
case 'HAS_ML':
updateDepsKVByFile(mod+".cmj", 'pervasives.cmj', depsMap)
break
}
})
var output = generateNinja(depsMap,targets,ninjaCwd, externalDeps)
output.push(phony(stdlibTarget,fileTargets(allTargets),ninjaCwd))
writeFile(
path.join(stdlibDir,ninjaOutput),
templateStdlibRules + output.join('\n') + '\n'
)
}
/**
*
* @param {string} text
*/
function getDeps(text) {
/**
* @type {string[]}
*/
var deps = []
text.replace(/(\/\*[\w\W]*?\*\/|\/\/[^\n]*|[.$]r)|\brequire\s*\(\s*["']([^"']*)["']\s*\)/g, function(_, ignore, id) {
if (!ignore)
deps.push(id);
return "" // TODO: examine the regex
});
return deps;
}
/**
*
* @param {string} x
*/
function baseName(x) {
return x.substr(0, x.indexOf('.'))
}
/**
*
*
*/
async function testNinja(){
var ninjaOutput = useEnv ? 'env.ninja' : 'build.ninja'
var ninjaCwd = `test`
var templateTestRules = `
${BSC_COMPILER}
bsc_flags = -absname -no-alias-deps -bs-no-version-header -bs-diagnose -bs-cross-module-opt -bs-package-name bs-platform -bs-package-output commonjs:jscomp/test -w -40-52 -warn-error A+8-3-30-26+101-102-103-104-52 -bin-annot -I runtime -I $stdlib -I others
rule cc
command = $bsc -bs-cmi -bs-cmj $bsc_flags -bs-no-implicit-include -I ${ninjaCwd} -c $in
description = $in -> $out
${mllRule}
${mllList(ninjaCwd, ['arith_lexer.mll','number_lexer.mll','simple_lexer_test.mll' ],
)}
`
var testDirFiles = fs.readdirSync(testDir,'ascii')
var sources = testDirFiles.filter(x=>{
return (x.endsWith('.ml') || x.endsWith('.mli')) &&
(!x.endsWith('bspack.ml'))
})
var depsMap = await ocamlDepForBscAsync(sources, testDir, new Map)
var targets = collectTarget(sources)
var output = generateNinja(depsMap, targets,ninjaCwd,[stdlibTarget])
writeFile(
path.join(testDir, ninjaOutput),
templateTestRules + output.join('\n') + '\n'
)
}
/**
*
* @param {DepsMap} depsMap
*/
function runJSCheckAsync(depsMap){
return new Promise((resolve) => {
var count = 0
var tasks = runtimeJsFiles.length
var updateTick = () =>{
count ++
if(count === tasks){
resolve(count)
}
}
runtimeJsFiles.forEach((name) => {
var jsFile = path.join(jsDir, name + ".js")
fs.readFile(jsFile, 'utf8', function (err, fileContent) {
if (err === null) {
var deps = getDeps(fileContent).map(x => path.parse(x).name + ".cmj")
fs.exists(path.join(runtimeDir, name + ".mli"), exist => {
if (exist) {
deps.push(name + ".cmi")
}
updateDepsKVsByFile(`${name}.cmj`, deps, depsMap)
updateTick()
})
} else {
// file non exist or reading error ignore
updateTick()
}
})
}
)
})
}
function checkEffect() {
var jsPaths = runtimeJsFiles.map(x => path.join(jsDir, x + ".js"))
var effect = jsPaths.map(x => {
return {
file: x,
content: fs.readFileSync(x, 'utf8')
}
}
).map(({ file, content: x }) => {
if (/No side effect|This output is empty/.test(x)) {
return {
file,
effect: 'pure'
}
} else if (/Not a pure module/.test(x)) {
return {
file,
effect: 'false'
}
} else {
return {
file,
effect: 'unknown'
}
}
}
).filter(({ effect }) => effect !== 'pure')
.map(({file, effect})=>{
return { file : path.basename(file), effect}
})
var black_list = new Set(["caml_builtin_exceptions.js", "caml_int32.js", "caml_int64.js", "caml_lexer.js", "caml_parser.js"])
var assert = require('assert')
assert (effect.length === black_list.size &&
effect.every(x=> black_list.has(x.file))
)
console.log(effect)
}
/**
*
* @param {string[]} domain
* @param {Map<string,Set<string>>} dependency_graph
* @returns {string[]}
*/
function sortFilesByDeps(domain, dependency_graph){
/**
* @type{string[]}
*/
var result = []
var workList = new Set(domain)
/**
*
* @param {Set<string>} visiting
* @param {string[]} path
* @param {string} current
*/
var visit = function(visiting,path,current){
if(visiting.has(current)){
throw new Error(`cycle: ${path.concat(current).join(' ')}`)
}
if(workList.has(current)){
visiting.add(current)
var next = dependency_graph.get(current)
if(next !== undefined && next.size > 0){
next.forEach(x=>{
visit(visiting, path.concat(current),x)
})
}
visiting.delete(current)
workList.delete(current)
result.push(current)
}
}
while(workList.size > 0){
visit(new Set(), [], workList.values().next().value )
}
return result
}
var emptyCount = 2
if (require.main === module) {
if(process.argv.includes('-env')){
useEnv = true
emptyCount ++
}
if(process.argv.includes('-check')){
checkEffect()
}
if (process.argv.length === emptyCount) {
updateDev()
updateRelease()
} else {
var dev = process.argv.includes('-dev')
var release = process.argv.includes('-release')
var all = process.argv.includes('-all')
if (all) {
updateDev()
updateRelease()
} else if (dev) {
updateDev()
} else if (release) {
updateRelease()
}
}
}
function updateRelease(){
if (!useEnv) {
runtimeNinja(false)
stdlibNinja(false)
othersNinja(false)
}
}
function updateDev(){
if(useEnv){
writeFile(path.join(jscompDir,'env.ninja'),`
${getEnnvConfigNinja()}
stdlib = ${version6() ? `stdlib-406` : `stdlib-402`}
subninja compilerEnv.ninja
subninja runtime/env.ninja
subninja others/env.ninja
subninja $stdlib/env.ninja
subninja test/env.ninja
build all: phony runtime others $stdlib test
`)
} else {
writeFile(path.join(jscompDir, 'build.ninja'), `
${getVendorConfigNinja()}
stdlib = ${version6() ? `stdlib-406` : `stdlib-402`}
snapshot_path = ${require('./buildocaml.js').getVersionPrefix()}
subninja compiler.ninja
subninja snapshot.ninja
subninja runtime/build.ninja
subninja others/build.ninja
subninja $stdlib/build.ninja
subninja test/build.ninja
build all: phony runtime others $stdlib test
`
)
writeFile(path.join(jscompDir, '..', 'lib', 'build.ninja'), `
ocamlopt = ocamlopt.opt
ext = exe
INCL= ${version6()?'4.06.1+BS' : '4.02.3+BS'}
include body.ninja
`)
}
runtimeNinja()
stdlibNinja(true)
testNinja()
othersNinja()
nativeNinja()
}
exports.updateDev = updateDev
exports.updateRelease = updateRelease
/**
*
* @param {string} dir
*/
function readdirSync(dir){
return fs.readdirSync(dir,'ascii')
}
/**
*
* @param {string} dir
*/
function test(dir){
return readdirSync(path.join(jscompDir,dir)).filter(x=> {
return (x.endsWith('.ml') || x.endsWith('.mli')) &&
!(x.endsWith('.cppo.ml') || x.endsWith('.cppo.mli'))
}).map(x=>path.join(dir,x))
}
/**
*
* @param {Set<string>} xs
* @returns {string}
*/
function setSortedToString(xs){
var arr = Array.from(xs).sort()
return arr.join(' ')
}
/**
* @returns {string}
*/
function getVendorConfigNinja(){
var prefix = `../native/${require('./buildocaml.js').getVersionPrefix()}/bin`
return `
ocamlopt = ${prefix}/ocamlopt.opt
ocamllex = ${prefix}/ocamllex.opt
ocamlmklib = ${prefix}/ocamlmklib
`
}
function getEnnvConfigNinja(){
return `
ocamlopt = ocamlopt.opt
ocamllex = ocamllex.opt
ocamlmklib = ocamlmklib
`
}
/**
* Note don't run `ninja -t clean -g`
* Since it will remove generated ml file which has
* an effect on depfile
*/
function nativeNinja() {
var ninjaOutput = useEnv ? 'compilerEnv.ninja' : 'compiler.ninja'
var sourceDirs = ['stubs', 'ext', 'common', 'syntax', 'depends', 'core', 'super_errors', 'outcome_printer', 'bsb', 'ounit', 'ounit_tests', 'main']
var includes = sourceDirs.map(x => `-I ${x}`).join(' ')
var cppoNative = `
${useEnv ? getEnnvConfigNinja() : getVendorConfigNinja()}
rule link
command = $ocamlopt -g -I +compiler-libs $flags $libs $in -o $out
build ${cppoFile}: link ${cppoMonoFile}
libs = unix.cmxa str.cmxa
${cppoRule()}
${cppoList('ext', [
['string_hash_set.ml', 'hash_set.cppo.ml', dTypeString],
['int_hash_set.ml', 'hash_set.cppo.ml', dTypeInt],
['ident_hash_set.ml', 'hash_set.cppo.ml', dTypeIdent],
['hash_set.ml', 'hash_set.cppo.ml', dTypeFunctor],
['hash_set_poly.ml', 'hash_set.cppo.ml', dTypePoly],
['int_vec.ml', 'vec.cppo.ml', dTypeInt],
['resize_array.ml', 'vec.cppo.ml', dTypeFunctor],
['string_set.ml', 'set.cppo.ml', dTypeString],
['set_int.ml', 'set.cppo.ml', dTypeInt],
['ident_set.ml', 'set.cppo.ml', dTypeIdent],
['string_map.ml', 'map.cppo.ml', dTypeString],
['int_map.ml', 'map.cppo.ml', dTypeInt],
['ident_map.ml', 'map.cppo.ml', dTypeIdent],
['ordered_hash_map_local_ident.ml', 'ordered_hash_map.cppo.ml', dTypeLocalIdent],
['ordered_hash_set_make.ml', 'ordered_hash_set.cppo.ml', dTypeFunctor],
['string_hashtbl.ml', 'hashtbl.cppo.ml', dTypeString],
['int_hashtbl.ml', 'hashtbl.cppo.ml', dTypeInt],
['ident_hashtbl.ml', 'hashtbl.cppo.ml', dTypeIdent],
['hashtbl_make.ml', 'hashtbl.cppo.ml', dTypeFunctor],
])}
${cppoList('outcome_printer',[
['tweaked_reason_oprint.ml','tweaked_reason_oprint.cppo.ml',''],
['reason_syntax_util.ml', 'reason_syntax_util.cppo.ml',''],
['reason_syntax_util.mli', 'reason_syntax_util.cppo.mli',''],
])}
`
var cppoNinjaFile = useEnv ? 'cppoEnv.ninja' : 'cppoVendor.ninja'
var templateNative = `
subninja ${cppoNinjaFile}
rule optc
command = $ocamlopt -I +compiler-libs ${includes} -g -w +6-40-30-23 -warn-error +a-40-30-23 -absname -c $in
description = $out : $in
rule archive
command = $ocamlopt -a $in -o $out
rule link
command = $ocamlopt -g -I +compiler-libs $flags $libs $in -o $out
rule mk_bsversion
command = node $in
generator = true
rule gcc
command = $ocamlopt -ccopt -fPIC -ccopt -O2 -ccopt -o -ccopt $out -c $in
build stubs/ext_basic_hash_stubs.o : gcc stubs/ext_basic_hash_stubs.c
rule ocamlmklib
command = $ocamlmklib $in -o $name
rule mk_keywords
command = ocaml $in
generator = true
build ext/js_reserved_map.ml: mk_keywords build_sorted.ml keywords.list
build stubs/libbs_hash.a stubs/dllbs_hash.so: ocamlmklib stubs/ext_basic_hash_stubs.o
name = stubs/bs_hash
rule stubslib
command = $ocamlopt -a $ml -o $out -cclib $clib
build stubs/stubs.cmxa : stubslib stubs/bs_hash_stubs.cmx stubs/libbs_hash.a
ml = stubs/bs_hash_stubs.cmx
clib = stubs/libbs_hash.a
rule p4of
command = camlp4of $flags -impl $in -printer o -o $out
generator = true
build core/js_fold.ml: p4of core/js_fold.mlp | core/j.ml
flags = -I core -filter map -filter trash
build core/js_map.ml: p4of core/js_map.mlp | core/j.ml
flags = -I core -filter Camlp4FoldGenerator -filter trash
build common/bs_version.ml : mk_bsversion build_version.js ../package.json
build ../lib/bsc.exe: link stubs/stubs.cmxa ext/ext.cmxa common/common.cmxa syntax/syntax.cmxa depends/depends.cmxa super_errors/super_errors.cmxa outcome_printer/outcome_printer.cmxa core/core.cmxa main/js_main.cmx
libs = ocamlcommon.cmxa
build ../lib/bsb.exe: link stubs/stubs.cmxa ext/ext.cmxa common/common.cmxa bsb/bsb.cmxa main/bsb_main.cmx
libs = ocamlcommon.cmxa unix.cmxa str.cmxa
build ../lib/bsb_helper.exe: link stubs/stubs.cmxa ext/ext.cmxa common/common.cmxa bsb/bsb.cmxa main/bsb_helper_main.cmx
libs = ocamlcommon.cmxa unix.cmxa str.cmxa
build ./bin/bspack.exe: link stubs/stubs.cmxa ext/ext.cmxa ./common/common.cmxa ./syntax/syntax.cmxa depends/depends.cmxa ./main/bspack_main.cmx
libs = unix.cmxa ocamlcommon.cmxa
flags = -I ./bin -w -40-30
build ./bin/cmjdump.exe: link ./stubs/stubs.cmxa ext/ext.cmxa common/common.cmxa syntax/syntax.cmxa depends/depends.cmxa core/core.cmxa main/cmjdump_main.cmx
libs = ocamlcommon.cmxa
rule bspack
command = ./bin/bspack.exe $flags -bs-main $main -o $out
depfile = $out.d
generator = true
build ./bin/tests.exe: link ounit/ounit.cmxa stubs/stubs.cmxa ext/ext.cmxa common/common.cmxa syntax/syntax.cmxa depends/depends.cmxa bsb/bsb.cmxa core/core.cmxa ounit_tests/ounit_tests.cmxa main/ounit_tests_main.cmx
libs = str.cmxa unix.cmxa ocamlcommon.cmxa
${mllRule}
${mllList('ext',['ext_json_parse.mll'])}
rule mk_shared
command = $ocamlopt -I +compiler-libs -shared $flags -o $out $in
build ../odoc_gen/generator.cmxs : mk_shared ../odoc_gen/generator.mli ../odoc_gen/generator.ml
flags = -I +ocamldoc -I ../odoc_gen -absname
`
/**
* @type { {name : string, libs: string[]}[]}
*/
var libs = []
sourceDirs.forEach(name => {
if (name !== 'main' && name !== 'stubs') {
libs.push({ name, libs: [] })
}
})
fs.writeFile(path.join(jscompDir, cppoNinjaFile), cppoNative, 'ascii', function (err) {
if (err !== null) {
throw err
}
cp.execSync(`ninja -f ${cppoNinjaFile}`, { cwd: jscompDir , stdio:[0,1,2], encoding : 'utf8'})
/**
* @type{string[]}
*/
var files = []
for (let dir of sourceDirs) {
files = files.concat(test(dir))
}
var out = cp.execSync(`${getOcamldepFile()} -one-line -native ${includes} ${files.join(' ')}`, { cwd: jscompDir, encoding: 'ascii' })
/**
* @type {Map<string,Set<string>>}
*/
var map = new Map()
var pairs = out.split('\n').map(x => x.split(':').map(x => x.trim()))
pairs.forEach(pair => {
var deps
var key = pair[0]
if (pair[1] !== undefined && (deps = pair[1].trim())) {
deps = deps.split(' ')
map.set(key, new Set(deps))
}
if (key.endsWith('cmx')) {
libs.forEach(x => {
if (key.startsWith(x.name)) {
x.libs.push(key)
}
})
}
})
// not ocamldep output
// when no mli exists no deps for cmi otherwise add cmi
var stmts = pairs.map((pair) => {
if (pair[0]) {
var target = pair[0]
var y = path.parse(target)
/**
* @type {Set<string>}
*/
var deps = map.get(target) || new Set()
if (y.ext === '.cmx') {
var intf = path.join(y.dir, y.name + ".cmi")
var ml = path.join(y.dir, y.name + '.ml')
return `build ${deps.has(intf) ? target : [target, intf].join(' ')} : optc ${ml} | ${setSortedToString(deps)}`
} else {
// === 'cmi'
var mli = path.join(y.dir, y.name + '.mli')
return `build ${target} : optc ${mli} | ${setSortedToString(deps)}`
}
}
})
libs.forEach(x => {
var output = sortFilesByDeps(x.libs, map)
var name = x.name
stmts.push(`build ${name}/${name}.cmxa : archive ${output.join(' ')}`)
})
writeFile(path.join(jscompDir, ninjaOutput),
templateNative +
stmts.join('\n') +
'\n'
)
})
}