@comyata/fe
Version:
Execute YAML and JSON files like programs and get clean data to work with as result.
349 lines (346 loc) • 13 kB
JavaScript
import { DataFile } from '@comyata/fe/DataFile';
import { DataFileRegistry, DataRef } from '@comyata/fe/DataFileRegistry';
import { CircularFileDependencyError, CircularProcessingDependencyError, ComputableFetchError } from '@comyata/fe/Errors';
import { ComputableError } from '@comyata/run/Errors';
import { isRelative } from '@comyata/fe/Helpers/isRelative';
import { timer } from '@comyata/run/Helpers/Timer';
import { Parser } from '@comyata/run/Parser';
import { runtime } from '@comyata/run/Runtime';
class SubscribeMap extends Map {
listeners = {
add: [],
delete: []
};
set(key, value) {
const map = super.set(key, value);
this.listeners.add.slice().forEach(l => l(key, value));
return map;
}
delete(key) {
const deleted = super.delete(key);
if (deleted) {
this.listeners.delete.slice().forEach(l => l(key));
}
return deleted;
}
clear() {
let keys;
if (this.listeners.delete.length) {
keys = Array.from(this.keys());
}
super.clear();
if (keys) {
this.listeners.delete.slice().forEach(l => keys?.forEach(key => l(key)));
}
}
onAdd(cb) {
this.listeners.add.push(cb);
return () => {
const i = this.listeners.add.indexOf(cb);
if (i === -1) return;
this.listeners.add.splice(i, 1);
};
}
onDelete(cb) {
this.listeners.delete.push(cb);
return () => {
const i = this.listeners.delete.indexOf(cb);
if (i === -1) return;
this.listeners.delete.splice(i, 1);
};
}
}
export class FileEngine {
files = new SubscribeMap();
constructor({
nodes,
compute,
importer,
processorOptions,
parserOptions
}) {
this.compute = compute;
this.parser = new Parser(nodes, parserOptions);
this.importer = importer;
this.processorOptions = processorOptions;
}
/**
* @todo risky method, only use outside of computations/runs, to not accidentally access the wrong files cache,
* safe to use outside of runs
*/
contextOf(contextBaseUrl) {
if (isRelative(contextBaseUrl)) {
// this is the special case, which depends on parent file or global resolve
throw new ComputableError(`Relative file not supported as ref: ${contextBaseUrl}`);
}
const source = this.files.get(contextBaseUrl);
if (source) {
if (source.importContext) {
const resolveRelative = source.importContext.resolveRelative;
if (resolveRelative) {
return {
resolveRelative: resolveRelative,
importer: source.importContext.importer
};
}
throw new ComputableError(`Source has no importContext for ${contextBaseUrl}`);
}
throw new ComputableError(`No source registered for ${contextBaseUrl}`);
}
const importer = this.importer?.match(contextBaseUrl);
if (importer) {
if (!('resolveDirectory' in importer) || !importer?.resolveDirectory) {
throw new ComputableError(`Importer ${importer.id} does not support directory context, required to load ${contextBaseUrl}`);
}
return importer.resolveDirectory(contextBaseUrl);
}
throw new ComputableError(`No importer registered for ${contextBaseUrl}`);
}
/**
* @todo maybe move into registry?
* @todo risky method, only use outside of computations/runs, to not accidentally access the wrong files cache;
* safe to use outside of runs
*/
fileRef(file, importContext) {
if (isRelative(file)) {
if (importContext?.resolveRelative) {
file = importContext.resolveRelative(file);
} else {
// todo: here it would be helpful to get the parentId,
// which could be undefined for in-memory register without parents that exist in registers
throw new ComputableError(`Relative file not supported for: ${JSON.stringify(file)}`);
}
}
const sourceRefExisting = this.files.get(file);
if (sourceRefExisting) return sourceRefExisting;
if (!this.importer) {
throw new ComputableError(`No importer registered, can not reference file: ${JSON.stringify(file)}`);
}
const importer = this.importer.match(file);
if (!importer) {
throw new ComputableError(`No importer matched for file: ${JSON.stringify(file)}`);
}
const fileContext = importer.resolveFile(file);
return this.newFile(file, fileContext);
}
register(id, objOrEval, importContext) {
const newFile = this.newFile(id, importContext);
newFile.value = {
current: objOrEval
};
newFile.node = this.parser.parse(objOrEval);
return newFile;
}
newFile(id, importContext) {
if (this.files.has(id)) {
throw new Error(`File already exists, can not overwrite ${id}`);
}
const dataFileRef = new DataFile(id, importContext);
this.files.set(id, dataFileRef);
return dataFileRef;
}
async fetchFile(dataFile,
//runtimeContext: Pick<RuntimeContext, 'refs' | 'fileLoader' | 'fileLoaderListener'>,
registry, statsList = []
// onLoaded: () => TNode,
) {
const dataRef = registry.refs.get(dataFile);
const stats = {
dur: 0,
step: 'load',
file: dataFile.fileId
};
statsList.push(stats);
if (dataRef?.value) {
stats.cached = 2;
return dataRef.value.current;
}
// when all other caches work correctly, this should never happen
// if(dataRef && dataFile?.value) {
// stats.cached = 3
// dataRef.value = dataFile?.value
// return dataRef.value.current
// }
const cached = registry.fileLoader.get(dataFile);
if (cached) {
const start = timer.start();
const r = await cached();
stats.cached = 1;
stats.dur = timer.end(start);
return r;
}
const loader = dataFile.importContext && 'load' in dataFile.importContext && dataFile.importContext.load ? dataFile.importContext.load : undefined;
if (loader) {
registry.fileLoader.set(dataFile, () => new Promise((resolve, reject) => {
let listener = registry.fileLoaderListener.get(dataFile);
if (!listener) {
listener = [];
registry.fileLoaderListener.set(dataFile, listener);
}
listener.push((result, err) => result ? resolve(result) : reject(err));
}));
const start = timer.start();
try {
const loaded = await loader();
const dataRef = DataRef.withValue(dataFile, loaded);
dataFile.value = dataRef.value;
registry.refs.set(dataFile, dataRef);
registry.fileLoader.delete(dataFile);
registry.fileLoaderListener.get(dataFile)?.forEach(l => l(dataRef.value?.current));
registry.fileLoaderListener.delete(dataFile);
stats.dur = timer.end(start);
stats.cached = 0;
return dataRef.value?.current;
} catch (e) {
registry.fileLoaderListener.get(dataFile)?.forEach(l => l(undefined, e));
registry.fileLoaderListener.delete(dataFile);
throw e;
}
}
throw new ComputableFetchError(dataFile, `File empty and without loader ${JSON.stringify(dataFile.fileId)}`);
}
run(dataFile, context = {},
// todo: combine baggage with this runtimeContext
runtimeContext = {
// todo: to support full temporary runs, without global cache,
// the fileRef must be reversed
registry: new DataFileRegistry(this.fileRef.bind(this), [dataFile]),
fileEvals: new Map(),
fileEvalsAwaiter: new Map(),
fileListener: new Map(),
usages: new Map()
}, {
filesChain = new Set(),
nodesChain = new Set(),
...baggage
} = {}) {
const computeStats = {
step: 'computeFile',
file: dataFile.fileId,
stats: [],
dur: 0,
usages: new Map()
};
const start = timer.start();
if (filesChain.has(dataFile)) {
throw new CircularFileDependencyError(filesChain, dataFile, `File import "${dataFile.fileId}" causes an import loop, not resolvable:\n\n` + Array.from(filesChain).map((f, i) => i + 1 + '. ' + JSON.stringify(f.fileId)).join('\n'));
}
const tryCached = () => {
const cached = runtimeContext.fileEvals.get(dataFile);
if (cached) {
if (typeof cached === 'function') {
return cached(filesChain, nodesChain).then(result => {
computeStats.cached = 1;
computeStats.dur = timer.end(start);
return [result, computeStats, runtimeContext];
});
}
computeStats.cached = 2;
computeStats.dur = timer.end(start);
return [cached.result, computeStats, runtimeContext];
}
return undefined;
};
const cachedRes = tryCached();
if (cachedRes) return Promise.resolve(cachedRes);
const lockFile = (lockingDataFile, dataEvalNode) => {
if (runtimeContext.fileEvals.has(lockingDataFile)) {
throw new ComputableError(`Eval context corrupted, fileEval already defined for ${JSON.stringify(lockingDataFile.fileId)}`);
}
runtimeContext.fileEvals.set(lockingDataFile, requestingFilesChain => new Promise((resolve, reject) => {
const lockingEvalAwaiter = runtimeContext.fileEvalsAwaiter.get(lockingDataFile);
for (const lastParentFile of requestingFilesChain) {
if (lockingEvalAwaiter?.has(lastParentFile)) {
reject(new CircularProcessingDependencyError(lastParentFile, lockingDataFile, `Circular file processing, target ${JSON.stringify(lockingDataFile.fileId)} already depends on ${JSON.stringify(lastParentFile.fileId)}`));
return;
}
}
if (requestingFilesChain?.has(lockingDataFile)) {
reject(new CircularFileDependencyError(requestingFilesChain, lockingDataFile, `requesting-circular XX-1 ${JSON.stringify(lockingDataFile.fileId)}\n\n`));
return;
}
let listener = runtimeContext.fileListener.get(lockingDataFile);
if (!listener) {
listener = [];
runtimeContext.fileListener.set(lockingDataFile, listener);
}
listener.push((result, err) => {
// if(lastParentFile) fileAwaiter.delete(lastParentFile)
return err ? reject(err) : resolve(result);
});
}));
return dataEvalNode;
};
const runProcess = dataEvalNode => {
const p = runtime(dataEvalNode, context, this.compute, this.processorOptions, {
nodesChain,
dataFile: dataFile,
...baggage,
filesChain: new Set(filesChain).add(dataFile),
fileEngine: this,
computeStats: computeStats,
runtimeContext: runtimeContext
});
return p.compute().then(fileRes => {
computeStats.stats.push(...p.stats);
computeStats.dur = timer.end(start);
computeStats.cached = 0;
return fileRes;
});
};
// todo: when .fileEvals is set, it must listen to all finishers and re-check caching,
// as multiple processors can currently come to `fetchFile`, leading to corrupted states
const handleRelease = prom => {
return prom.then(result => {
runtimeContext.fileEvals.set(dataFile, {
result: result
});
runtimeContext.fileListener.get(dataFile)?.forEach(l => l(result));
runtimeContext.fileListener.delete(dataFile);
return [result, computeStats, runtimeContext];
}).catch(e => {
runtimeContext.fileEvals.delete(dataFile);
runtimeContext.fileListener.get(dataFile)?.forEach(l => l(undefined, e));
runtimeContext.fileListener.delete(dataFile);
return Promise.reject(e);
});
};
const dataRef = (() => {
let dataRef = runtimeContext.registry.refs.get(dataFile);
if (dataRef) {
// dataRef.node ||= dataFile.node
// dataRef.value ||= dataFile.value
return dataRef;
}
dataRef = new DataRef(dataFile);
runtimeContext.registry.refs.set(dataFile, dataRef);
return dataRef;
})();
return dataRef.node ?
// this will always be in sync, thus doesn't need cache check
handleRelease(runProcess(lockFile(dataFile, dataRef.node))) : this.fetchFile(dataFile, runtimeContext.registry, computeStats.stats).then(value => {
// todo: maybe different tryCached for fetch and process, to not have deadlocks here?
const cachedRes = tryCached();
if (cachedRes) {
// console.log('after-fetch-cached', dataFile.fileId)
return cachedRes;
}
// todo: parse must only be done when locked? or as sync just this hard check?
const statsParse = {
dur: 0,
step: 'parse',
file: dataFile.fileId,
cached: 1
};
computeStats.stats.push(statsParse);
if (!dataRef.node) {
const startParse = timer.start();
dataRef.node = this.parser.parse(value);
dataFile.node = dataRef.node;
statsParse.dur = timer.end(startParse);
statsParse.cached = 0;
}
return handleRelease(runProcess(lockFile(dataFile, dataRef.node)));
});
}
}