@shexjs/util
Version:
Shape Expressions validation and utilities.
1,338 lines (1,234 loc) • 71 kB
JavaScript
// **ShExUtil** provides ShEx utility functions
const ShExUtilCjsModule = (function () {
const ShExTerm = require("@shexjs/term");
const {ShExVisitor, ShExIndexVisitor} = require('@shexjs/visitor')
const Hierarchy = require('hierarchy-closure')
const ShExHumanErrorWriter = require('./shex-human-error-writer.js')
const SX = {};
SX._namespace = "http://www.w3.org/ns/shex#";
["Schema", "@context", "imports", "startActs", "start", "shapes",
"ShapeDecl", "ShapeOr", "ShapeAnd", "shapeExprs", "nodeKind",
"NodeConstraint", "iri", "bnode", "nonliteral", "literal", "datatype", "length", "minlength", "maxlength", "pattern", "flags", "mininclusive", "minexclusive", "maxinclusive", "maxexclusive", "totaldigits", "fractiondigits", "values",
"ShapeNot", "shapeExpr",
"Shape", "abstract", "closed", "extra", "expression", "extends", "restricts", "semActs",
"ShapeRef", "reference", "ShapeExternal",
"EachOf", "OneOf", "expressions", "min", "max", "annotation",
"TripleConstraint", "inverse", "negated", "predicate", "valueExpr",
"Inclusion", "include", "Language", "languageTag",
"IriStem", "LiteralStem", "LanguageStem", "stem",
"IriStemRange", "LiteralStemRange", "LanguageStemRange", "exclusion",
"Wildcard", "SemAct", "name", "code",
"Annotation", "object"].forEach(p => {
SX[p] = SX._namespace+p;
});
const RDF = {};
RDF._namespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
["type", "first", "rest", "nil"].forEach(p => {
RDF[p] = RDF._namespace+p;
});
const OWL = {}
OWL._namespace = "http://www.w3.org/2002/07/owl#";
["Thing"].forEach(p => {
OWL[p] = OWL._namespace+p;
});
const Missed = {}; // singleton
const UNBOUNDED = -1;
function extend (base) {
if (!base) base = {};
for (let i = 1, l = arguments.length, arg; i < l && (arg = arguments[i] || {}); i++)
for (let name in arg)
base[name] = arg[name];
return base;
}
function isShapeRef (expr) {
return typeof expr === "string" // test for JSON-LD @ID
}
let isInclusion = isShapeRef;
class MissingReferenceError extends Error {
constructor (ref, labelStr, known) {
super(`Structural error: reference to ${ref} not found in ${labelStr}`);
this.reference = ref;
this.known = known;
}
/** append directly after `error.message`
*/
notFoundIn () {
return ":\n" + this.known.map(
u => u.substr(0, 2) === '_:' ? u : '<' + u + '>'
).join("\n ") + ".";
}
}
class MissingDeclRefError extends MissingReferenceError {
constructor (ref, known) {
super(ref, "shape declarations", known);
}
}
class MissingTripleExprRefError extends MissingReferenceError {
constructor (ref, known) {
super(ref, "triple expressions", known);
}
}
const ShExUtil = {
SX: SX,
RDF: RDF,
version: function () {
return "0.5.0";
},
/* getAST - compile a traditional regular expression abstract syntax tree.
* Tested but not used at present.
*/
getAST: function (schema) {
return {
type: "AST",
shapes: schema.shapes.reduce(function (ret, shape) {
ret[shape.id] = {
type: "ASTshape",
expression: _compileShapeToAST(shape.shapeExpr.expression, [], schema)
};
return ret;
}, {})
};
/* _compileShapeToAST - compile a shape expression to an abstract syntax tree.
*
* currently tested but not used.
*/
function _compileShapeToAST (expression, tripleConstraints, schema) {
function Epsilon () {
this.type = "Epsilon";
}
function TripleConstraint (ordinal, predicate, inverse, negated, valueExpr) {
this.type = "TripleConstraint";
// this.ordinal = ordinal; @@ does 1card25
this.inverse = !!inverse;
this.negated = !!negated;
this.predicate = predicate;
if (valueExpr !== undefined)
this.valueExpr = valueExpr;
}
function Choice (disjuncts) {
this.type = "Choice";
this.disjuncts = disjuncts;
}
function EachOf (conjuncts) {
this.type = "EachOf";
this.conjuncts = conjuncts;
}
function SemActs (expression, semActs) {
this.type = "SemActs";
this.expression = expression;
this.semActs = semActs;
}
function KleeneStar (expression) {
this.type = "KleeneStar";
this.expression = expression;
}
function _compileExpression (expr, schema) {
let repeated, container;
/* _repeat: map expr with a min and max cardinality to a corresponding AST with Groups and Stars.
expr 1 1 => expr
expr 0 1 => Choice(expr, Eps)
expr 0 3 => Choice(EachOf(expr, Choice(EachOf(expr, Choice(expr, EPS)), Eps)), Eps)
expr 2 5 => EachOf(expr, expr, Choice(EachOf(expr, Choice(EachOf(expr, Choice(expr, EPS)), Eps)), Eps))
expr 0 * => KleeneStar(expr)
expr 1 * => EachOf(expr, KleeneStar(expr))
expr 2 * => EachOf(expr, expr, KleeneStar(expr))
@@TODO: favor Plus over Star if Epsilon not in expr.
*/
function _repeat (expr, min, max) {
if (min === undefined) { min = 1; }
if (max === undefined) { max = 1; }
if (min === 1 && max === 1) { return expr; }
const opts = max === UNBOUNDED ?
new KleeneStar(expr) :
Array.from(Array(max - min)).reduce(function (ret, elt, ord) {
return ord === 0 ?
new Choice([expr, new Epsilon]) :
new Choice([new EachOf([expr, ret]), new Epsilon]);
}, undefined);
const reqd = min !== 0 ?
new EachOf(Array.from(Array(min)).map(function (ret) {
return expr; // @@ something with ret
}).concat(opts)) : opts;
return reqd;
}
if (typeof expr === "string") { // Inclusion
const included = schema._index.tripleExprs[expr].expression;
return _compileExpression(included, schema);
}
else if (expr.type === "TripleConstraint") {
// predicate, inverse, negated, valueExpr, annotations, semActs, min, max
const valueExpr = "valueExprRef" in expr ?
schema.valueExprDefns[expr.valueExprRef] :
expr.valueExpr;
const ordinal = tripleConstraints.push(expr)-1;
const tp = new TripleConstraint(ordinal, expr.predicate, expr.inverse, expr.negated, valueExpr);
repeated = _repeat(tp, expr.min, expr.max);
return expr.semActs ? new SemActs(repeated, expr.semActs) : repeated;
}
else if (expr.type === "OneOf") {
container = new Choice(expr.expressions.map(function (e) {
return _compileExpression(e, schema);
}));
repeated = _repeat(container, expr.min, expr.max);
return expr.semActs ? new SemActs(repeated, expr.semActs) : repeated;
}
else if (expr.type === "EachOf") {
container = new EachOf(expr.expressions.map(function (e) {
return _compileExpression(e, schema);
}));
repeated = _repeat(container, expr.min, expr.max);
return expr.semActs ? new SemActs(repeated, expr.semActs) : repeated;
}
else throw Error("unexpected expr type: " + expr.type);
}
return expression ? _compileExpression(expression, schema) : new Epsilon();
}
},
// tests
// console.warn("HERE:", ShExJtoAS({"type":"Schema","shapes":[{"id":"http://all.example/S1","type":"Shape","expression":
// { "id":"http://all.example/S1e", "type":"EachOf","expressions":[ ] },
// // { "id":"http://all.example/S1e","type":"TripleConstraint","predicate":"http://all.example/p1"},
// "extra":["http://all.example/p3","http://all.example/p1","http://all.example/p2"]
// }]}).shapes['http://all.example/S1']);
ShExJtoAS: function (schema) {
const _ShExUtil = this;
// 2.1- > 2.2
const updated2_1to2_2 = (schema.shapes || []).reduce((acc, sh, ord) => {
if (sh.type === "ShapeDecl")
return acc;
const id = sh.id;
delete sh.id;
const newDecl = {
type: "ShapeDecl",
id: id,
shapeExpr: sh,
};
schema.shapes[ord] = newDecl;
return acc.concat([newDecl]);
}, []);
// if (updated2_1to2_2.length > 0)
// console.log("Updated 2.1 -> 2.2: " + updated2_1to2_2.map(decl => decl.id).join(", "));
schema._prefixes = schema._prefixes || { };
// schema._base = schema._prefixes || ""; // leave undefined to signal no provided base
schema._index = ShExIndexVisitor.index(schema);
return schema;
},
AStoShExJ: function (schema) {
schema["@context"] = schema["@context"] || "http://www.w3.org/ns/shex.jsonld";
delete schema["_index"];
delete schema["_prefixes"];
delete schema["_base"];
delete schema["_locations"];
delete schema["_sourceMap"];
return schema;
},
// tests
// const shexr = ShExUtil.ShExRtoShExJ({ "type": "Schema", "shapes": [
// { "id": "http://a.example/S1", "type": "Shape",
// "expression": {
// "type": "TripleConstraint", "predicate": "http://a.example/p1",
// "valueExpr": {
// "type": "ShapeAnd", "shapeExprs": [
// { "type": "NodeConstraint", "nodeKind": "bnode" },
// { "id": "http://a.example/S2", "type": "Shape",
// "expression": {
// "type": "TripleConstraint", "predicate": "http://a.example/p2" } }
// // "http://a.example/S2"
// ] } } },
// { "id": "http://a.example/S2", "type": "Shape",
// "expression": {
// "type": "TripleConstraint", "predicate": "http://a.example/p2" } }
// ] });
// console.warn("HERE:", shexr.shapes[0].expression.valueExpr);
// ShExUtil.ShExJtoAS(shexr);
// console.warn("THERE:", shexr.shapes["http://a.example/S1"].expression.valueExpr);
ShExRtoShExJ: function (schema) {
// compile a list of known shapeExprs
const knownShapeExprs = new Map();
if ("shapes" in schema)
schema.shapes.forEach(sh => knownShapeExprs.set(sh.id, null))
class ShExRVisitor extends ShExVisitor {
constructor (knownShapeExprs) {
super()
this.knownShapeExprs = knownShapeExprs;
this.knownTripleExpressions = {};
}
visitShapeExpr (expr, ...args) {
if (typeof expr === "string")
return expr;
if ("id" in expr) {
if (this.knownShapeExprs.has(expr.id) || Object.keys(expr).length === 1) {
const already = this.knownShapeExprs.get(expr.id);
if (typeof expr.expression === "object") {
if (!already)
this.knownShapeExprs.set(expr.id, super.visitShapeExpr(expr, label));
}
return expr.id;
}
delete expr.id;
}
return super.visitShapeExpr(expr, ...args);
};
visitTripleExpr (expr, ...args) {
if (typeof expr === "string") { // shortcut for recursive references e.g. 1Include1
return expr;
} else if ("id" in expr) {
if (expr.id in this.knownTripleExpressions) {
this.knownTripleExpressions[expr.id].refCount++;
return expr.id;
}
}
const ret = super.visitTripleExpr(expr, ...args);
// Everything from RDF has an ID, usually a BNode.
this.knownTripleExpressions[expr.id] = { refCount: 1, expr: ret };
return ret;
}
cleanIds () {
for (let k in this.knownTripleExpressions) {
const known = this.knownTripleExpressions[k];
if (known.refCount === 1 && known.expr.id.startsWith("_:"))
delete known.expr.id;
};
}
}
// normalize references to those shapeExprs
const v = new ShExRVisitor(knownShapeExprs);
if ("start" in schema)
schema.start = v.visitShapeExpr(schema.start);
if ("shapes" in schema)
schema.shapes = schema.shapes.map(sh => v.visitShapeDecl(sh));
// remove extraneous BNode IDs
v.cleanIds();
return schema;
},
valGrep: function (obj, type, f) {
const _ShExUtil = this;
const ret = [];
for (let i in obj) {
const o = obj[i];
if (typeof o === "object") {
if ("type" in o && o.type === type)
ret.push(f(o));
ret.push.apply(ret, _ShExUtil.valGrep(o, type, f));
}
}
return ret;
},
valToN3js: function (res, factory) {
return this.valGrep(res, "TestedTriple", function (t) {
const ret = JSON.parse(JSON.stringify(t));
if (typeof t.object === "object")
ret.object = ("\"" + t.object.value + "\"" + (
"type" in t.object ? "^^" + t.object.type :
"language" in t.object ? "@" + t.object.language :
""
));
return ret;
});
},
/* canonicalize: move all tripleExpression references to their first expression.
*
*/
canonicalize: function (schema, trimIRI) {
const ret = JSON.parse(JSON.stringify(schema));
ret["@context"] = ret["@context"] || "http://www.w3.org/ns/shex.jsonld";
delete ret._prefixes;
delete ret._base;
let index = ret._index || ShExIndexVisitor.index(schema);
delete ret._index;
let sourceMap = ret._sourceMap;
delete ret._sourceMap;
let locations = ret._locations;
delete ret._locations;
// Don't delete ret.productions as it's part of the AS.
class MyVisitor extends ShExVisitor {
constructor(index) {
super();
this.index = index;
this.knownExpressions = [];
}
visitInclusion (inclusion) {
if (this.knownExpressions.indexOf(inclusion) === -1 &&
inclusion in this.index.tripleExprs) {
this.knownExpressions.push(inclusion)
return super.visitTripleExpr(this.index.tripleExprs[inclusion]);
}
return super.visitInclusion(inclusion);
}
visitTripleExpr (expression) {
if (typeof expression === "object" && "id" in expression) {
if (this.knownExpressions.indexOf(expression.id) === -1) {
this.knownExpressions.push(expression.id)
return super.visitTripleExpr(this.index.tripleExprs[expression.id]);
}
return expression.id; // Inclusion
}
return super.visitTripleExpr(expression);
}
visitExtra (l) {
return l.slice().sort();
return ret;
}
}
v = new MyVisitor(index);
if (trimIRI) {
v.visitIRI = function (i) {
return i.replace(trimIRI, "");
}
if ("imports" in ret)
ret.imports = v.visitImports(ret.imports);
}
if ("shapes" in ret) {
ret.shapes = Object.keys(index.shapeExprs).map(k => {
if ("extra" in index.shapeExprs[k])
index.shapeExprs[k].extra.sort();
return v.visitShapeDecl(index.shapeExprs[k]);
});
}
return ret;
},
BiDiClosure: function () {
return {
needs: {},
neededBy: {},
inCycle: [],
test: function () {
function expect (l, r) { const ls = JSON.stringify(l), rs = JSON.stringify(r); if (ls !== rs) throw Error(ls+" !== "+rs); }
// this.add(1, 2); expect(this.needs, { 1:[2] }); expect(this.neededBy, { 2:[1] });
// this.add(3, 4); expect(this.needs, { 1:[2], 3:[4] }); expect(this.neededBy, { 2:[1], 4:[3] });
// this.add(2, 3); expect(this.needs, { 1:[2,3,4], 2:[3,4], 3:[4] }); expect(this.neededBy, { 2:[1], 3:[2,1], 4:[3,2,1] });
this.add(2, 3); expect(this.needs, { 2:[3] }); expect(this.neededBy, { 3:[2] });
this.add(1, 2); expect(this.needs, { 1:[2,3], 2:[3] }); expect(this.neededBy, { 3:[2,1], 2:[1] });
this.add(1, 3); expect(this.needs, { 1:[2,3], 2:[3] }); expect(this.neededBy, { 3:[2,1], 2:[1] });
this.add(3, 4); expect(this.needs, { 1:[2,3,4], 2:[3,4], 3:[4] }); expect(this.neededBy, { 3:[2,1], 2:[1], 4:[3,2,1] });
this.add(6, 7); expect(this.needs, { 6:[7] , 1:[2,3,4], 2:[3,4], 3:[4] }); expect(this.neededBy, { 7:[6] , 3:[2,1], 2:[1], 4:[3,2,1] });
this.add(5, 6); expect(this.needs, { 5:[6,7], 6:[7] , 1:[2,3,4], 2:[3,4], 3:[4] }); expect(this.neededBy, { 7:[6,5], 6:[5] , 3:[2,1], 2:[1], 4:[3,2,1] });
this.add(5, 7); expect(this.needs, { 5:[6,7], 6:[7] , 1:[2,3,4], 2:[3,4], 3:[4] }); expect(this.neededBy, { 7:[6,5], 6:[5] , 3:[2,1], 2:[1], 4:[3,2,1] });
this.add(7, 8); expect(this.needs, { 5:[6,7,8], 6:[7,8], 7:[8], 1:[2,3,4], 2:[3,4], 3:[4] }); expect(this.neededBy, { 7:[6,5], 6:[5], 8:[7,6,5], 3:[2,1], 2:[1], 4:[3,2,1] });
this.add(4, 5);
expect(this.needs, { 1:[2,3,4,5,6,7,8], 2:[3,4,5,6,7,8], 3:[4,5,6,7,8], 4:[5,6,7,8], 5:[6,7,8], 6:[7,8], 7:[8] });
expect(this.neededBy, { 2:[1], 3:[2,1], 4:[3,2,1], 5:[4,3,2,1], 6:[5,4,3,2,1], 7:[6,5,4,3,2,1], 8:[7,6,5,4,3,2,1] });
},
add: function (needer, needie, negated) {
const r = this;
if (!(needer in r.needs))
r.needs[needer] = [];
if (!(needie in r.neededBy))
r.neededBy[needie] = [];
// // [].concat.apply(r.needs[needer], [needie], r.needs[needie]). emitted only last element
r.needs[needer] = r.needs[needer].concat([needie], r.needs[needie]).
filter(function (el, ord, l) { return el !== undefined && l.indexOf(el) === ord; });
// // [].concat.apply(r.neededBy[needie], [needer], r.neededBy[needer]). emitted only last element
r.neededBy[needie] = r.neededBy[needie].concat([needer], r.neededBy[needer]).
filter(function (el, ord, l) { return el !== undefined && l.indexOf(el) === ord; });
if (needer in this.neededBy) this.neededBy[needer].forEach(function (e) {
r.needs[e] = r.needs[e].concat([needie], r.needs[needie]).
filter(function (el, ord, l) { return el !== undefined && l.indexOf(el) === ord; });
});
if (needie in this.needs) this.needs[needie].forEach(function (e) {
r.neededBy[e] = r.neededBy[e].concat([needer], r.neededBy[needer]).
filter(function (el, ord, l) { return el !== undefined && l.indexOf(el) === ord; })
});
// this.neededBy[needie].push(needer);
if (r.needs[needer].indexOf(needer) !== -1)
r.inCycle = r.inCycle.concat(r.needs[needer]);
},
trim: function () {
function _trim (a) {
// filter(function (el, ord, l) { return l.indexOf(el) === ord; })
for (let i = a.length-1; i > -1; --i)
if (a.indexOf(a[i]) < i)
a.splice(i, i+1);
}
for (k in this.needs)
_trim(this.needs[k]);
for (k in this.neededBy)
_trim(this.neededBy[k]);
},
foundIn: {},
addIn: function (tripleExpr, shapeExpr) {
this.foundIn[tripleExpr] = shapeExpr;
}
}
},
/** @@TODO tests
* options:
* no: don't do anything; just report nestable shapes
* transform: function to change shape labels
*/
nestShapes: function (schema, options = {}) {
const _ShExUtil = this;
const index = schema._index || ShExIndexVisitor.index(schema);
if (!('no' in options)) { options.no = false }
let shapeLabels = Object.keys(index.shapeExprs || [])
let shapeReferences = {}
shapeLabels.forEach(label => {
const shape = index.shapeExprs[label].shapeExpr
noteReference(label, null) // just note the shape so we have a complete list at the end
if (shape.type === 'Shape') {
if ('extends' in shape) {
shape.extends.forEach(
// !!! assumes simple reference, not e.g. AND
parent => noteReference(parent, shape)
)
}
if ('expression' in shape) {
(_ShExUtil.simpleTripleConstraints(shape) || []).forEach(tc => {
let target = _ShExUtil.getValueType(tc.valueExpr, true)
noteReference(target, {type: 'tc', shapeLabel: label, tc: tc})
})
}
} else if (shape.type === 'NodeConstraint') {
// can't have any refs to other shapes
} else {
throw Error('nestShapes currently only supports Shapes and NodeConstraints')
}
})
let nestables = Object.keys(shapeReferences).filter(
label => shapeReferences[label].length === 1
&& shapeReferences[label][0].type === 'tc' // no inheritance support yet
&& label in index.shapeExprs
&& index.shapeExprs[label].shapeExpr.type === 'Shape' // Don't nest e.g. valuesets for now. @@ needs an option
&& !index.shapeExprs[label].abstract // shouldn't have a ref to an unEXTENDed ABSTRACT shape anyways.
).filter(
nestable => !('noNestPattern' in options)
|| !nestable.match(RegExp(options.noNestPattern))
).reduce((acc, label) => {
acc[label] = {
referrer: shapeReferences[label][0].shapeLabel,
predicate: shapeReferences[label][0].tc.predicate
}
return acc
}, {})
if (!options.no) {
let oldToNew = {}
if (options.rename) {
if (!('transform' in options)) {
options.transform = (function () {
let map = shapeLabels.reduce((acc, k, idx) => {
acc[k] = '_:renamed' + idx
return acc
}, {})
return function (id, shapeExpr) {
return map[id]
}
})()
}
Object.keys(nestables).forEach(oldName => {
let shapeExpr = index.shapeExprs[oldName]
let newName = options.transform(oldName, shapeExpr)
oldToNew[oldName] = shapeExpr.id = newName
shapeLabels[shapeLabels.indexOf(oldName)] = newName
nestables[newName] = nestables[oldName]
nestables[newName].was = oldName
delete nestables[oldName]
// @@ maybe update index when done?
index.shapeExprs[newName] = index.shapeExprs[oldName]
delete index.shapeExprs[oldName]
if (shapeReferences[oldName].length !== 1) { throw Error('assertion: ' + oldName + ' doesn\'t have one reference: [' + shapeReferences[oldName] + ']') }
let ref = shapeReferences[oldName][0]
if (ref.type === 'tc') {
if (typeof ref.tc.valueExpr === 'string') { // ShapeRef
ref.tc.valueExpr = newName
} else {
throw Error('assertion: rename not implemented for TripleConstraint expr: ' + ref.tc.valueExpr)
// _ShExUtil.setValueType(ref, newName)
}
} else if (ref.type === 'Shape') {
throw Error('assertion: rename not implemented for Shape: ' + ref)
} else {
throw Error('assertion: ' + ref.type + ' not TripleConstraint or Shape')
}
})
Object.keys(nestables).forEach(k => {
let n = nestables[k]
if (n.referrer in oldToNew) {
n.newReferrer = oldToNew[n.referrer]
}
})
// Restore old order for more concise diffs.
let shapesCopy = {}
shapeLabels.forEach(label => shapesCopy[label] = index.shapeExprs[label])
index.shapeExprs = shapesCopy
} else {
const doomed = []
const ids = schema.shapes.map(s => s.id)
Object.keys(nestables).forEach(oldName => {
const borged = index.shapeExprs[oldName].shapeExpr
// In principle, the ShExJ shouldn't have a Decl if the above criteria are met,
// but the ShExJ may be generated by something which emits Decls regardless.
shapeReferences[oldName][0].tc.valueExpr = borged
const delme = ids.indexOf(oldName)
if (schema.shapes[delme].id !== oldName)
throw Error('assertion: found ' + schema.shapes[delme].id + ' instead of ' + oldName)
doomed.push(delme)
delete index.shapeExprs[oldName]
})
doomed.sort((l, r) => r - l).forEach(delme => {
const id = schema.shapes[delme].id
if (!nestables[id])
throw Error('deleting unexpected shape ' + id)
delete schema.shapes[delme].id
schema.shapes.splice(delme, 1)
})
}
}
// console.dir(nestables)
// console.dir(shapeReferences)
return nestables
function noteReference (id, reference) {
if (!(id in shapeReferences)) {
shapeReferences[id] = []
}
if (reference) {
shapeReferences[id].push(reference)
}
}
},
/** @@TODO tests
*
*/
getPredicateUsage: function (schema, untyped = {}) {
const _ShExUtil = this;
// populate shapeHierarchy
let shapeHierarchy = Hierarchy.create()
Object.keys(schema.shapes).forEach(label => {
let shapeExpr = schema.shapes[label].shapeExpr
if (shapeExpr.type === 'Shape') {
(shapeExpr.extends || []).forEach(
superShape => shapeHierarchy.add(superShape.reference, label)
)
}
})
Object.keys(schema.shapes).forEach(label => {
if (!(label in shapeHierarchy.parents))
shapeHierarchy.parents[label] = []
})
let predicates = { } // IRI->{ uses: [shapeLabel], commonType: shapeExpr }
Object.keys(schema.shapes).forEach(shapeLabel => {
let shapeExpr = schema.shapes[shapeLabel].shapeExpr
if (shapeExpr.type === 'Shape') {
let tcs = _ShExUtil.simpleTripleConstraints(shapeExpr) || []
tcs.forEach(tc => {
let newType = _ShExUtil.getValueType(tc.valueExpr)
if (!(tc.predicate in predicates)) {
predicates[tc.predicate] = {
uses: [shapeLabel],
commonType: newType,
polymorphic: false
}
if (typeof newType === 'object') {
untyped[tc.predicate] = {
shapeLabel,
predicate: tc.predicate,
newType,
references: []
}
}
} else {
predicates[tc.predicate].uses.push(shapeLabel)
let curType = predicates[tc.predicate].commonType
if (typeof curType === 'object' || curType === null) {
// another use of a predicate with no commonType
// console.warn(`${shapeLabel} ${tc.predicate}:${newType} uses untypable predicate`)
untyped[tc.predicate].references.push({ shapeLabel, newType })
} else if (typeof newType === 'object') {
// first use of a predicate with no detectable commonType
predicates[tc.predicate].commonType = null
untyped[tc.predicate] = {
shapeLabel,
predicate: tc.predicate,
curType,
newType,
references: []
}
} else if (curType === newType) {
; // same type again
} else if (shapeHierarchy.parents[curType] && shapeHierarchy.parents[curType].indexOf(newType) !== -1) {
predicates[tc.predicate].polymorphic = true; // already covered by current commonType
} else {
let idx = shapeHierarchy.parents[newType] ? shapeHierarchy.parents[newType].indexOf(curType) : -1
if (idx === -1) {
let intersection = shapeHierarchy.parents[curType]
? shapeHierarchy.parents[curType].filter(
lab => -1 !== shapeHierarchy.parents[newType].indexOf(lab)
)
: []
if (intersection.length === 0) {
untyped[tc.predicate] = {
shapeLabel,
predicate: tc.predicate,
curType,
newType,
references: []
}
// console.warn(`${shapeLabel} ${tc.predicate} : ${newType} isn\'t related to ${curType}`)
predicates[tc.predicate].commonType = null
} else {
predicates[tc.predicate].commonType = intersection[0]
predicates[tc.predicate].polymorphic = true
}
} else {
predicates[tc.predicate].commonType = shapeHierarchy.parents[newType][idx]
predicates[tc.predicate].polymorphic = true
}
}
}
})
}
})
return predicates
},
/** @@TODO tests
*
*/
simpleTripleConstraints: function (shape) {
if (!('expression' in shape)) {
return []
}
if (shape.expression.type === 'TripleConstraint') {
return [ shape.expression ]
}
if (shape.expression.type === 'EachOf' &&
!(shape.expression.expressions.find(
expr => expr.type !== 'TripleConstraint'
))) {
return shape.expression.expressions
}
throw Error('can\'t (yet) express ' + JSON.stringify(shape))
},
getValueType: function (valueExpr) {
if (typeof valueExpr === 'string') { return valueExpr }
if (valueExpr.reference) { return valueExpr.reference }
if (valueExpr.nodeKind === 'iri') { return OWL.Thing } // !! push this test to callers
if (valueExpr.datatype) { return valueExpr.datatype }
// if (valueExpr.extends && valueExpr.extends.length === 1) { return valueExpr.extends[0] }
return valueExpr // throw Error('no value type for ' + JSON.stringify(valueExpr))
},
/** getDependencies: find which shappes depend on other shapes by inheritance
* or inclusion.
* TODO: rewrite in terms of Visitor.
*/
getDependencies: function (schema, ret) {
ret = ret || this.BiDiClosure();
(schema.shapes || []).forEach(function (shapeDecl) {
function _walkShapeExpression (shapeExpr, negated) {
if (typeof shapeExpr === "string") { // ShapeRef
ret.add(shapeDecl.id, shapeExpr);
} else if (shapeExpr.type === "ShapeOr" || shapeExpr.type === "ShapeAnd") {
shapeExpr.shapeExprs.forEach(function (expr) {
_walkShapeExpression(expr, negated);
});
} else if (shapeExpr.type === "ShapeNot") {
_walkShapeExpression(shapeExpr.shapeExpr, negated ^ 1); // !!! test negation
} else if (shapeExpr.type === "Shape") {
_walkShape(shapeExpr, negated);
} else if (shapeExpr.type === "NodeConstraint") {
// no impact on dependencies
} else if (shapeExpr.type === "ShapeExternal") {
} else
throw Error("expected Shape{And,Or,Ref,External} or NodeConstraint in " + JSON.stringify(shapeExpr));
}
function _walkShape (shape, negated) {
function _walkTripleExpression (tripleExpr, negated) {
function _exprGroup (exprs, negated) {
exprs.forEach(function (nested) {
_walkTripleExpression(nested, negated) // ?? negation allowed?
});
}
function _walkTripleConstraint (tc, negated) {
if (tc.valueExpr)
_walkShapeExpression(tc.valueExpr, negated);
if (negated && ret.inCycle.indexOf(shapeDecl.id) !== -1) // illDefined/negatedRefCycle.err
throw Error("Structural error: " + shapeDecl.id + " appears in negated cycle");
}
if (typeof tripleExpr === "string") { // Inclusion
ret.add(shapeDecl.id, tripleExpr);
} else {
if ("id" in tripleExpr)
ret.addIn(tripleExpr.id, shapeDecl.id)
if (tripleExpr.type === "TripleConstraint") {
_walkTripleConstraint(tripleExpr, negated);
} else if (tripleExpr.type === "OneOf" || tripleExpr.type === "EachOf") {
_exprGroup(tripleExpr.expressions);
} else {
throw Error("expected {TripleConstraint,OneOf,EachOf,Inclusion} in " + tripleExpr);
}
}
}
(["extends", "restricts"]).forEach(attr => {
if (shape[attr] && shape[attr].length > 0)
shape[attr].forEach(function (i) {
ret.add(shapeDecl.id, i);
});
})
if (shape.expression)
_walkTripleExpression(shape.expression, negated);
}
_walkShapeExpression(shapeDecl.shapeExpr, 0); // 0 means false for bitwise XOR
});
return ret;
},
/** partition: create subset of a schema with only desired shapes and
* their dependencies.
*
* @schema: input schema
* @partition: shape name or array of desired shape names
* @deps: (optional) dependency tree from getDependencies.
* map(shapeLabel -> [shapeLabel])
*/
partition: function (schema, includes, deps, cantFind) {
const inputIndex = schema._index || ShExIndexVisitor.index(schema)
const outputIndex = { shapeExprs: new Map(), tripleExprs: new Map() };
includes = includes instanceof Array ? includes : [includes];
// build dependency tree if not passed one
deps = deps || this.getDependencies(schema);
cantFind = cantFind || function (what, why) {
throw new Error("Error: can't find shape " +
(why ?
why + " dependency " + what :
what));
};
const partition = {};
for (let k in schema)
partition[k] = k === "shapes" ? [] : schema[k];
includes.forEach(function (i) {
if (i in outputIndex.shapeExprs) {
// already got it.
} else if (i in inputIndex.shapeExprs) {
const adding = inputIndex.shapeExprs[i];
partition.shapes.push(adding);
outputIndex.shapeExprs[adding.id] = adding;
if (i in deps.needs)
deps.needs[i].forEach(function (n) {
// Turn any needed TE into an SE.
if (n in deps.foundIn)
n = deps.foundIn[n];
if (n in outputIndex.shapeExprs) {
} else if (n in inputIndex.shapeExprs) {
const needed = inputIndex.shapeExprs[n];
partition.shapes.push(needed);
outputIndex.shapeExprs[needed.id] = needed;
} else
cantFind(n, i);
});
} else {
cantFind(i, "supplied");
}
});
return partition;
},
/** @@TODO flatten: return copy of input schema with all shape and value class
* references substituted by a copy of their referent.
*
* @schema: input schema
*/
flatten: function (schema, deps, cantFind) {
const v = new ShExVisitor();
return v.visitSchema(schema);
},
// @@ put predicateUsage here
emptySchema: function () {
return {
type: "Schema"
};
},
absolutizeResults: function (parsed, base) {
// !! duplicate of Validation-test.js:84: const referenceResult = parseJSONFile(resultsFile...)
function mapFunction (k, obj) {
// resolve relative URLs in results file
if (["shape", "reference", "node", "subject", "predicate", "object"].indexOf(k) !== -1 &&
(typeof obj[k] === "string" && !obj[k].startsWith("_:"))) { // !! needs ShExTerm.ldTermIsIri
obj[k] = new URL(obj[k], base).href;
}}
function resolveRelativeURLs (obj) {
Object.keys(obj).forEach(function (k) {
if (typeof obj[k] === "object") {
resolveRelativeURLs(obj[k]);
}
if (mapFunction) {
mapFunction(k, obj);
}
});
}
resolveRelativeURLs(parsed);
return parsed;
},
getProofGraph: function (res, db, dataFactory) {
function _dive1 (solns) {
if (solns.type === "NodeConstraintTest") {
} else if (solns.type === "SolutionList" ||
solns.type === "ShapeAndResults" ||
solns.type === "ExtensionResults") {
solns.solutions.forEach(s => {
if (s.solution) // no .solution for <S> {}
_dive1(s.solution);
});
} else if (solns.type === "ShapeOrResults") {
_dive1(solns.solution);
} else if (solns.type === "ShapeTest") {
if ("solution" in solns)
_dive1(solns.solution);
} else if (solns.type === "OneOfSolutions" ||
solns.type === "EachOfSolutions") {
solns.solutions.forEach(s => {
_dive1(s);
});
} else if (solns.type === "OneOfSolution" ||
solns.type === "EachOfSolution") {
solns.expressions.forEach(s => {
_dive1(s);
});
} else if (solns.type === "TripleConstraintSolutions") {
solns.solutions.map(s => {
if (s.type !== "TestedTriple")
throw Error("unexpected result type: " + s.type);
const subject = ShExTerm.ld2RdfJsTerm(s.subject);
const predicate = ShExTerm.ld2RdfJsTerm(s.predicate);
const object = ShExTerm.ld2RdfJsTerm(s.object);
const graph = "graph" in s ? ShExTerm.ld2RdfJsTerm(s.graph) : dataFactory.defaultGraph();
db.addQuad(dataFactory.quad(subject, predicate, object, graph));
if ("referenced" in s) {
_dive1(s.referenced);
}
});
} else if (solns.type === "ExtendedResults") {
_dive1(solns.extensions);
if ("local" in solns)
_dive1(solns.local);
} else if (["ShapeNotResults", "Recursion"].indexOf(solns.type) !== -1) {
} else {
throw Error("unexpected expr type "+solns.type+" in " + JSON.stringify(solns));
}
}
_dive1(res);
return db;
},
MissingReferenceError,
MissingDeclRefError,
MissingTripleExprRefError,
HierarchyVisitor: function (schemaP, optionsP, negativeDepsP, positiveDepsP) {
const visitor = new SchemaStructureValidator(schemaP, optionsP, negativeDepsP, positiveDepsP);
return visitor;
},
validateSchema: function (schema, options) { // obselete, but may need other validations in the future.
// Stand-alone class but left in function scope to minimize symbol space
class SchemaStructureValidator extends ShExVisitor {
constructor (schema, options, negativeDeps, positiveDeps) {
super();
this.schema = schema;
this.options = options;
this.negativeDeps = negativeDeps;
this.positiveDeps = positiveDeps;
this.currentLabel = null;
this.currentExtra = null;
this.currentNegated = false;
this.inTE = false;
this.index = schema.index || ShExIndexVisitor.index(schema);
}
visitShape (shape, ...args) {
const lastExtra = this.currentExtra;
this.currentExtra = shape.extra;
const ret = super.visitShape(shape, ...args);
this.currentExtra = lastExtra;
return ret;
};
visitShapeNot (shapeNot, ...args) {
const lastNegated = this.currentNegated;
this.currentNegated ^= true;
const ret = super.visitShapeNot(shapeNot, ...args);
this.currentNegated = lastNegated;
return ret;
};
visitTripleConstraint (expr, ...args) {
const lastNegated = this.currentNegated;
if (this.currentExtra && this.currentExtra.indexOf(expr.predicate) !== -1)
this.currentNegated ^= true;
this.inTE = true;
const ret = super.visitTripleConstraint(expr, ...args);
this.inTE = false;
this.currentNegated = lastNegated;
return ret;
};
visitShapeRef (shapeRef, ...args) {
if (!(shapeRef in this.index.shapeExprs)) {
const error = this.firstError(new MissingDeclRefError(shapeRef, Object.keys(this.index.shapeExprs)), shapeRef);
if (this.options.missingReferent) {
this.options.missingReferent(error, (this.schema._locations || {})[this.currentLabel]);
} else {
throw error;
}
}
if (!this.inTE && shapeRef === this.currentLabel)
throw this.firstError(Error("Structural error: circular reference to " + this.currentLabel + "."), shapeRef);
if (!this.options.skipCycleCheck)
(this.currentNegated ? this.negativeDeps : this.positiveDeps).add(this.currentLabel, shapeRef);
return super.visitShapeRef(shapeRef, ...args);
};
visitInclusion (inclusion, ...args) {
let refd;
if (!(refd = this.index.tripleExprs[inclusion]))
throw this.firstError(new MissingTripleExprRefError(inclusion, Object.keys(this.index.tripleExprs)), inclusion);
// if (refd.type !== "Shape")
// throw Error("Structural error: " + inclusion + " is not a simple shape.");
return super.visitInclusion(inclusion, ...args);
};
firstError(e, obj) {
if ("_sourceMap" in this.schema)
e.location = (this.schema._sourceMap.get(obj) || [undefined])[0];
return e;
}
static validate (schema, options) {
const negativeDeps = Hierarchy.create();
const positiveDeps = Hierarchy.create();
const visitor = new SchemaStructureValidator(schema, options, negativeDeps, positiveDeps);
(schema.shapes || []).forEach(function (shape) {
visitor.currentLabel = shape.id;
visitor.visitShapeDecl(shape, shape.id);
visitor.currentLabel = null;
});
let circs = Object.keys(negativeDeps.children).filter(
k => negativeDeps.children[k].filter(
k2 => k2 in negativeDeps.children && negativeDeps.children[k2].indexOf(k) !== -1
|| k2 in positiveDeps.children && positiveDeps.children[k2].indexOf(k) !== -1
).length > 0
);
if (circs.length)
throw visitor.firstError(Error("Structural error: circular negative dependencies on " + circs.join(',') + "."), circs[0]);
}
}
SchemaStructureValidator.validate(schema, options);
},
/** isWellDefined: assert that schema is well-defined.
*
* @schema: input schema
* @@TODO
*/
isWellDefined: function (schema, options) {
this.validateSchema(schema, options);
// const deps = this.getDependencies(schema);
return schema;
},
walkVal: function (val, cb) {
const _ShExUtil = this;
if (typeof val === "string") { // ShapeRef
return null; // 1NOTRefOR1dot_pass-inOR
}
switch (val.type) {
case "SolutionList": // dependent_shape
return val.solutions.reduce((ret, exp) => {
const n = _ShExUtil.walkVal(exp, cb);
if (n)
Object.keys(n).forEach(k => {
if (k in ret)
ret[k] = ret[k].concat(n[k]);
else
ret[k] = n[k];
})
return ret;
}, {});
case "NodeConstraintTest": // 1iri_pass-iri
return _ShExUtil.walkVal(val.shapeExpr, cb);
case "NodeConstraint": // 1iri_pass-iri
return null;
case "ShapeTest": // 0_empty
const vals = [];
visitSolution(val, vals); // A ShapeTest is a sort of Solution.
const ret = vals.length
? {'http://shex.io/reflex': vals}
: {};
if ("solution" in val)
Object.assign(ret, _ShExUtil.walkVal(val.solution, cb))
return Object.keys(ret).length ? ret : null;
case "Shape": // 1NOTNOTdot_passIv1
return null;
case "ShapeNotTest": // 1NOT_vsANDvs__passIv1
return _ShExUtil.walkVal(val.shapeExpr, cb);
case "ShapeNotResults": // NOT1dotOR2dot_pass-empty
return null; // we don't bind variables from negative tests
case "Failure": // NOT1dotOR2dot_pass-empty
return null; // !!TODO
case "ShapeNot": // 1NOTNOTIRI_passIo1,
return _ShExUtil.walkVal(val.shapeExpr, cb);
case "ShapeOrResults": // 1dotRefOR3_passShape1
return _ShExUtil.walkVal(val.solution, cb);
case "ShapeOr": // 1NOT_literalORvs__passIo1
return val.shapeExprs.reduce((ret, exp) => {
const n = _ShExUtil.walkVal(exp, cb);
if (n)
Object.keys(n).forEach(k => {
if (k in ret)
ret[k] = ret[k].concat(n[k]);
else
ret[k] = n[k];
})
return ret;
}, {});
case "ShapeAndResults": // 1iriRef1_pass-iri
case "ExtensionResults": // extends-abstract-multi-empty_pass-missingOptRef1
return val.solutions.reduce((ret, exp) => {
const n = _ShExUtil.walkVal(exp, cb);
if (n)
Object.keys(n).forEach(k => {
if (k in ret)
ret[k] = ret[k].concat(n[k]);
else
ret[k] = n[k];
})
return ret;
}, {});
case "ShapeAnd": // 1NOT_literalANDvs__passIv1
return val.shapeExprs.reduce((ret, exp) => {
const n = _ShExUtil.walkVal(exp, cb);
if (n)
Object.keys(n).forEach(k => {
if (k in ret)
ret[k] = ret[k].concat(n[k]);
else
ret[k] = n[k];
})
return ret;
}, {});
case "ExtendedResults": // extends-abstract-multi-empty_pass-missingOptRef1
return (["extensions", "local"]).reduce((ret, exp) => {
const n = _ShExUtil.walkVal(exp, cb);
if (n)
Object.keys(n).forEach(k => {
if (k in ret)
ret[k] = ret[k].concat(n[k]);
else
ret[k] = n[k];
})
return ret;
}, {});
case "EachOfSolutions":
case "OneOfSolutions":
// 1dotOne2dot_pass_p1
return val.solutions.reduce((ret, sln) => {
sln.expressions.forEach(exp => {
const n = _ShExUtil.walkVal(exp, cb);
if (n)
Object.keys(n).forEach(k => {
if (k in ret)
ret[k] = ret[k].concat(n[k]);
else
ret[k] = n[k];
})
});
return ret;
}, {});
case "TripleConstraintSolutions": // 1dot_pass-noOthers
if ("solutions" in val) {
const ret = {};
const vals = [];
ret[val.predicate] = vals;
val.solutions.forEach(sln => visitSolution(sln, vals));
return vals.length ? ret : null;
} else {
return null;
}
case "Recursion": // 3circRefPlus1_pass-recursiveData
return null;
default:
// console.log(val);
throw Error("unknown shapeExpression type in " + JSON.stringify(val));
}
return val;
function visitSolution (sln, vals) {
const toAdd = [];
if (chaseList(sln.referenced, toAdd)) { // parse 1val1IRIREF.ttl
[].push.apply(vals, toAdd);
} else { // 1dot_pass-noOthers
const newElt = cb(sln) || {};
if ("referenced" in sln) {
const t = _ShExUtil.walkVal(sln.referenced, cb);
if (t)
newElt.nested = t;
}
if (Object.keys(newElt).length > 0)
vals.push(newElt);
}
function chaseList (li) {
if (!li) return false;
if (li.node === RDF.nil) return true;
if ("solution" in li && "solutions" in li.solution &&
li.solution.solutions.length === 1 &&
"expressions" in li.solution.solutions[0] &&
li.solution.solutions[0].expressions.length === 2 &&
"predicate" in li.solution.solutions[0].expressions[0] &&
li.solution.solutions[0].expressions[0].predicate === RDF.first &&
li.solution.solutions[0].expressions[1].predicate === RDF.rest) {
const expressions = li.solution.solutions[0].expressions;
const ent = expressions[0];
const rest = expressions[1].solutions[0];
const member = ent.solutions[0];
let newElt = cb(member);
if ("referenced" in member) {
const t = _ShExUtil.walkVal(member.referenced, cb);
if (t) {
if (newElt)
newElt.nested = t;
else
newElt = t;
}
}
if (newElt)
vals.push(newElt);
return rest.object === RDF.nil ?
true :
chaseList(rest.referenced.type === "ShapeOrResults" // heuristic for `nil OR @<list>` idiom
? rest.referenced.solution
: rest.referenced);
}
}
}
},
/**
* Convert val results to a property tree.
* @exports
* @returns {@code {p1:[{p2: v2},{p3: v3}]}}
*/
valToValues: function (val) {
return this.walkVal (val, function (sln) {
return "object" in sln ? { ldterm: sln.object } : null;
});
},
valToExtension: function (val, lookfor) {
const map = this.walkVal (val, function (sln) {
return "extensions" in sln ? { extensions: sln.extensions } : null;
});
function extensions (obj) {
const list = [];
let crushed = {};
function crush (elt) {
if (crushed === null)
return elt;
if (Array.isArray(elt)) {
crushed = null;
return elt;
}
for (k in elt) {
if (k in crushed) {
crushed = null
return elt;
}
crushed[k] = elt[k];
}
return elt;
}
for (let k in obj) {
if (k === "extensions") {
if (obj[k])
list.push(crush(obj[k][lookfor]));
} else if (k === "nested") {
const nested = extensions(obj[k]);
if (Array.isArray(nested))
nested.forEach(crush);
else
crush(nested);
list.push(nested);
} else {
list.push(crush(extensions(obj[k])));
}
}
return list.length === 1 ? li