rdf-dataset-fragmenter
Version:
Fragments an RDF dataset into multiple parts
107 lines • 8.57 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.DatasetSummaryBloom = void 0;
const bloem_1 = require("bloem");
const MurmurHash3 = require("imurmurhash");
const DatasetSummary_1 = require("./DatasetSummary");
class DatasetSummaryBloom extends DatasetSummary_1.DatasetSummary {
/* eslint-enable ts/naming-convention */
constructor(args) {
super(args);
this.location = args.location;
this.hashBits = args.hashBits;
this.hashCount = args.hashCount;
this.projectedProperties = new Map();
this.projectedResources = new Map();
}
register(quad) {
const subjectValue = quad.subject.termType === 'NamedNode' ? quad.subject.value : undefined;
const predicateValue = quad.predicate.termType === 'NamedNode' ? quad.predicate.value : undefined;
const objectValue = quad.object.termType === 'NamedNode' ? quad.object.value : undefined;
if (subjectValue && predicateValue) {
this.project(this.projectedResources, subjectValue, predicateValue);
this.project(this.projectedProperties, predicateValue, subjectValue);
}
if (subjectValue && objectValue) {
this.project(this.projectedResources, subjectValue, objectValue);
this.project(this.projectedResources, objectValue, subjectValue);
}
if (predicateValue && objectValue) {
this.project(this.projectedProperties, predicateValue, objectValue);
this.project(this.projectedResources, objectValue, predicateValue);
}
}
serialize() {
const output = [];
if (this.projectedProperties.size > 0 || this.projectedResources.size > 0) {
const projections = new Map([
[DatasetSummaryBloom.MEM_PROP_PROJECTEDPROPERTY, this.projectedProperties],
[DatasetSummaryBloom.MEM_PROP_PROJECTEDRESOURCE, this.projectedResources],
]);
const hashFunctionIri = DatasetSummaryBloom.createFragmentIri(this.location, this.dataset, DatasetSummaryBloom.MEM_CLASS_HASHFUNCTION.value);
output.push(DatasetSummary_1.DF.quad(hashFunctionIri, DatasetSummaryBloom.RDF_TYPE, DatasetSummaryBloom.MEM_CLASS_HASHFUNCTION), DatasetSummary_1.DF.quad(hashFunctionIri, DatasetSummaryBloom.RDF_TYPE, DatasetSummaryBloom.MEM_CLASS_HASHFUNCTIONFNV),
// Filter size is associated with the hash function
DatasetSummary_1.DF.quad(hashFunctionIri, DatasetSummaryBloom.MEM_PROP_HASHSIZE, DatasetSummary_1.DF.literal(this.hashCount.toString(10), DatasetSummaryBloom.XSD_INTEGER)));
for (const [projection, projectionMapping] of projections) {
for (const [projectedValue, filter] of projectionMapping) {
const bitfieldBase64 = filter.bitfield.toBuffer().toString('base64');
const collectionIri = DatasetSummaryBloom.createFragmentIri(this.location, this.dataset, DatasetSummaryBloom.MEM_CLASS_MEMBERCOLLECTION.value, projection.value, projectedValue);
output.push(DatasetSummary_1.DF.quad(collectionIri, DatasetSummaryBloom.RDF_TYPE, DatasetSummaryBloom.MEM_CLASS_MEMBERCOLLECTION), DatasetSummary_1.DF.quad(collectionIri, DatasetSummaryBloom.MEM_PROP_SOURCECOLLECTION, DatasetSummary_1.DF.namedNode(this.dataset)), DatasetSummary_1.DF.quad(collectionIri, projection, DatasetSummary_1.DF.namedNode(projectedValue)));
const filterIri = DatasetSummaryBloom.createFragmentIri(this.location, this.dataset, DatasetSummaryBloom.MEM_CLASS_BLOOMFILTER.value, projection.value, projectedValue);
output.push(DatasetSummary_1.DF.quad(filterIri, DatasetSummaryBloom.RDF_TYPE, DatasetSummaryBloom.MEM_CLASS_MEMBERSHIPFUNCTION), DatasetSummary_1.DF.quad(filterIri, DatasetSummaryBloom.RDF_TYPE, DatasetSummaryBloom.MEM_CLASS_APPROXIMATEMEMBERSHIPFUNCTION), DatasetSummary_1.DF.quad(filterIri, DatasetSummaryBloom.RDF_TYPE, DatasetSummaryBloom.MEM_CLASS_BLOOMFILTER), DatasetSummary_1.DF.quad(filterIri, DatasetSummaryBloom.MEM_PROP_HASHFUNCTION, hashFunctionIri), DatasetSummary_1.DF.quad(filterIri, DatasetSummaryBloom.MEM_PROP_MEMBERCOLLECTION, collectionIri),
// Filter bit size needs to be associated with the membership function
DatasetSummary_1.DF.quad(filterIri, DatasetSummaryBloom.MEM_PROP_BITSIZE, DatasetSummary_1.DF.literal(this.hashBits.toString(10), DatasetSummaryBloom.XSD_INTEGER)),
// The binary representation is associated with the membership function as base64
DatasetSummary_1.DF.quad(filterIri, DatasetSummaryBloom.MEM_PROP_BINARYREPRESENTATION, DatasetSummary_1.DF.literal(bitfieldBase64, DatasetSummaryBloom.XSD_BASE64)));
}
}
}
return { iri: this.location, quads: output };
}
project(map, key, value) {
let filter = map.get(key);
if (!filter) {
filter = new bloem_1.Bloem(this.hashBits, this.hashCount, Buffer.alloc(this.hashBits / 8));
map.set(key, filter);
}
filter.add(Buffer.from(value));
}
/**
* Create a fragment IRI on the provided base,
* using a hex digest of the hashes of all additional values provided.
*/
static createFragmentIri(base, ...values) {
const hash = MurmurHash3();
for (const value of values) {
hash.hash(value);
}
return DatasetSummary_1.DF.namedNode(`${base}#${hash.result().toString(16)}`);
}
}
exports.DatasetSummaryBloom = DatasetSummaryBloom;
/* eslint-disable ts/naming-convention */
DatasetSummaryBloom.MEM_PREFIX = 'http://semweb.mmlab.be/ns/membership#';
DatasetSummaryBloom.MEM_CLASS_MEMBERSHIPFUNCTION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}MembershipFunction`);
DatasetSummaryBloom.MEM_CLASS_APPROXIMATEMEMBERSHIPFUNCTION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}ApproximateMembershipFunction`);
DatasetSummaryBloom.MEM_CLASS_BLOOMFILTER = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}BloomFilter`);
DatasetSummaryBloom.MEM_CLASS_HASHFUNCTION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}HashFunction`);
DatasetSummaryBloom.MEM_CLASS_HASHFUNCTIONFNV = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}FowlerNollVo`);
DatasetSummaryBloom.MEM_CLASS_HASHFUNCTIONMD5 = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}MD5`);
DatasetSummaryBloom.MEM_CLASS_MEMBERCOLLECTION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}MemberCollection`);
DatasetSummaryBloom.MEM_PROP_SOURCECOLLECTION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}sourceCollection`);
DatasetSummaryBloom.MEM_PROP_MEMBERCOLLECTION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}memberCollection`);
DatasetSummaryBloom.MEM_PROP_HASHFUNCTION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}hashFunction`);
DatasetSummaryBloom.MEM_PROP_BINARYREPRESENTATION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}binaryRepresentation`);
DatasetSummaryBloom.MEM_PROP_BITSIZE = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}bitSize`);
DatasetSummaryBloom.MEM_PROP_HASHSIZE = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}hashSize`);
// The projectedProperty exists, projectedResource does not exist in the vocabulary.
// This generator for the Bloom filters assumes that the projectedProperty indicates which instance of rdf:Property,
// that is is the predicate value, is represented by the filter. The filter should then be applicable for triples or
// patterns with that same predicate value, for use in filtering rdf:Resource instances (subject and obect values).
// Following the same line of thought, the projectedResource is used to indicate which resource IRI the filter
// applies to, and can be used to test whether specific predicate values or other resource IRIs occur in triples
// alongside a given resource IRI (subject or object value).
// The filters will only consider named nodes (IRIs), and will ignore blank nodes, literals and variables.
DatasetSummaryBloom.MEM_PROP_PROJECTEDPROPERTY = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}projectedProperty`);
DatasetSummaryBloom.MEM_PROP_PROJECTEDRESOURCE = DatasetSummary_1.DF.namedNode(`${DatasetSummaryBloom.MEM_PREFIX}projectedResource`);
//# sourceMappingURL=DatasetSummaryBloom.js.map