rdf-dataset-fragmenter
Version:
Fragments an RDF dataset into multiple parts
136 lines • 9.49 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DatasetSummaryVoID = void 0;
const imurmurhash_1 = __importDefault(require("imurmurhash"));
const rdf_string_1 = require("rdf-string");
const DatasetSummary_1 = require("./DatasetSummary");
class DatasetSummaryVoID extends DatasetSummary_1.DatasetSummary {
// Entire dataset-related data
totalQuads;
distinctSubjects;
distinctObjects;
vocabularies;
// Property partition-related data
totalQuadsByPredicate;
distinctSubjectsByPredicate;
distinctObjectsByPredicate;
// Class partition-related data
entitiesByClass;
/* eslint-disable ts/naming-convention */
static VOID_PREFIX = 'http://rdfs.org/ns/void#';
static VOID_TRIPLES = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}triples`);
static VOID_ENTITIES = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}entities`);
static VOID_CLASS = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}class`);
static VOID_CLASSES = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}classes`);
static VOID_PROPERTY = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}property`);
static VOID_PROPERTIES = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}properties`);
static VOID_URISPACE = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}uriSpace`);
static VOID_DATASET = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}Dataset`);
static VOID_DISTINCT_SUBJECTS = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}distinctSubjects`);
static VOID_DISTINCT_OBJECTS = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}distinctObjects`);
static VOID_PROPERTY_PARTITION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}propertyPartition`);
static VOID_CLASS_PARTITION = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}classPartition`);
static VOID_VOCABULARY = DatasetSummary_1.DF.namedNode(`${DatasetSummaryVoID.VOID_PREFIX}vocabulary`);
static VOCABULARY_TRIM_REGEX = /#?(\w+)$/u;
/* eslint-enable ts/naming-convention */
constructor(args) {
super(args);
this.totalQuads = 0;
this.distinctSubjects = new Set();
this.distinctObjects = new Set();
this.vocabularies = new Set();
this.totalQuadsByPredicate = new Map();
this.distinctSubjectsByPredicate = new Map();
this.distinctObjectsByPredicate = new Map();
this.entitiesByClass = new Map();
}
register(quad) {
this.totalQuads++;
if (quad.subject.termType === 'NamedNode' || quad.subject.termType === 'BlankNode') {
const subjectString = (0, rdf_string_1.termToString)(quad.subject);
this.distinctSubjects.add(subjectString);
if (quad.predicate.termType === 'NamedNode') {
let predicateSubjects = this.distinctSubjectsByPredicate.get(quad.predicate.value);
if (!predicateSubjects) {
predicateSubjects = new Set();
this.distinctSubjectsByPredicate.set(quad.predicate.value, predicateSubjects);
}
predicateSubjects.add(subjectString);
if (quad.predicate.value === DatasetSummaryVoID.RDF_TYPE.value &&
quad.object.termType === 'NamedNode') {
let entitiesByClass = this.entitiesByClass.get(quad.object.value);
if (!entitiesByClass) {
entitiesByClass = new Set();
this.entitiesByClass.set(quad.object.value, entitiesByClass);
}
entitiesByClass.add(subjectString);
this.vocabularies.add(DatasetSummaryVoID.vocabularyFromIri(quad.object.value));
}
}
}
if (quad.object.termType === 'NamedNode' ||
quad.object.termType === 'BlankNode' ||
quad.object.termType === 'Literal') {
const objectString = (0, rdf_string_1.termToString)(quad.object);
this.distinctObjects.add(objectString);
if (quad.predicate.termType === 'NamedNode') {
let predicateObjects = this.distinctObjectsByPredicate.get(quad.predicate.value);
if (!predicateObjects) {
predicateObjects = new Set();
this.distinctObjectsByPredicate.set(quad.predicate.value, predicateObjects);
}
predicateObjects.add(objectString);
}
}
if (quad.predicate.termType === 'NamedNode') {
this.totalQuadsByPredicate.set(quad.predicate.value, (this.totalQuadsByPredicate.get(quad.predicate.value) ?? 0) + 1);
this.vocabularies.add(DatasetSummaryVoID.vocabularyFromIri(quad.predicate.value));
}
}
serialize() {
const output = [];
if (this.totalQuads > 0) {
const datasetIri = DatasetSummary_1.DF.namedNode(this.dataset);
output.push(DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.RDF_TYPE, DatasetSummaryVoID.VOID_DATASET), DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_URISPACE, DatasetSummary_1.DF.literal(this.dataset)), DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_CLASSES, DatasetSummary_1.DF.literal(this.entitiesByClass.size.toString(10), DatasetSummaryVoID.XSD_INTEGER)), DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_TRIPLES, DatasetSummary_1.DF.literal(this.totalQuads.toString(10), DatasetSummaryVoID.XSD_INTEGER)), DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_PROPERTIES, DatasetSummary_1.DF.literal(this.totalQuadsByPredicate.size.toString(10), DatasetSummaryVoID.XSD_INTEGER)), DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_DISTINCT_SUBJECTS, DatasetSummary_1.DF.literal(this.distinctSubjects.size.toString(10), DatasetSummaryVoID.XSD_INTEGER)), DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_DISTINCT_OBJECTS, DatasetSummary_1.DF.literal(this.distinctObjects.size.toString(10), DatasetSummaryVoID.XSD_INTEGER)));
for (const vocabulary of this.vocabularies) {
output.push(DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_VOCABULARY, DatasetSummary_1.DF.namedNode(vocabulary)));
}
for (const [predicate, count] of this.totalQuadsByPredicate) {
const partitionIri = DatasetSummary_1.DF.namedNode(`${this.dataset}#${DatasetSummaryVoID.hashString(predicate)}`);
output.push(DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_PROPERTY_PARTITION, partitionIri), DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.RDF_TYPE, DatasetSummaryVoID.VOID_DATASET), DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.VOID_PROPERTY, DatasetSummary_1.DF.namedNode(predicate)), DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.VOID_TRIPLES, DatasetSummary_1.DF.literal(count.toString(10), DatasetSummaryVoID.XSD_INTEGER)));
const distinctSubjectsForPredicate = this.distinctSubjectsByPredicate.get(predicate);
if (distinctSubjectsForPredicate) {
output.push(DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.VOID_DISTINCT_SUBJECTS, DatasetSummary_1.DF.literal(distinctSubjectsForPredicate.size.toString(10), DatasetSummaryVoID.XSD_INTEGER)));
}
const distinctObjectsForPredicate = this.distinctObjectsByPredicate.get(predicate);
if (distinctObjectsForPredicate) {
output.push(DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.VOID_DISTINCT_OBJECTS, DatasetSummary_1.DF.literal(distinctObjectsForPredicate.size.toString(10), DatasetSummaryVoID.XSD_INTEGER)));
}
}
for (const [rdfclass, entities] of this.entitiesByClass) {
const partitionIri = DatasetSummary_1.DF.namedNode(`${this.dataset}#${DatasetSummaryVoID.hashString(rdfclass)}`);
output.push(DatasetSummary_1.DF.quad(datasetIri, DatasetSummaryVoID.VOID_CLASS_PARTITION, partitionIri), DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.RDF_TYPE, DatasetSummaryVoID.VOID_DATASET), DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.VOID_CLASS, DatasetSummary_1.DF.namedNode(rdfclass)), DatasetSummary_1.DF.quad(partitionIri, DatasetSummaryVoID.VOID_ENTITIES, DatasetSummary_1.DF.literal(entities.size.toString(), DatasetSummaryVoID.XSD_INTEGER)));
}
}
return { iri: this.dataset, quads: output };
}
/**
* Return the hex digest of the string hash using MurmurHash3.
*/
static hashString(value) {
return (0, imurmurhash_1.default)(value).result().toString(16);
}
/**
* Takes an IRI and returns the corresponding vocabulary, as defined in
* https://www.w3.org/TR/void/#vocabularies
* @param iri The predicate or class IRI
*/
static vocabularyFromIri(iri) {
return iri.replace(DatasetSummaryVoID.VOCABULARY_TRIM_REGEX, '');
}
}
exports.DatasetSummaryVoID = DatasetSummaryVoID;
//# sourceMappingURL=DatasetSummaryVoID.js.map