@rdfc/sparql-ingest-processor-ts
Version:
SPARQL Update function to be within RDF-Connect pipelines
315 lines (314 loc) • 16.8 kB
JavaScript
import { Processor, extendLogger } from "@rdfc/js-runner";
import { SDS } from "@treecg/types";
import { Agent } from "undici";
import { DataFactory } from "rdf-data-factory";
import { RdfStore } from "rdf-stores";
import { Parser } from "n3";
import { writeFile } from "fs/promises";
import { CREATE, DELETE, UPDATE } from "./SPARQLQueries.js";
import { doSPARQLRequest, getObjects, sanitizeQuads } from "./Utils.js";
const df = new DataFactory();
export var OperationMode;
(function (OperationMode) {
OperationMode["REPLICATION"] = "Replication";
OperationMode["SYNC"] = "Sync";
})(OperationMode || (OperationMode = {}));
export class SPARQLIngest extends Processor {
globalDispatcher;
transactionMembers = [];
memberBatch = [];
requestsPerformance = [];
batchCount = 0;
createTransactionQueriesLogger;
doSPARQLRequestLogger;
async init() {
this.createTransactionQueriesLogger = extendLogger(this.logger, "createTransactionQueries");
this.doSPARQLRequestLogger = extendLogger(this.logger, "doSPARQLRequest");
this.globalDispatcher = new Agent({
headersTimeout: (this.config.measurePerformance?.queryTimeout || 600) * 1000,
bodyTimeout: (this.config.measurePerformance?.queryTimeout || 600) * 1000,
});
if (!this.config.operationMode) {
this.config.operationMode = OperationMode.SYNC;
}
if (!this.config.memberBatchSize) {
this.config.memberBatchSize = 100;
}
if (this.config.accessToken === "") {
this.config.accessToken = undefined;
}
}
async transform() {
for await (const rawQuads of this.memberStream.strings()) {
this.logger.debug(`Raw member data received: \n${rawQuads}`);
const quads = new Parser().parse(rawQuads);
this.logger.verbose(`Parsed ${quads.length} quads from received member data`);
const store = RdfStore.createDefault();
quads.forEach(q => {
if (q.graph.equals(df.defaultGraph()) && this.config.targetNamedGraph) {
store.addQuad(df.quad(q.subject, q.predicate, q.object, df.namedNode(this.config.targetNamedGraph)));
}
else {
store.addQuad(q);
}
});
sanitizeQuads(store);
let query;
const memberIRI = getObjects(store, null, SDS.terms.payload, SDS.terms.custom("DataDescription"))[0];
if (memberIRI) {
this.logger.verbose(`Member IRI found in SDS metadata: ${memberIRI.value}`);
const sdsQuads = store.getQuads(null, null, null, SDS.terms.custom("DataDescription"));
sdsQuads.forEach(q => store.removeQuad(q));
if (this.config.transactionConfig) {
const transactionId = getObjects(store, null, df.namedNode(this.config.transactionConfig.transactionIdPath), null)[0];
if (transactionId) {
store.removeQuad(df.quad(memberIRI, df.namedNode(this.config.transactionConfig.transactionIdPath), transactionId));
const isLastOfTransaction = getObjects(store, null, df.namedNode(this.config.transactionConfig.transactionEndPath), null)[0];
if (isLastOfTransaction) {
this.logger.info(`Last member of ${transactionId.value} received!`);
this.verifyTransaction(this.transactionMembers.map(ts => ts.store), this.config.transactionConfig.transactionIdPath, transactionId);
store.removeQuad(df.quad(memberIRI, df.namedNode(this.config.transactionConfig.transactionEndPath), isLastOfTransaction));
this.transactionMembers.push({
memberId: memberIRI.value,
transactionId: transactionId.value,
store
});
}
else if (this.transactionMembers.length > 0) {
this.verifyTransaction(this.transactionMembers.map(ts => ts.store), this.config.transactionConfig.transactionIdPath, transactionId);
this.transactionMembers.push({
memberId: memberIRI.value,
transactionId: transactionId.value,
store
});
continue;
}
else {
this.logger.info(`New transaction ${transactionId.value} started!`);
if (this.transactionMembers.length > 0) {
this.logger.error(`Received new transaction ${transactionId.value}, `
+ `but older transaction ${this.transactionMembers[0].transactionId} hasn't been finalized `);
throw new Error(`Received new transaction ${transactionId.value}, `
+ `but older transaction ${this.transactionMembers[0].transactionId} hasn't been finalized `);
}
this.transactionMembers.push({
memberId: memberIRI.value,
transactionId: transactionId.value,
store
});
continue;
}
}
}
if (this.config.changeSemantics) {
if (this.transactionMembers.length > 0) {
query = [this.createTransactionQueries(this.transactionMembers, this.config)];
this.transactionMembers = [];
}
else {
const ctv = store.getQuads(null, df.namedNode(this.config.changeSemantics.changeTypePath))[0];
if (ctv.object.value === this.config.changeSemantics.createValue) {
this.logger.info(`Preparing 'INSERT DATA {}' SPARQL query for member ${memberIRI.value}`);
query = CREATE(store, this.config.forVirtuoso);
}
else if (ctv.object.value === this.config.changeSemantics.updateValue) {
this.logger.info(`Preparing 'DELETE {} INSERT {} WHERE {}' SPARQL query for member ${memberIRI.value}`);
query = UPDATE(store, this.config.forVirtuoso);
}
else if (ctv.object.value === this.config.changeSemantics.deleteValue) {
this.logger.info(`Preparing 'DELETE WHERE {}' SPARQL query for member ${memberIRI.value}`);
query = DELETE(store, memberIRI.value, this.config.memberShape);
}
else {
this.logger.error(`[sparqlIngest] Unrecognized change type value: ${ctv.object.value}`);
throw new Error(`[sparqlIngest] Unrecognized change type value: ${ctv.object.value}`);
}
}
}
else {
if (this.transactionMembers.length > 0) {
this.transactionMembers.forEach(ts => {
ts.store.getQuads().forEach(q => store.addQuad(q));
});
this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL query for transaction member ${memberIRI.value}`);
query = UPDATE(store, this.config.forVirtuoso);
}
else {
if (this.config.operationMode === OperationMode.REPLICATION) {
this.memberBatch.push(...store.getQuads());
this.batchCount++;
if (this.batchCount < this.config.memberBatchSize) {
continue;
}
}
else {
this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL queries for member ${memberIRI.value}`);
query = UPDATE(store, this.config.forVirtuoso);
}
}
}
}
else {
if (this.config.operationMode === OperationMode.REPLICATION) {
this.memberBatch.push(...store.getQuads());
this.batchCount++;
if (this.batchCount < this.config.memberBatchSize) {
continue;
}
}
else {
this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL queries for received quads (${store.size})`);
query = UPDATE(store, this.config.forVirtuoso);
}
}
if (query && query.length > 0) {
if (this.config.graphStoreUrl) {
try {
const t0 = Date.now();
await doSPARQLRequest(query, this.config, this.globalDispatcher, this.logger);
const reqTime = Date.now() - t0;
if (this.config.measurePerformance) {
this.requestsPerformance.push(reqTime);
}
this.logger.info(`Executed query on remote SPARQL server ${this.config.graphStoreUrl} (took ${reqTime} ms)`);
}
catch (error) {
if (!this.config.measurePerformance || this.config.measurePerformance.failureIsFatal) {
this.logger.error(`Error executing query on remote SPARQL server ${this.config.graphStoreUrl}: ${error}`);
throw error;
}
else {
if (this.config.measurePerformance) {
this.requestsPerformance.push(-1);
}
}
}
}
if (this.sparqlWriter) {
if (this.config.forVirtuoso) {
for (const q of query) {
await this.sparqlWriter.string(q);
}
}
else {
await this.sparqlWriter.string(query.join(";\n"));
}
}
}
else {
if (this.config.operationMode === OperationMode.REPLICATION) {
try {
const t0 = Date.now();
await doSPARQLRequest(this.memberBatch, this.config, this.globalDispatcher, this.logger);
const reqTime = Date.now() - t0;
if (this.config.measurePerformance) {
this.requestsPerformance.push(reqTime);
}
this.logger.info(`Executed query on remote SPARQL server ${this.config.graphStoreUrl} (took ${reqTime} ms)`);
this.batchCount = 0;
this.memberBatch = [];
}
catch (error) {
if (!this.config.measurePerformance || this.config.measurePerformance.failureIsFatal) {
this.logger.error(`Error executing query on remote SPARQL server ${this.config.graphStoreUrl}: ${error}`);
throw error;
}
else {
if (this.config.measurePerformance) {
this.requestsPerformance.push(-1);
}
}
}
}
else {
this.logger.warn(`No query generated for member ${memberIRI.value}`);
}
}
}
if (this.config.operationMode === OperationMode.REPLICATION && this.memberBatch.length > 0) {
try {
const t0 = Date.now();
await doSPARQLRequest(this.memberBatch, this.config, this.globalDispatcher, this.logger);
const reqTime = Date.now() - t0;
if (this.config.measurePerformance) {
this.requestsPerformance.push(reqTime);
}
this.logger.info(`Executed query on remote SPARQL server ${this.config.graphStoreUrl} (took ${reqTime} ms)`);
this.batchCount = 0;
this.memberBatch = [];
}
catch (error) {
if (!this.config.measurePerformance || this.config.measurePerformance.failureIsFatal) {
this.logger.error(`Error executing query on remote SPARQL server ${this.config.graphStoreUrl}: ${error}`);
throw error;
}
else {
if (this.config.measurePerformance) {
this.requestsPerformance.push(-1);
}
}
}
}
if (this.sparqlWriter) {
this.logger.info("Closing SPARQL writer");
await this.sparqlWriter.close();
}
if (this.config.measurePerformance) {
await writeFile(`${this.config.measurePerformance.outputPath}/${this.config.measurePerformance.name}.json`, JSON.stringify(this.requestsPerformance), "utf-8");
}
await this.globalDispatcher.close();
}
async produce() {
}
verifyTransaction(stores, transactionIdPath, transactionId) {
for (const store of stores) {
const tIds = getObjects(store, null, df.namedNode(transactionIdPath), null);
for (const tid of tIds) {
if (!tid.equals(transactionId)) {
this.logger.error(`[sparqlIngest] Received non-matching transaction ID ${transactionId.value} `
+ `with previous transaction: ${tid.value}`);
throw new Error(`[sparqlIngest] Received non-matching transaction ID ${transactionId.value} `
+ `with previous transaction: ${tid.value}`);
}
}
}
}
createTransactionQueries(transactionMembers, config) {
this.createTransactionQueriesLogger.info(`Creating multi-operation SPARQL UPDATE query for ${transactionMembers.length}`
+ ` members of transaction ${transactionMembers[0].transactionId}`);
const createStore = RdfStore.createDefault();
const updateStore = RdfStore.createDefault();
const deleteStore = RdfStore.createDefault();
const deleteMembers = [];
const transactionQueryBuilder = [];
for (const tsm of transactionMembers) {
const ctv = tsm.store.getQuads(null, df.namedNode(config.changeSemantics.changeTypePath))[0];
if (ctv.object.value === config.changeSemantics.createValue) {
tsm.store.getQuads().forEach(q => createStore.addQuad(q));
}
else if (ctv.object.value === config.changeSemantics.updateValue) {
tsm.store.getQuads().forEach(q => updateStore.addQuad(q));
}
else if (ctv.object.value === config.changeSemantics.deleteValue) {
tsm.store.getQuads().forEach(q => deleteStore.addQuad(q));
deleteMembers.push(tsm.memberId);
}
else {
this.createTransactionQueriesLogger.error(`[sparqlIngest] Unrecognized change type value: ${ctv.object.value}`);
throw new Error(`[sparqlIngest] Unrecognized change type value: ${ctv.object.value}`);
}
}
if (createStore.size > 0) {
transactionQueryBuilder.push(...CREATE(createStore, config.forVirtuoso));
}
if (updateStore.size > 0) {
transactionQueryBuilder.push(...UPDATE(updateStore, config.forVirtuoso));
}
if (deleteStore.size > 0) {
deleteMembers.forEach(dm => {
transactionQueryBuilder.push(...DELETE(deleteStore, dm, config.memberShape));
});
}
return transactionQueryBuilder.join(";\n");
}
}