@csvw-rdf-convertor/core
Version:
This library was generated with [Nx](https://nx.dev).
275 lines (274 loc) • 14.2 kB
JavaScript
import { LogLevel } from './conversion-options.js';
import { normalizeDescriptor } from './descriptor.js';
import { defaultResolveJsonldFn, defaultResolveStreamFn } from './req-resolve.js';
import { transformStream } from './transformation-stream.js';
import { coerceArray } from './utils/coerce.js';
import { commonPrefixes } from './utils/prefix.js';
import { IssueTracker } from './utils/issue-tracker.js';
import { MemoryLevel } from 'memory-level';
import { DataFactory, StreamParser } from 'n3';
import { Quadstore } from 'quadstore';
import { Engine } from 'quadstore-comunica';
import { JsonLdParser } from 'jsonld-streaming-parser';
import { RdfXmlParser } from 'rdfxml-streaming-parser';
import { parseTemplate } from 'url-template';
import { CsvLocationTracker } from './utils/code-location.js';
export class Rdf2CsvwConvertor {
/**
* Main conversion function. Converts the rdf data to csvw format.
* @param url Url of rdf data to convert
* @param descriptor CSVW descriptor to use for the conversion. If not provided, a new descriptor will be created from the rdf data.
* @returns A stream of csvw data.
*/ async convert(url, descriptor) {
let wrapper;
if (descriptor === undefined) {
wrapper = this.createDescriptor(url);
} else {
wrapper = await normalizeDescriptor(descriptor, this.options, this.issueTracker);
}
await this.openStore();
// Now we have a descriptor either from user or from rdf data.
// TODO: What if we do not have enough memory to hold all the quads in the store?
const readableStream = await this.options.resolveRdfStreamFn(url, this.options.baseIri);
let parser;
if (url.match(/\.(rdf|xml)([?#].*)?$/)) {
parser = new RdfXmlParser();
} else if (url.match(/\.jsonld([?#].*)?$/)) {
parser = new JsonLdParser();
} else {
// TODO: By default, N3.Parser parses a permissive superset of Turtle, TriG, N-Triples, and N-Quads. For strict compatibility with any of those languages, pass a format argument upon creation.
parser = new StreamParser();
}
const useNamedGraphs = url.match(/\.(nq|trig)([?#].*)?$/) !== null;
for await (const chunk of readableStream){
parser.write(chunk);
}
parser.end();
await this.store.putStream(parser);
const tables = wrapper.isTableGroup ? wrapper.getTables() : [
wrapper.descriptor
];
const streams = {};
let openedStreamsCount = 0;
for (const table of tables){
var _table_tableSchema;
// TODO: use IssueTracker
if (!((_table_tableSchema = table.tableSchema) == null ? void 0 : _table_tableSchema.columns)) {
if (this.options.logLevel >= LogLevel.Warn) console.warn(`Skipping table ${table.url}: no columns found`);
continue;
}
if (table.suppressOutput === true) {
if (this.options.logLevel >= LogLevel.Warn) console.warn(`Skipping table ${table.url}: suppressOutput set to true`);
continue;
}
const tableWithRequiredColumns = table;
// See https://w3c.github.io/csvw/metadata/#tables for jsonld descriptor specification
// and https://www.w3.org/TR/csv2rdf/ for conversion in the other direction
// TODO: rdf lists
// TODO: skip columns
// TODO: row number in url templates
// TODO: escape special chars (even the dot or percentage sign) in SPARQL query.
const columns = tableWithRequiredColumns.tableSchema.columns.map((col, i)=>{
var _wrapper_descriptor_context_, _wrapper_descriptor_context;
var _wrapper_descriptor_context__language;
const defaultLang = (_wrapper_descriptor_context__language = (_wrapper_descriptor_context = wrapper.descriptor['@context']) == null ? void 0 : (_wrapper_descriptor_context_ = _wrapper_descriptor_context[1]) == null ? void 0 : _wrapper_descriptor_context_['@language']) != null ? _wrapper_descriptor_context__language : '@none';
let name = `_col.${i + 1}`;
if (col.name !== undefined) {
name = encodeURIComponent(col.name);
} else if (col.titles !== undefined) {
if (typeof col.titles === 'string' || Array.isArray(col.titles)) {
name = encodeURIComponent(coerceArray(col.titles)[0]);
} else {
// TODO: use else (startsWith(defaultLang)) as in core/src/lib/csvw2rdf/convertor.ts, or set inherited properties just away in normalizeDescriptor().
if (defaultLang in col.titles) {
name = encodeURIComponent(coerceArray(col.titles[defaultLang])[0]);
}
}
}
let title = `_col.${i + 1}`;
if (col.titles !== undefined) {
if (typeof col.titles === 'string' || Array.isArray(col.titles)) {
title = coerceArray(col.titles)[0];
} else {
// TODO: use else (startsWith(defaultLang)) as in core/src/lib/csvw2rdf/convertor.ts, or set inherited properties just away in normalizeDescriptor().
if (defaultLang in col.titles) {
title = coerceArray(col.titles[defaultLang])[0];
}
}
} else if (col.name !== undefined) {
title = col.name;
}
// note that queryVariable does not contain dot that is special char in SPARQL.
return {
name: name,
title: title,
queryVariable: `_col${i + 1}`
};
});
const query = this.createQuery(tableWithRequiredColumns, columns, useNamedGraphs);
if (this.options.logLevel >= LogLevel.Debug) console.debug(query);
const stream = transformStream(// XXX: quadstore-comunica does not support unionDefaultGraph option of comunica, so UNION must be used manually in the query.
await this.engine.queryBindings(query, {
baseIRI: '.'
}), tableWithRequiredColumns, columns, this.issueTracker);
openedStreamsCount++;
streams[tableWithRequiredColumns.url] = [
columns.filter((col, i)=>!tableWithRequiredColumns.tableSchema.columns[i].virtual),
stream
];
}
for (const [, [, stream]] of Object.entries(streams)){
stream.once('end', ()=>{
if (--openedStreamsCount === 0) this.store.close();
});
}
return streams;
}
/**
* Creates SPARQL query.
* @param table CSV Table
* @param columns Columns including virtual ones
* @param useNamedGraphs Query from named graphs in the SPARQL query
* @returns SPARQL query as a string
*/ createQuery(table, columns, useNamedGraphs) {
const lines = [];
for(let index = 0; index < table.tableSchema.columns.length; index++){
const column = table.tableSchema.columns[index];
const referencedBy = table.tableSchema.columns.find((col)=>{
if (col.propertyUrl && this.expandIri(col.propertyUrl) === 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type') return col !== column && col.aboutUrl && col.aboutUrl === column.aboutUrl;
else return col !== column && col.valueUrl && col.valueUrl === column.aboutUrl;
});
if (!referencedBy || table.tableSchema.primaryKey && table.tableSchema.primaryKey === column.name) {
const patterns = this.createTriplePatterns(table, columns, index, column.propertyUrl && this.expandIri(column.propertyUrl) === 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' ? `?${columns[index].queryVariable}` : '?_blank');
lines.push(...patterns.split('\n'));
}
}
return `SELECT ${columns.filter((col, i)=>!table.tableSchema.columns[i].virtual).map((column)=>`?${column.queryVariable}`).join(' ')}
WHERE {
${!useNamedGraphs ? lines.join('\n') : ` {
${lines.map((line)=>` ${line}`).join('\n')}
}
UNION
{
GRAPH ?_graph {
${lines.map((line)=>` ${line}`).join('\n')}
}
}`}
}`;
}
/**
* Creates SPARQL triple patterns for use in SELECT WHERE query.
* Triples are created recursively if there are references between the columns.
* @param table CSV Table
* @param columns Columns including virtual ones
* @param index Index of the column for which triples are created
* @param subject Subject of the triple, it must match the other end of the reference between columns
* @returns Triple patterns for given column as a string
*/ createTriplePatterns(table, columns, index, subject) {
const column = table.tableSchema.columns[index];
const predicate = column.propertyUrl ? `<${this.expandIri(parseTemplate(column.propertyUrl).expand({
_column: index + 1,
_source_column: index + 1,
_name: columns[index].name
}))}>` : `<${table.url}#${columns[index].name}>`;
const referencingIndex = table.tableSchema.columns.findIndex((col)=>col.propertyUrl && this.expandIri(col.propertyUrl) === 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' && col.aboutUrl == column.valueUrl);
let object;
if (column.valueUrl && parseTemplate(column.valueUrl).expand({
_column: index + 1,
_source_column: index + 1,
_name: columns[index].name
}) === column.valueUrl) {
if (column.datatype === 'anyURI' || predicate === '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>') object = `<${this.expandIri(column.valueUrl)}>`;
else object = `"${column.valueUrl}"`;
} else {
if (referencingIndex !== -1) object = `?${columns[referencingIndex].queryVariable}`;
else object = `?${columns[index].queryVariable}`;
}
const lines = [
` ${subject} ${predicate} ${object} .`
];
if (column.lang) {
// TODO: Should we be more benevolent and use LANGMATCHES instead of string equality?
// TODO: Should we lower our expectations if the matching language is not found?
lines.push(` FILTER (LANG(${object}) = '${column.lang}')`);
}
if (column.propertyUrl && this.expandIri(column.propertyUrl) === 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type') {
if (column.aboutUrl) {
table.tableSchema.columns.forEach((col, i)=>{
if (col !== column && col.aboutUrl === column.aboutUrl) {
const patterns = this.createTriplePatterns(table, columns, i, subject);
lines.push(...patterns.split('\n'));
}
});
}
} else {
if (column.valueUrl) {
table.tableSchema.columns.forEach((col, i)=>{
if (col !== column && col.aboutUrl === column.valueUrl) {
const patterns = this.createTriplePatterns(table, columns, i, object);
lines.push(...patterns.split('\n'));
}
});
}
}
if (!column.required) {
return ` OPTIONAL {
${lines.map((line)=>` ${line}`).join('\n')}
}`;
} else {
return lines.map((line)=>`${line}`).join('\n');
}
}
/**
* Expands an IRI based on the common prefixes.
* @param iri - IRI to be expanded
* @returns Expanded IRI
*/ expandIri(iri) {
const i = iri.indexOf(':');
if (i === -1) return iri;
const prefix = iri.slice(0, i);
if (prefix in commonPrefixes) {
return commonPrefixes[prefix] + iri.slice(i + 1);
}
return iri;
}
/**
* Creates a new descriptor from the rdf data, used only if no descriptor is provided.
* @param rdfData The rdf data to create the descriptor from
*/ createDescriptor(rdfData) {
return {};
}
/**
* Sets the default options for the options not provided.
* @param options
*/ setDefaults(options) {
options != null ? options : options = {};
var _options_pathOverrides, _options_baseIri, _options_logLevel, _options_resolveJsonldFn, _options_resolveRdfStreamFn, _options_descriptorNotProvided;
return {
pathOverrides: (_options_pathOverrides = options.pathOverrides) != null ? _options_pathOverrides : [],
baseIri: (_options_baseIri = options.baseIri) != null ? _options_baseIri : '',
logLevel: (_options_logLevel = options.logLevel) != null ? _options_logLevel : LogLevel.Warn,
resolveJsonldFn: (_options_resolveJsonldFn = options.resolveJsonldFn) != null ? _options_resolveJsonldFn : defaultResolveJsonldFn,
resolveRdfStreamFn: (_options_resolveRdfStreamFn = options.resolveRdfStreamFn) != null ? _options_resolveRdfStreamFn : defaultResolveStreamFn,
descriptorNotProvided: (_options_descriptorNotProvided = options.descriptorNotProvided) != null ? _options_descriptorNotProvided : false
};
}
async openStore() {
const backend = new MemoryLevel();
// different versions of RDF.js types in quadstore and n3
this.store = new Quadstore({
backend,
dataFactory: DataFactory
});
this.engine = new Engine(this.store);
await this.store.open();
}
constructor(options){
this.location = new CsvLocationTracker();
this.issueTracker = new IssueTracker(this.location, {
collectIssues: false
});
this.options = this.setDefaults(options);
}
}
//# sourceMappingURL=rdf2csvw-convertor.js.map