@csvw-rdf-convertor/core
Version:
This library was generated with [Nx](https://nx.dev).
905 lines (904 loc) • 40.6 kB
JavaScript
import { _ as _extends } from "@swc/helpers/_/_extends";
import { LogLevel } from '../conversion-options.js';
import { CSVParser } from '../csv-parser.js';
import { normalizeDescriptor } from '../descriptor.js';
import { defaultResolveJsonldFn, defaultResolveStreamFn, defaultResolveTextFn } from '../req-resolve.js';
import { coerceArray } from '../utils/coerce.js';
import { commonPrefixes, dateTypes, dtUris, invalidValuePrefix, numericTypes } from '../utils/prefix.js';
import { tz } from '@date-fns/tz';
import { format } from 'date-fns';
import * as uts46 from 'idna-uts46-hx';
import { DataFactory } from 'n3';
import { Readable } from 'readable-stream';
import { parseTemplate } from 'url-template';
import { CsvLocationTracker } from '../utils/code-location.js';
import { IssueTracker } from '../utils/issue-tracker.js';
import { parseDate } from '../utils/parse-date.js';
import { NumberParser } from '../utils/parse-number.js';
import { replaceUrl } from '../utils/replace-url.js';
import { validateTableGroup } from '../validation/table-group.js';
import { validateTable } from '../validation/table.js';
const { namedNode, blankNode, literal, defaultGraph, quad } = DataFactory;
const { rdf, csvw, xsd } = commonPrefixes;
/**
* Class responsible for converting from CSVW to RDF. This class should not be used in parallel.
*/ export class Csvw2RdfConvertor {
/**
* Main method for converting a CSVW to RDF,
* see {@link https://w3c.github.io/csvw/csv2rdf/#json-ld-to-rdf} for more information.
* @param descriptor descriptor either as parsed or stringified JSON
* @param originalUrl original URL of the descriptor
* @returns RDF stream
*/ convert(descriptor, originalUrl) {
if (this.used) {
throw new Error('Csvw2RdfConvertor can be used only once');
}
this.used = true;
normalizeDescriptor(descriptor, this.options, this.issueTracker, originalUrl).then((input)=>{
this.input = input;
return this.convertInner();
}).catch((err)=>{
this.outputStream.destroy(err);
});
return this.outputStream;
}
createOutputStream() {
const outputStream = new Readable({
objectMode: true,
read () {
// no-op
}
});
return outputStream;
}
/**
* Convert CSVW to RDF from a CSV file URL. The descriptor will be resolved according to
* {@link https://www.w3.org/TR/2015/REC-tabular-data-model-20151217/#locating-metadata}.
* see {@link https://w3c.github.io/csvw/csv2rdf/#json-ld-to-rdf} for more information.
* @param url url of the CSV file
* @returns RDF stream
*/ convertFromCsvUrl(url) {
if (this.used) {
throw new Error('Csvw2RdfConvertor can be used only once');
}
this.used = true;
this.resolveMetadata(url).then(([wrapper, resolvedUrl])=>{
this.options.baseIri = resolvedUrl;
const tablesWithoutUrl = Array.from(wrapper.getTables()).filter((table)=>!table.url);
if (tablesWithoutUrl.length > 1) {
this.issueTracker.addError('Multiple tables without URL found in the descriptor');
}
if (tablesWithoutUrl.length === 1) {
tablesWithoutUrl[0].url = url;
}
this.input = wrapper;
return this.convertInner();
}).catch((err)=>{
this.outputStream.destroy(err);
});
return this.outputStream;
}
convertInner() {
if (!this.options.baseIri) {
var _this_input_descriptor_id;
this.options.baseIri = (_this_input_descriptor_id = this.input.descriptor['@id']) != null ? _this_input_descriptor_id : '';
}
if (this.input.isTableGroup) {
validateTableGroup(this.input.descriptor, {
input: this.input,
issueTracker: this.issueTracker
});
this.tg = this.input.descriptor;
} else {
this.tg = undefined;
validateTable(this.input.descriptor, {
input: this.input,
issueTracker: this.issueTracker
});
}
// 1
const groupNode = this.createNode(this.input.isTableGroup ? this.input.descriptor : {});
if (!this.options.minimal) {
//2
this.emitTriple(groupNode, namedNode(rdf + 'type'), namedNode(csvw + 'TableGroup'));
//3
if (this.input.isTableGroup) {
this.emitExternalProps(this.input.descriptor, groupNode);
}
}
//4
const tablePromises = [];
for (const table of this.input.getTables()){
if (table.suppressOutput) continue;
tablePromises.push(this.convertTable({
table
}).then((tableNode)=>{
// 4.2
if (!this.options.minimal) {
this.emitTriple(groupNode, namedNode(csvw + 'table'), tableNode);
}
}));
}
return Promise.all(tablePromises).then(()=>{
this.outputStream.push(null);
});
}
/**
* Locates metadata for tabular data,
* see {@link https://www.w3.org/TR/2015/REC-tabular-data-model-20151217/#locating-metadata} for more information.
* @param csvUrl CSV file url
* @returns URL of metadata file for the given CSV file
*/ async resolveMetadata(csvUrl) {
let expandedUrl = replaceUrl(csvUrl, this.options.pathOverrides);
expandedUrl = new URL(expandedUrl, this.options.baseIri || expandedUrl).href;
// metadata in a document linked to using a Link header associated with the tabular data file.
let result = await this.verifyMetadataUrl(expandedUrl, expandedUrl);
if (result) return result;
// metadata located through default paths which may be overridden by a site-wide location configuration.
const cleanUrl = new URL(expandedUrl);
cleanUrl.hash = '';
for (const template of (await this.getWellKnownUris(expandedUrl))){
let resolvedUrl = new URL(template.expand({
url: cleanUrl.href
}), expandedUrl).href;
resolvedUrl = replaceUrl(resolvedUrl, this.options.pathOverrides);
result = await this.verifyMetadataUrl(resolvedUrl, expandedUrl);
if (result) return result;
}
return [
await normalizeDescriptor({
'@context': 'http://www.w3.org/ns/csvw',
'rdfs:comment': [],
tableSchema: {
columns: []
},
url: csvUrl.split('/').pop()
}, this.options, this.issueTracker, expandedUrl),
expandedUrl
];
}
/**
* Verify that the metadata URL is valid and contains a table description that matches the CSV file.
* @param url url to the descriptor
* @param csvUrl url to the CSV file
* @returns either descriptor and url or null if the descriptor is not valid
*/ async verifyMetadataUrl(url, csvUrl) {
try {
const descriptor = await this.options.resolveJsonldFn(url, this.options.baseIri);
const wrapper = await normalizeDescriptor(JSON.parse(descriptor), this.options, this.issueTracker, url);
for (const t of wrapper.getTables()){
var _t_url;
let expandedUrl = replaceUrl((_t_url = t.url) != null ? _t_url : '', this.options.pathOverrides);
expandedUrl = new URL(expandedUrl, url).href;
if (!t.url || expandedUrl === csvUrl) {
return [
wrapper,
url
];
}
}
this.issueTracker.addWarning(`Metadata file ${url} does not contain a table that matches the CSV file ${csvUrl}`);
} catch (e) {
// ignore errors
}
return null;
}
/**
* Retrieves URI templates from well-known URI file.
* @param url /.well-known/csvm is resolved relative to this url
* @returns URI templates of metadata locations
*/ async getWellKnownUris(url) {
url = new URL('/.well-known/csvm', url).href;
url = replaceUrl(url, this.options.pathOverrides);
try {
const text = await this.options.resolveWkfFn(url, this.options.baseIri);
if (!text) return Csvw2RdfConvertor.defaultWKs;
return text.split('\n').filter((template)=>template.trim()).map((template)=>parseTemplate(template.trim()));
} catch (e) {
return Csvw2RdfConvertor.defaultWKs;
}
}
/**
* Converts a table to RDF.
*/ async convertTable(ctx) {
this.location.update({
table: ctx.table.url
});
var _ctx_table_dialect, _ref;
ctx.dialect = (_ref = (_ctx_table_dialect = ctx.table.dialect) != null ? _ctx_table_dialect : this.input.descriptor.dialect) != null ? _ref : {};
//4.1
const tableNode = this.createNode(ctx.table);
//4.2 is done in the caller
if (!this.options.minimal) {
//4.2 is done in the caller
//4.3
this.emitTriple(tableNode, namedNode(rdf + 'type'), namedNode(csvw + 'Table'));
//4.4
this.emitTriple(tableNode, namedNode(csvw + 'url'), namedNode(ctx.table.url));
//4.5
this.emitExternalProps(ctx.table, tableNode);
}
//4.6
let rowNum = 0;
const csvStream = (await this.options.resolveCsvStreamFn(ctx.table.url, this.options.baseIri)).pipeThrough(new CSVParser(ctx.dialect));
const iter = csvStream[Symbol.asyncIterator]();
const maybeRow1 = await this.processCsvHeader(iter, ctx);
this.prepareTemplates(ctx);
const rowsOffset = this.getSrcRowsOffset(ctx);
if (maybeRow1) {
ctx.row = maybeRow1;
const rowNode = this.convertTableRow(++rowNum, rowsOffset, ctx);
if (!this.options.minimal) {
this.emitTriple(tableNode, namedNode(csvw + 'row'), rowNode);
}
}
for await (const row of iter){
ctx.row = row;
const rowNode = this.convertTableRow(++rowNum, rowsOffset, ctx);
if (!this.options.minimal) {
this.emitTriple(tableNode, namedNode(csvw + 'row'), rowNode);
}
}
return tableNode;
}
getSrcRowsOffset(ctx) {
var _ctx_dialect_header, _ctx_dialect_headerRowCount;
const headerRows = (_ctx_dialect_headerRowCount = ctx.dialect.headerRowCount) != null ? _ctx_dialect_headerRowCount : ((_ctx_dialect_header = ctx.dialect.header) != null ? _ctx_dialect_header : true) ? 1 : 0;
var _ctx_dialect_skipRows;
return headerRows + ((_ctx_dialect_skipRows = ctx.dialect.skipRows) != null ? _ctx_dialect_skipRows : 0);
}
/**
* Prepares URI templates for the conversion.
*/ prepareTemplates(ctx) {
const templates = {
about: {},
property: {},
value: {}
};
const types = [
'about',
'property',
'value'
];
for (const col of ctx.columns){
if (col.suppressOutput) continue;
ctx.col = col;
for (const type of types){
const template = this.inherit(`${type}Url`, col, ctx);
if (template === undefined) continue;
templates[type][col.name] = parseTemplate(template);
}
}
ctx.templates = templates;
}
/**
* Processes the header of a CSV file and its embedded metadata.
* @param {AsyncIterator<string[]>} stream - Input stream
* @returns The first row of the current table if there is no header and there are no columns defined in the table schema.
* This row is used to determine the column count and must be passed to the {@link Csvw2RdfConvertor#convertTableRow} method.
*/ async processCsvHeader(stream, ctx) {
var _this_input_descriptor_context_, _this_input_descriptor_context;
var _this_inherit, _ref;
const defaultLang = (_ref = (_this_inherit = this.inherit('lang', ctx.table, ctx)) != null ? _this_inherit : (_this_input_descriptor_context = this.input.descriptor['@context']) == null ? void 0 : (_this_input_descriptor_context_ = _this_input_descriptor_context[1]) == null ? void 0 : _this_input_descriptor_context_['@language']) != null ? _ref : '@none';
if (ctx.table.tableSchema === undefined) ctx.table.tableSchema = {};
const schema = ctx.table.tableSchema;
var _ctx_dialect_header, _ctx_dialect_headerRowCount;
const headerRowCount = (_ctx_dialect_headerRowCount = ctx.dialect.headerRowCount) != null ? _ctx_dialect_headerRowCount : ((_ctx_dialect_header = ctx.dialect.header) != null ? _ctx_dialect_header : true) ? 1 : 0;
if (!schema.columns) {
schema.columns = [];
}
ctx.columns = schema.columns;
const physicalColCount = ctx.columns.length ? ctx.columns.filter((c)=>!c.virtual).length : undefined;
if (physicalColCount !== undefined && ctx.columns.slice(0, physicalColCount).find((c)=>c.virtual)) {
this.issueTracker.addError('Table schema has virtual columns before physical ones');
ctx.columns.sort((a, b)=>(a.virtual ? 1 : 0) - (b.virtual ? 1 : 0));
}
const maybeRow = await this.processMicrosyntax(defaultLang, headerRowCount, physicalColCount, stream, ctx);
if (maybeRow) return maybeRow;
this.columnTitlesToNames(defaultLang, ctx);
this.validateDuplicateColumns(ctx.columns.filter((col)=>col.name !== undefined));
return undefined;
}
/**
* Processes the microsyntax of the CSV file. This currently includes only column titles.
* @param defaultLang default language of the csv file
* @param headerRowCount number of header rows
* @param physicalColCount number of nonvirtual columns defined in the table schema or undefined if the table schema has no columns defined
* @param stream CSV stream
*/ async processMicrosyntax(defaultLang, headerRowCount, physicalColCount, stream, ctx) {
for (const col of ctx.columns){
if (col.titles && typeof col.titles === 'object' && !Array.isArray(col.titles) && '@none' in col.titles && !(defaultLang in col.titles)) {
col.titles[defaultLang] = col.titles['@none'];
delete col.titles['@none'];
}
}
for(let i = 0; i < headerRowCount; ++i){
const header = await stream.next();
if (header.done) {
this.issueTracker.addError('CSV stream ended before header was read');
return;
}
var _ctx_dialect_skipColumns;
const vals = header.value.slice((_ctx_dialect_skipColumns = ctx.dialect.skipColumns) != null ? _ctx_dialect_skipColumns : 0);
if (physicalColCount !== undefined && vals.length !== physicalColCount) {
this.issueTracker.addWarning(`Header row ${i} has ${vals.length} columns, but the table schema has ${physicalColCount} non-virtual columns`);
}
this.headerRowToTitles(vals, defaultLang, i === 0, ctx);
}
if (!ctx.columns.length) {
const row = await stream.next();
if (row.done) return;
ctx.columns = ctx.table.tableSchema.columns = row.value.map((_, i)=>({
name: '_col.' + (i + 1)
}));
return row.value;
}
return undefined;
}
headerRowToTitles(vals, defaultLang, firstRow, ctx) {
for(let j = 0; j < vals.length; ++j){
if (!vals[j]) continue;
let modified = false;
let col = ctx.columns[j];
if (!col) {
col = {};
ctx.columns[j] = col;
}
if (col.titles === undefined) col.titles = [
vals[j]
];
else if (Array.isArray(col.titles)) {
if (col.titles.includes(vals[j])) return;
col.titles.push(vals[j]);
modified = true;
} else if (typeof col.titles === 'string') {
if (col.titles !== vals[j]) {
col.titles = [
col.titles,
vals[j]
];
modified = true;
}
} else if (col.titles[defaultLang] === undefined) {
col.titles[defaultLang] = vals[j];
modified = Object.keys(col.titles).length > 1;
} else if (typeof col.titles[defaultLang] === 'string') {
if (col.titles[defaultLang] !== vals[j]) {
col.titles[defaultLang] = [
col.titles[defaultLang],
vals[j]
];
modified = true;
}
} else if (!col.titles[defaultLang].includes(vals[j])) {
col.titles[defaultLang].push(vals[j]);
modified = true;
}
if (modified && firstRow) {
const title = coerceArray(typeof col.titles === 'object' && !Array.isArray(col.titles) ? col.titles[defaultLang] : col.titles)[0];
if (title === vals[j]) {
this.issueTracker.addWarning(`Column title language is different from header in the CSV file "${vals[j]}"@${defaultLang}`);
} else {
this.issueTracker.addWarning(`Column title "${title}" is different from header in the CSV file "${vals[j]}"`);
}
}
}
}
columnTitlesToNames(defaultLang, ctx) {
for(let i = 0; i < ctx.columns.length; ++i){
const col = ctx.columns[i];
if (col.name) continue;
if (!col.titles) {
col.name = '_col.' + (i + 1);
continue;
}
if (typeof col.titles === 'string') col.name = col.titles;
else if (Array.isArray(col.titles)) col.name = col.titles[0];
else {
if (defaultLang in col.titles) {
col.name = coerceArray(col.titles[defaultLang])[0];
} else {
for(const key in col.titles){
if (key.startsWith(defaultLang)) {
col.name = coerceArray(col.titles[key])[0];
break;
}
}
}
}
col.name = col.name ? encodeURIComponent(col.name).replaceAll('-', '%2D') : '_col.' + (i + 1);
}
}
validateDuplicateColumns(columns) {
for(let i = 0; i < columns.length; ++i){
for(let j = 0; j < i; ++j){
if (columns[i].name === columns[j].name) {
this.issueTracker.addError(`Duplicate column name "${columns[i].name}"`);
}
}
}
}
/**
* Converts table row to RDF by row number.
* @param {number} rowNum - The row number.
* @param {number} rowsOffset - The offset of the rows.
*/ convertTableRow(rowNum, rowsOffset, ctx) {
this.location.update({
row: rowNum
});
//4.6.1
const rowNode = blankNode();
//4.6.2 done by caller
if (!this.options.minimal) {
//4.6.3
this.emitTriple(rowNode, namedNode(rdf + 'type'), namedNode(csvw + 'Row'));
//4.6.4
this.emitTriple(rowNode, namedNode(csvw + 'rownum'), literal(rowNum.toString(), namedNode(xsd + 'integer')));
//4.6.5
this.emitTriple(rowNode, namedNode(csvw + 'url'), namedNode(ctx.table.url + '#row=' + (rowNum + rowsOffset)));
//4.6.7
// implementation dependent, based on notes on the table, we skip this
}
this.convertTableRowValues(rowNode, rowsOffset, ctx);
if (!this.options.minimal) {
var _ctx_table_tableSchema;
//4.6.6
const titles = coerceArray((_ctx_table_tableSchema = ctx.table.tableSchema) == null ? void 0 : _ctx_table_tableSchema.rowTitles);
const titlemap = {};
for(let i = 0; i < titles.length; i++){
titlemap[ctx.columns[i].name] = i;
}
for (const title of titles){
ctx.col = ctx.columns[titlemap[title]];
const lang = this.inherit('lang', ctx.col, ctx);
const val = ctx.rowRecord[title];
if (!val) continue;
this.emitTriple(rowNode, namedNode(csvw + 'title'), literal(val, lang));
}
}
return rowNode;
}
/**
* Converts the values of a table row to RDF.
*/ convertTableRowValues(rowNode, rowsOffset, ctx) {
var _ctx_dialect_skipColumns;
const colsOffset = (_ctx_dialect_skipColumns = ctx.dialect.skipColumns) != null ? _ctx_dialect_skipColumns : 0;
//4.6.8
const defaultCellSubj = blankNode();
const totalCols = Math.max(ctx.columns.length, ctx.row.length);
ctx.rowRecord = {};
// fill rowRecord (we need all of the row values to process template uris)
for(let i = 0; i < totalCols; ++i){
ctx.col = ctx.columns[i];
this.location.update({
column: i
});
const [dtUri, dt] = this.normalizeDatatype(ctx);
ctx.rowRecord[ctx.col.name] = this.interpretDatatype(ctx.row[i], dtUri, dt, ctx);
}
// now we can safely process the values
for(let i = 0; i < totalCols; ++i){
ctx.col = ctx.columns[i];
if (ctx.col.suppressOutput) continue;
this.convertRowCell(defaultCellSubj, rowNode, i, rowsOffset, colsOffset, ctx);
}
}
/**
* Converts a cell of the current row to RDF.
* @param {BlankNode} defaultSubj - Default subject
* @param {BlankNode} rowNode - The row node
* @param {number} colNum - The column number.
* @param {number} rowsOffset - The offset of the rows.
* @param {number} colsOffset - The offset of the columns.
*/ convertRowCell(defaultSubj, rowNode, colNum, rowsOffset, colsOffset, ctx) {
this.location.update({
column: colNum
});
if (ctx.rowRecord[ctx.col.name] === null) {
if (ctx.col.required) {
this.issueTracker.addWarning('Null value in a required column');
}
return;
}
const [dtUri] = this.normalizeDatatype(ctx);
const rowNum = this.location.value.row;
//4.6.8.1
const subject = ctx.templates.about[ctx.col.name] === undefined ? defaultSubj : this.templateUri(ctx.templates.about[ctx.col.name], colNum + colsOffset, rowNum + rowsOffset, ctx.table.url, ctx);
if (!this.options.minimal) {
//4.6.8.2
this.emitTriple(rowNode, namedNode(csvw + 'describes'), subject);
}
const predicate = ctx.templates.property[ctx.col.name] === undefined ? namedNode(ctx.table.url + '#' + ctx.col.name) : this.templateUri(ctx.templates.property[ctx.col.name], colNum + colsOffset, rowNum + rowsOffset, ctx.table.url, ctx);
const lang = this.inherit('lang', ctx.col, ctx);
if (ctx.templates.value[ctx.col.name] === undefined) {
const val = ctx.rowRecord[ctx.col.name];
if (Array.isArray(val)) {
if (this.inherit('ordered', ctx.col, ctx)) {
const head = this.createRDFList(val.map((v)=>this.datatypeToLiteral(v, dtUri, lang)));
this.emitTriple(subject, predicate, head);
} else {
for (const item of val){
this.emitTriple(subject, predicate, this.datatypeToLiteral(item, dtUri, lang));
}
}
} else {
this.emitTriple(subject, predicate, this.datatypeToLiteral(val, dtUri, lang));
}
} else {
const val = this.templateUri(ctx.templates.value[ctx.col.name], colNum + colsOffset, rowNum + rowsOffset, ctx.table.url, ctx);
this.emitTriple(subject, predicate, val);
}
}
/**
* Get expanded datatype URI and description for the current column.
* @returns [datatype URI, datatype description]
*/ normalizeDatatype(ctx) {
var _this_inherit;
const dtOrBuiltin = (_this_inherit = this.inherit('datatype', ctx.col, ctx)) != null ? _this_inherit : 'string';
const dt = typeof dtOrBuiltin === 'string' ? {
base: dtOrBuiltin
} : dtOrBuiltin;
let dtUri = dt['@id'];
if (!dtUri) {
var _dt;
var _base;
(_base = (_dt = dt).base) != null ? _base : _dt.base = 'string';
if (dt.base in dtUris) {
dtUri = dtUris[dt.base];
} else {
dtUri = xsd + dt.base;
}
} else {
dtUri = this.expandIri(dtUri);
}
return [
dtUri,
dt
];
}
/**
* Creates an RDF list https://ontola.io/blog/ordered-data-in-rdf based on rules provided at https://w3c.github.io/csvw/csv2rdf/#json-ld-to-rdf.
* @param values - Values of the list
* @returns The head of the rdf list
*/ createRDFList(values) {
const head = blankNode();
let current = head;
for(let i = 0; i < values.length - 1; ++i){
this.emitTriple(current, namedNode(rdf + 'first'), values[i]);
const next = blankNode();
this.emitTriple(current, namedNode(rdf + 'rest'), next);
current = next;
}
this.emitTriple(current, namedNode(rdf + 'first'), values[values.length - 1]);
this.emitTriple(current, namedNode(rdf + 'rest'), namedNode(rdf + 'nil'));
return head;
}
/**
* Emits a triple to the output stream.
*/ emitTriple(first, second, third) {
this.outputStream.push(quad(first, second, third, defaultGraph()));
}
emitExternalProps(object, node) {
if (object.notes === undefined) return;
for (const prop of this.input.getExternalProps(object.notes, node)){
this.outputStream.push(prop);
}
}
/**
* Creates a named node or a blank node based on the input.
* @param input - Input object
*/ createNode(input) {
if (input['@id'] === undefined) {
return blankNode();
} else {
return namedNode(input['@id']);
}
}
/**
* Inteprets the datatype of a value based on the current column description.
* @param {string} value - string value to be interpreted
* @returns Correctly built RDF literal
*/ interpretDatatype(value, dtUri, dt, ctx) {
const normalizedValue = this.normalizeValue(value, dtUri, ctx);
if (normalizedValue === null) return null;
if (Array.isArray(normalizedValue)) {
const formatted = normalizedValue.map((val)=>this.reformatValue(val, dtUri, dt, ctx)).filter((val)=>val !== null);
return formatted;
} else {
const object = this.reformatValue(normalizedValue, dtUri, dt, ctx);
return object;
}
}
/**
* Convert string value to RDF literal based on the datatype URI.
* Quadstore cannot store NaN as a literal, so we use a temporary prefix for numeric types.
* This is later replaced in the {@link Csvw2RdfConvertor#replacerStream} method.
* @param value - string value to be converted
* @param dtUri - datatype URI
* @param lang - language tag
* @returns RDF literal
*/ datatypeToLiteral(value, dtUri, lang) {
if (dtUri !== xsd + 'string' && value.startsWith(invalidValuePrefix)) {
return literal(value.slice(invalidValuePrefix.length), namedNode(xsd + 'string'));
}
if (dtUri === xsd + 'string' && lang) {
return literal(value, lang);
}
return literal(value, namedNode(dtUri));
}
/**
* Reformat string value into a valid string representation of the datatype.
* @param value - string value to be converted
* @param dtUri - datatype URI
* @param dt - datatype description
* @returns the reformatted value
*/ reformatValue(value, dtUri, dt, ctx) {
var _ctx_col_default;
if (value === '') value = (_ctx_col_default = ctx.col.default) != null ? _ctx_col_default : '';
if (this.isValueNull(value, ctx)) return null;
if (numericTypes.has(dtUri)) {
value = this.numberParser.parse(value, dt.format, dtUri, dt);
} else if (dateTypes.has(dtUri)) {
value = this.reformatDate(value, dtUri, dt);
} else if (dtUri === xsd + 'boolean') {
value = this.reformatBoolean(value, dt);
} else if (dt.format instanceof RegExp && dtUri !== csvw + 'json' && dtUri !== xsd + 'xml' && dtUri !== xsd + 'html') {
if (!value.match(dt.format)) {
this.issueTracker.addWarning(`Value "${value}" does not match the format "${dt.format}"`);
if (dtUri !== xsd + 'string') {
return invalidValuePrefix + value;
}
}
}
if (!this.validateValueLength(value, dtUri, dt)) {
return invalidValuePrefix + value;
}
return value;
}
validateValueLength(value, dtUri, dt) {
const valLength = this.getValueLength(value, dtUri);
if (dt.length !== undefined && dt.length !== valLength) {
this.issueTracker.addWarning(`Value "${value}" does not match the length "${dt.length}"`);
return false;
} else if (dt.minLength !== undefined && dt.minLength > valLength) {
this.issueTracker.addWarning(`Value "${value}" does not match the minLength "${dt.minLength}"`);
return false;
} else if (dt.maxLength !== undefined && dt.maxLength < valLength) {
this.issueTracker.addWarning(`Value "${value}" does not match the maxLength "${dt.maxLength}"`);
return false;
}
return true;
}
getValueLength(value, dtUri) {
if (value === undefined) return 0;
switch(dtUri){
case xsd + 'hexBinary':
return value.length / 2;
case xsd + 'base64Binary':
return atob(value).length;
default:
return value.length;
}
}
reformatBoolean(value, dt) {
if (dt.format) {
const [trueVal, falseVal] = dt.format.split('|');
if (value === trueVal) value = 'true';
else if (value === falseVal) value = 'false';
else {
this.issueTracker.addWarning(`Value "${value}" does not match the format "${dt.format}"`);
return invalidValuePrefix + value;
}
} else {
if (value === 'true' || value === '1') value = 'true';
else if (value === 'false' || value === '0') value = 'false';
else {
this.issueTracker.addWarning(`Value "${value}" does not match the format "true|false" or "1|0"`);
return invalidValuePrefix + value;
}
}
return value;
}
reformatDate(value, dtUri, dt) {
const date = parseDate(value, dtUri, dt.format);
if (Number.isNaN(date.getTime())) {
if (dt.format) {
this.issueTracker.addWarning(`Value "${value}" does not match the format "${dt.format}"`);
} else {
this.issueTracker.addWarning(`Value "${value}" is not a valid date`);
}
return invalidValuePrefix + value;
}
if (!this.validateDateMinMax(date, dtUri, dt)) {
return invalidValuePrefix + value;
}
let resultFormat = dtUri === xsd + 'date' ? 'yyyy-MM-dd' : dtUri === xsd + 'time' ? 'HH:mm:ss' : "yyyy-MM-dd'T'HH:mm:ss";
let millis = date.getMilliseconds();
if (millis) {
resultFormat += '.';
while(millis % 10 === 0){
millis /= 10;
}
resultFormat += 'S'.repeat(millis.toString().length);
}
if (date.timeZone) {
resultFormat += 'XXX';
value = format(date, resultFormat, {
in: tz(date.timeZone)
});
} else {
value = format(date, resultFormat);
}
return value;
}
validateDateMinMax(value, dtUri, dt) {
var _dt_minimum, _ref;
const minimum = (_ref = (_dt_minimum = dt.minimum) != null ? _dt_minimum : dt.minInclusive) != null ? _ref : undefined;
var _dt_maximum, _ref1;
const maximum = (_ref1 = (_dt_maximum = dt.maximum) != null ? _dt_maximum : dt.maxInclusive) != null ? _ref1 : undefined;
var _dt_minExclusive;
const minExclusive = (_dt_minExclusive = dt.minExclusive) != null ? _dt_minExclusive : undefined;
var _dt_maxExclusive;
const maxExclusive = (_dt_maxExclusive = dt.maxExclusive) != null ? _dt_maxExclusive : undefined;
if (minimum !== undefined && value < parseDate(minimum, dtUri)) {
this.issueTracker.addWarning(`Value "${value}" does not meet the minimum "${minimum}"`);
return false;
}
if (maximum !== undefined && value > parseDate(maximum, dtUri)) {
this.issueTracker.addWarning(`Value "${value}" does not meet the maximum "${maximum}"`);
return false;
}
if (minExclusive !== undefined && value <= parseDate(minExclusive, dtUri)) {
this.issueTracker.addWarning(`Value "${value}" does not meet the minimum exclusive "${minExclusive}"`);
return false;
}
if (maxExclusive !== undefined && value >= parseDate(maxExclusive, dtUri)) {
this.issueTracker.addWarning(`Value "${value}" does not meet the maximum exclusive "${maxExclusive}"`);
return false;
}
return true;
}
/**
* Check if value should be considered null based on the current column description.
* @param value - string value to be checked
* @returns true if the value is null, false otherwise
*/ isValueNull(value, ctx) {
const nullVal = this.inherit('null', ctx.col, ctx);
if (nullVal === undefined) return value === '';
if (nullVal === value) return true;
if (Array.isArray(nullVal)) {
return nullVal.some((n)=>n === value);
}
return false;
}
/**
* Handle whitespace normalization for the given value and datatype.
* @param value - string value to be normalized
* @param dtype - datatype URI
* @returns normalized value
*/ normalizeValue(value, dtype, ctx) {
if (!Csvw2RdfConvertor.normalizeWsTypes.has(dtype)) {
value = value.replace(/\s+/, ' ').trim();
} else if (dtype === xsd + 'normalizedString') {
value = value.replace(/\t\r\n/g, ' ');
}
var _ctx_col_default;
if (value === '') value = (_ctx_col_default = ctx.col.default) != null ? _ctx_col_default : '';
const sep = this.inherit('separator', ctx.col, ctx);
if (sep !== undefined) {
if (value === '') return [];
if (this.isValueNull(value, ctx)) return null;
const parts = value.split(sep);
if (dtype !== xsd + 'string' && dtype !== xsd + 'anyAtomicType') {
return parts.map((part)=>part.trim());
}
return parts;
}
return value;
}
/**
* get value of inherited property
* @param mostSpecific - most specific object to consider
*/ inherit(prop, mostSpecific, ctx) {
const levels = [
ctx.col,
ctx.table.tableSchema,
ctx.table,
this.tg
];
let found = false;
for (const level of levels){
if (level === mostSpecific) found = true;
else if (!found) continue;
if ((level == null ? void 0 : level[prop]) !== undefined) {
return level[prop];
}
}
return undefined;
}
/**
* Sets default values to options if no value is provided.
* @param options
* @returns Corrected options
*/ setDefaults(options) {
options != null ? options : options = {};
var _options_pathOverrides, _options_resolveJsonldFn, _options_resolveCsvStreamFn, _options_resolveWkfFn, _options_baseIri, _options_templateIris, _options_minimal, _options_logLevel;
return {
pathOverrides: (_options_pathOverrides = options.pathOverrides) != null ? _options_pathOverrides : [],
resolveJsonldFn: (_options_resolveJsonldFn = options.resolveJsonldFn) != null ? _options_resolveJsonldFn : defaultResolveJsonldFn,
resolveCsvStreamFn: (_options_resolveCsvStreamFn = options.resolveCsvStreamFn) != null ? _options_resolveCsvStreamFn : defaultResolveStreamFn,
resolveWkfFn: (_options_resolveWkfFn = options.resolveWkfFn) != null ? _options_resolveWkfFn : defaultResolveTextFn,
baseIri: (_options_baseIri = options.baseIri) != null ? _options_baseIri : '',
templateIris: (_options_templateIris = options.templateIris) != null ? _options_templateIris : false,
minimal: (_options_minimal = options.minimal) != null ? _options_minimal : false,
logLevel: (_options_logLevel = options.logLevel) != null ? _options_logLevel : LogLevel.Warn
};
}
/**
* Expands an IRI based on the common prefixes.
* @param iri - IRI to be expanded
* @returns Expanded IRI
*/ expandIri(iri) {
const i = iri.indexOf(':');
if (i === -1) return iri;
const prefix = iri.slice(0, i);
if (prefix in commonPrefixes) {
return commonPrefixes[prefix] + iri.slice(i + 1);
}
return iri;
}
/**
* Expands a template URI.
* @param template - Template to be expanded
* @param srcCol - Source column number ({@link Csvw2RdfConvertor#location} column + {@link CsvwDialectDescription#skipColumns})
* @param srcRow - Source row number ({@link Csvw2RdfConvertor#location} row + {@link CsvwDialectDescription#headerRowCount} + {@link CsvwDialectDescription#skipRows})
* @param colVals - Column values
* @param baseIri - Base IRI
* @returns Expanded URI node
*/ templateUri(template, srcCol, srcRow, baseIri, ctx) {
var _URL_parse;
let uri = template.expand(_extends({}, ctx.rowRecord, {
_column: this.location.value.column,
_sourceColumn: srcCol,
_row: this.location.value.row,
_sourceRow: srcRow,
_name: decodeURIComponent(ctx.col.name)
}));
uri = this.expandIri(uri);
var _URL_parse_href;
uri = (_URL_parse_href = (_URL_parse = URL.parse(uri)) == null ? void 0 : _URL_parse.href) != null ? _URL_parse_href : baseIri + uri;
if (this.options.templateIris) {
const parsed = URL.parse(uri);
uri = parsed.href.replace(parsed.hostname, uts46.toUnicode(parsed.hostname));
uri = decodeURI(uri);
}
return namedNode(uri);
}
/**
* Creates a new instance of the convertor.
* @param {Csvw2RdfOptions} options - Options for the convertor.
*/ constructor(options){
this.location = new CsvLocationTracker();
this.outputStream = this.createOutputStream();
this.issueTracker = new IssueTracker(this.location, {
eventEmitter: this.outputStream,
collectIssues: false
});
this.numberParser = new NumberParser(this.issueTracker);
this.used = false;
this.options = this.setDefaults(options);
}
}
Csvw2RdfConvertor.defaultWKs = [
parseTemplate('{+url}-metadata.json'),
parseTemplate('csv-metadata.json')
];
Csvw2RdfConvertor.normalizeWsTypes = new Set([
xsd + 'string',
xsd + 'normalizedString',
dtUris.xml,
dtUris.html,
dtUris.json,
xsd + 'anyAtomicType'
]);
//# sourceMappingURL=convertor.js.map