rdf-test-suite
Version:
Executes the RDF and SPARQL test suites.
218 lines • 10.3 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Util = void 0;
const fs_1 = require("fs");
const jsonld_streaming_parser_1 = require("jsonld-streaming-parser");
const rdfxml_streaming_parser_1 = require("rdfxml-streaming-parser");
const stream_1 = require("stream");
const DocumentLoaderCached_1 = require("./DocumentLoaderCached");
const GeneralizedN3StreamParser_1 = require("./GeneralizedN3StreamParser");
const readable_web_to_node_stream_1 = require("readable-web-to-node-stream");
// tslint:disable-next-line:no-var-requires
const isStream = require('is-stream');
/**
* Utility functions
*/
class Util {
/**
* Determine the content type of the given URL based on the headers.
* @param {string} url The URL to get the content type from.
* @param {Headers} headers The headers of the given URL.
* @return {string} The content type.
*/
static identifyContentType(url, headers) {
const contentType = headers.get('Content-Type');
return (contentType && contentType.indexOf('application/octet-stream') < 0 ? contentType : false)
|| Util.EXTENSION_TO_CONTENTTYPE[url.substr(url.lastIndexOf('\.') + 1)]
|| 'unknown';
}
/**
* Convert https to http
* @param {string} url A URL.
* @return {string} An http URL.
*/
static normalizeBaseUrl(url) {
if (url.startsWith('https://')) {
return url.replace('https', 'http');
}
return url;
}
/**
* Fetch the given RDF document and parse it.
* @param {string} url A URL.
* @param {IFetchOptions} options Options for fetching.
* @return {Promise<[string , Stream]>} A promise resolving to a pair of a URL and a parsed RDF stream.
*/
static fetchRdf(url_1) {
return __awaiter(this, arguments, void 0, function* (url, options = {}) {
const response = yield Util.fetchCached(url, options);
const contentType = Util.identifyContentType(response.url, response.headers);
return [response.url, Util.parseRdfRaw(contentType, options.normalizeUrl ? Util.normalizeBaseUrl(response.url) : response.url, response.body, options)];
});
}
/**
* Parses RDF based on the content type.
* @param {string} contentType The content type of the given text stream.
* @param {string} baseIRI The base IRI of the stream.
* @param {NodeJS.ReadableStream} data Text stream in a certain RDF serialization.
* @param {IFetchOptions} options Options for fetching.
* @return {Stream} A parsed RDF stream.
*/
static parseRdfRaw(contentType, baseIRI, data, options = {}) {
if (contentType.indexOf('application/x-turtle') >= 0
|| contentType.indexOf('text/turtle') >= 0
|| contentType.indexOf('application/n-triples') >= 0
|| contentType.indexOf('application/n-quads') >= 0
|| contentType.indexOf('application/trig') >= 0) {
return data.pipe(new GeneralizedN3StreamParser_1.GeneralizedN3StreamParser({ baseIRI, format: contentType, }));
}
if (contentType.indexOf('application/rdf+xml') >= 0) {
return data.pipe(new rdfxml_streaming_parser_1.RdfXmlParser({ baseIRI }));
}
if (contentType.indexOf('application/ld+json') >= 0) {
const documentLoader = new DocumentLoaderCached_1.DocumentLoaderCached(options);
return data.pipe(new jsonld_streaming_parser_1.JsonLdParser({ baseIRI, documentLoader }));
}
if (baseIRI.endsWith('.ttl')) {
return data.pipe(new GeneralizedN3StreamParser_1.GeneralizedN3StreamParser({ baseIRI, format: 'text/turtle', }));
}
if (baseIRI.endsWith('.trig')) {
return data.pipe(new GeneralizedN3StreamParser_1.GeneralizedN3StreamParser({ baseIRI, format: 'application/trig', }));
}
throw new Error(`Could not parse the RDF serialization ${contentType} on ${baseIRI}`);
}
/**
* Fetch the given URL or retrieve it from a local file cache.
* @param {string} url The URL to fetch.
* @param {IFetchOptions} options Options for fetching.
* @param {RequestInit} init Fetch init options.
* @return {Promise<IFetchResponse>} A promise resolving to the response.
*/
static fetchCached(url_1) {
return __awaiter(this, arguments, void 0, function* (url, options = {}, init) {
// First check local file mappings
if (options.urlToFileMappings) {
for (const urlToFileMapping of options.urlToFileMappings) {
if (url.startsWith(urlToFileMapping.url)) {
let pathSuffix = url.substr(urlToFileMapping.url.length);
// Remove hashes from path
const hashPos = pathSuffix.indexOf('#');
if (hashPos >= 0) {
pathSuffix = pathSuffix.substr(0, hashPos);
}
// Resolve file path
const filePath = urlToFileMapping.path + pathSuffix;
if (!(0, fs_1.existsSync)(filePath)) {
throw new Error(`Could not find file ` + filePath);
}
return {
body: (0, fs_1.createReadStream)(filePath),
headers: new Headers({}),
url: urlToFileMapping.url + pathSuffix,
};
}
}
}
const cachePathLocal = options.cachePath ? options.cachePath + encodeURIComponent(url) : null;
if (cachePathLocal && (0, fs_1.existsSync)(cachePathLocal)) {
// Read from cache
return {
body: (0, fs_1.createReadStream)(cachePathLocal),
headers: new Headers(JSON.parse((0, fs_1.readFileSync)(cachePathLocal + '.headers', { encoding: 'utf8' }))),
url: (0, fs_1.readFileSync)(cachePathLocal + '.url', { encoding: 'utf8' }),
};
}
else {
// Do actual fetch
const response = yield fetch(url, init);
if (!response.ok) {
throw new Error(`Could not find ${url}`);
}
/* istanbul ignore next */
const body = isStream(response.body) || response.body === null ?
response.body :
new readable_web_to_node_stream_1.ReadableWebToNodeStream(response.body);
const body1 = body.pipe(new stream_1.PassThrough());
const body2 = body.pipe(new stream_1.PassThrough());
// Remove unneeded headers (copy to make sure the Headers object is not immutable)
const headers = new Headers(response.headers);
headers.delete('content-length');
headers.delete('content-encoding');
if (cachePathLocal) {
// Save in cache
const writeStream = (0, fs_1.createWriteStream)(cachePathLocal, 'utf8');
body1.pipe(writeStream);
yield new Promise((resolve) => setImmediate(resolve)); // To fix the problem of files being empty sometimes
(0, fs_1.writeFileSync)(cachePathLocal + '.url', response.url || url);
const headersRaw = {};
headers.forEach((value, key) => headersRaw[key] = value);
(0, fs_1.writeFileSync)(cachePathLocal + '.headers', JSON.stringify(headersRaw));
}
return {
body: body2,
headers: headers,
url: response.url || url,
};
}
});
}
/**
* Resolve all values in a hash.
* @param {{[p: string]: Promise<T>}} data A hash with promise values.
* @return {Promise<{[p: string]: T}>} A hash with resolved promise values.
*/
static promiseValues(data) {
return __awaiter(this, void 0, void 0, function* () {
const newData = {};
for (const key in data) {
newData[key] = yield data[key];
}
return newData;
});
}
/**
* Convert a license string to a URI.
* @param {string} license A license string.
* @return {string} A license URI.
*/
static licenseToUri(license) {
// TODO: make this more error-prone like here:
// https://github.com/LinkedSoftwareDependencies/npm-extraction-server/blob/master/lib/npm/NpmContext.js#L151
return 'http://opensource.org/licenses/' + license;
}
/**
* Return a string in a given color
* @param str The string that should be printed in
* @param color A given color
*/
static withColor(str, color) {
return `${color}${str}${Util.COLOR_RESET}`;
}
}
exports.Util = Util;
Util.COLOR_RESET = '\x1b[0m';
Util.COLOR_RED = '\x1b[31m';
Util.COLOR_GREEN = '\x1b[32m';
Util.COLOR_YELLOW = '\x1b[33m';
Util.COLOR_BLUE = '\x1b[34m';
Util.COLOR_MAGENTA = '\x1b[35m';
Util.COLOR_CYAN = '\x1b[36m';
Util.COLOR_GRAY = '\x1b[90m';
Util.EXTENSION_TO_CONTENTTYPE = {
jsonld: 'application/ld+json',
nq: 'application/n-quads',
nt: 'application/n-triples',
srj: 'application/sparql-results+json',
srx: 'application/sparql-results+xml',
ttl: 'text/turtle',
};
//# sourceMappingURL=Util.js.map