modern-netcdf
Version:
Modern NetCDF file reader and utilities
240 lines (205 loc) • 7.68 kB
JavaScript
const http = require('http');
const https = require('https');
const { URL } = require('url');
class DataSource {
/**
* @param {ArrayBuffer|string} source - ArrayBuffer data or URL string
*/
constructor(source) {
this.source = source;
this.buffer = null;
}
/**
* Load the entire file into memory and return an ArrayBuffer.
* For browser/Node fetch we use different APIs.
* @returns {Promise<ArrayBuffer>}
*/
async getArrayBuffer() {
if (this.buffer) {
return this.buffer;
}
if (this.source instanceof ArrayBuffer) {
this.buffer = this.source;
return this.buffer;
}
if (ArrayBuffer.isView(this.source)) {
this.buffer = this.source.buffer;
return this.buffer;
}
if (typeof this.source === 'string') {
try {
const url = new URL(this.source);
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error('Unsupported URL scheme');
}
this.buffer = await this.fetchRemote(url);
return this.buffer;
} catch (error) {
if (error.message === 'Unsupported URL scheme') {
throw error;
}
throw new Error(`Failed to fetch remote file: ${error.message}`);
}
}
throw new Error('Unsupported source type');
}
/**
* Fetch an entire remote resource as ArrayBuffer. Uses the global `fetch` API when
* available (browsers, Node ≥18). Falls back to Node's http/https modules otherwise.
* @param {string|URL} url
* @returns {Promise<ArrayBuffer>}
*/
async fetchRemote(url) {
const urlObj = typeof url === 'string' ? new URL(url) : url;
// Prefer the Fetch API if present (browser or modern Node).
if (typeof fetch === 'function') {
const resp = await fetch(urlObj.toString());
if (!resp.ok) {
throw new Error(`HTTP error: ${resp.status}`);
}
return resp.arrayBuffer();
}
// Fallback: Node http/https stream implementation (unchanged from previous code).
return new Promise((resolve, reject) => {
const client = urlObj.protocol === 'https:' ? https : http;
const options = {
hostname: urlObj.hostname,
port: urlObj.port || (urlObj.protocol === 'https:' ? 443 : 80),
path: urlObj.pathname || '/',
method: 'GET'
};
const request = client.request(options, (response) => {
if (response.statusCode !== 200) {
response.resume();
reject(new Error(`HTTP error: ${response.statusCode}`));
return;
}
const chunks = [];
response.on('data', (chunk) => chunks.push(chunk));
response.on('end', () => {
const buffer = Buffer.concat(chunks);
resolve(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
});
});
request.on('error', (error) => {
// eslint-disable-next-line no-console
console.error('Request error:', error);
reject(new Error(`Network error: ${error.message}`));
});
request.end();
});
}
/**
* Retrieve a byte-range from a remote resource.
* Uses global fetch when available, falling back to Node http/https streams.
*/
async getRange(start, end) {
if (typeof this.source !== 'string') {
throw new Error('Range requests only supported for remote sources');
}
if (start < 0 || end < start) {
throw new Error('Invalid range');
}
const urlStr = this.source;
// Browser / modern Node path with fetch --------------------------
if (typeof fetch === 'function') {
const resp = await fetch(urlStr, {
headers: { Range: `bytes=${start}-${end}` }
});
if (resp.status === 416) {
throw new Error('Requested range not satisfiable');
}
if (resp.status !== 206 && resp.status !== 200) {
throw new Error(`HTTP error: ${resp.status}`);
}
const ab = await resp.arrayBuffer();
// If server ignored Range (status 200), trim.
return ab.byteLength > end - start + 1 ? ab.slice(start, end + 1) : ab;
}
// Node http/https fallback --------------------------------------
try {
const url = new URL(urlStr);
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error('Unsupported URL scheme');
}
return new Promise((resolve, reject) => {
const client = url.protocol === 'https:' ? https : http;
const options = {
hostname: url.hostname,
port: url.port || (url.protocol === 'https:' ? 443 : 80),
path: url.pathname || '/',
method: 'GET',
headers: {
Range: `bytes=${start}-${end}`
}
};
const request = client.request(options, (response) => {
if (response.statusCode === 416) {
reject(new Error('Requested range not satisfiable'));
return;
}
if (response.statusCode !== 206 && response.statusCode !== 200) {
response.resume();
reject(new Error(`HTTP error: ${response.statusCode}`));
return;
}
const chunks = [];
response.on('data', (chunk) => chunks.push(chunk));
response.on('end', () => {
const buffer = Buffer.concat(chunks);
resolve(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
});
});
request.on('error', (error) => {
// eslint-disable-next-line no-console
console.error('Range request error:', error);
reject(new Error(`Network error: ${error.message}`));
});
request.end();
});
} catch (error) {
throw new Error(`Failed to fetch range: ${error.message}`);
}
}
/**
* Read a contiguous byte range given an offset and length.
* For in-memory sources this performs a zero-copy slice of the existing ArrayBuffer.
* For remote (URL) sources the call is delegated to `getRange()`.
* If the remote server does not honour the Range header (HTTP 200 response)
* we gracefully fall back to the full download **once** and log a warning.
* @param {number} offset Start byte offset (inclusive)
* @param {number} length Number of bytes to read (must be > 0)
* @returns {Promise<ArrayBuffer>}
*/
async read(offset, length) {
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length <= 0) {
throw new Error('Invalid offset/length for read()');
}
// Fast-path for local in-memory data.
if (!(typeof this.source === 'string')) {
const ab = await this.getArrayBuffer();
if (offset + length > ab.byteLength) {
throw new Error('Read range exceeds buffer length');
}
return ab.slice(offset, offset + length);
}
// Remote source – delegate to getRange(). Ensure we only return the requested window.
const inclusiveEnd = offset + length - 1;
let buffer = await this.getRange(offset, inclusiveEnd);
// Some servers ignore Range and return the whole file (HTTP 200). `getRange()` already
// accepts the response, so trim to the requested slice here.
if (buffer.byteLength > length) {
if (!this._warnedNoRange) {
// eslint-disable-next-line no-console
console.warn('Server did not honour HTTP Range request; falling back to full download');
this._warnedNoRange = true;
}
buffer = buffer.slice(offset, offset + length);
}
return buffer;
}
close() {
this.buffer = null;
}
}
module.exports = DataSource;