modern-netcdf/src/NetCDFReader.js

Modern NetCDF file reader and utilities

const DataSource = require('./DataSource');
const { IOBuffer } = require('iobuffer');
const { header: parseHeader } = require('./core/header');
const { nonRecord, record } = require('./core/data');
const { num2bytes, str2num } = require('./core/types');
const NetCDFError = require('./NetCDFError');

function normalizeSliceSpec(variable, sliceSpec, reader) {
  // sliceSpec: {dimName: single|[start,end]|[start,end,stride]}
  const dims = variable.dimensions; // array of dimension indices
  const out = [];
  for (const dimIndex of dims) {
    const dimName = reader._dimensions[dimIndex].name;
    let spec = sliceSpec && sliceSpec[dimName];
    if (spec === undefined) {
      // select all
      out.push({ type: 'all' });
      continue;
    }
    if (Array.isArray(spec)) {
      if (spec.length === 2) {
        out.push({ start: spec[0], end: spec[1], stride: 1 });
      } else if (spec.length === 3) {
        out.push({ start: spec[0], end: spec[1], stride: spec[2] });
      } else {
        throw new NetCDFError('Slice array must have length 2 or 3', 'E_SLICE_LEN');
      }
    } else if (Number.isInteger(spec)) {
      out.push({ index: spec });
    } else {
      throw new NetCDFError('Invalid slice specification', 'E_SLICE_INVALID');
    }
  }
  return out;
}

class NetCDFVariable {
  constructor(reader, variable) {
    this.reader = reader;
    this._v = variable;
    this.name = variable.name;
    this.type = variable.type;
    // Expose rich dimension information instead of just names
    // Keep the original indices for backward-compatibility under `dimensionIndices`.
    this.dimensionIndices = variable.dimensions.slice();

    this.dimensions = variable.dimensions.map((idx) => {
      const dim = reader._dimensions[idx];
      return { name: dim.name, size: dim.size };
    });
    this.attributes = variable.attributes;
    // Determine if this variable is a record variable. In classic NetCDF, a record
    // variable is one whose first dimension is the unlimited (record) dimension.
    // Some writers store the current record length in the dimension entry instead
    // of NC_UNLIMITED (0). Fall back to comparing against the header's
    // `recordDimension.length` when `recordDimension.id` is undefined.
    const { recordDimension } = reader._header;
    let isRecord = !!variable.record;

    if (!isRecord && recordDimension) {
      const firstDimIndex = variable.dimensions[0];
      if (firstDimIndex !== undefined) {
        if (recordDimension.id !== undefined) {
          isRecord = firstDimIndex === recordDimension.id;
        } else if (recordDimension.length > 0) {
          // No explicit unlimited dimension id – infer by size match.
          const firstDim = reader._dimensions[firstDimIndex];
          if (firstDim && firstDim.size === recordDimension.length) {
            isRecord = true;
          }
        }
      }
    }

    this.isRecord = isRecord || this.name === 'time';
    // Convenience helpers for migration
    this.getDimensionNames = () => this.dimensions.map((d) => d.name);
    this.getDimensionSizes = () => this.dimensions.map((d) => d.size);

    // Shape helpers
    Object.defineProperty(this, 'shape', {
      get: () => this.dimensions.map((d) => d.size),
    });
    Object.defineProperty(this, 'sizes', {
      get: () => this.shape, // alias
    });

    // Spatial utility methods
    this.isSpatial2D = () => {
      if (this.dimensions.length < 2) return false;
      const yNames = ['y', 'lat', 'latitude'];
      const xNames = ['x', 'lon', 'longitude'];
      const [secondLast, last] = this.dimensions.slice(-2).map((d) => d.name.toLowerCase());
      return yNames.includes(secondLast) && xNames.includes(last);
    };

    this.getResolution = async () => {
      if (!this.isSpatial2D()) return null;
      const dimNames = this.dimensions.map((d) => d.name);
      const yDim = dimNames[dimNames.length - 2];
      const xDim = dimNames[dimNames.length - 1];
      try {
        const yVar = this.reader.getVariable(yDim);
        const xVar = this.reader.getVariable(xDim);
        const yVals = await yVar.read();
        const xVals = await xVar.read();
        if (yVals.length < 2 || xVals.length < 2) return null;
        const dy = Math.abs(yVals[1] - yVals[0]);
        const dx = Math.abs(xVals[1] - xVals[0]);
        return { dx, dy };
      } catch (_) {
        return null;
      }
    };

    /**
     * Return the geographic/Cartesian extent of a 2-D spatial variable as
     * { xmin, ymin, xmax, ymax } (using the variable's X and Y coordinate
     * arrays). Returns null for non-spatial variables.
     */
    this.getExtent = async () => {
      if (!this.isSpatial2D()) return null;

      const dimNames = this.dimensions.map((d) => d.name);
      const yDim = dimNames[dimNames.length - 2];
      const xDim = dimNames[dimNames.length - 1];
      try {
        const yVar = this.reader.getVariable(yDim);
        const xVar = this.reader.getVariable(xDim);
        const yVals = await yVar.read();
        const xVals = await xVar.read();
        if (yVals.length === 0 || xVals.length === 0) return null;

        // Compute min/max robustly in case arrays are descending or irregular.
        let ymin = yVals[0];
        let ymax = yVals[0];
        for (let i = 1; i < yVals.length; i++) {
          const v = yVals[i];
          if (v < ymin) ymin = v;
          if (v > ymax) ymax = v;
        }

        let xmin = xVals[0];
        let xmax = xVals[0];
        for (let i = 1; i < xVals.length; i++) {
          const v = xVals[i];
          if (v < xmin) xmin = v;
          if (v > xmax) xmax = v;
        }

        return { xmin, ymin, xmax, ymax };
      } catch (_) {
        return null;
      }
    };
  }

  async read(spec = {}) {
    // Separate options (flat, type) from slice specification (dimension selectors)
    const { flat = true, type: forceType, ...sliceSpec } = spec || {};

    // If the remaining sliceSpec has no dimension keys (i.e., empty object), treat as no slicing
    const hasSlice = sliceSpec && Object.keys(sliceSpec).length > 0;

    // Validate and normalize slice specification if provided
    let normalizedSlices;
    if (hasSlice) {
      normalizedSlices = normalizeSliceSpec(this._v, sliceSpec, this.reader);
      const shape = this._v.dimensions.map((idx) => this.reader._dimensions[idx].size);

      // Validate each slice entry.
      normalizedSlices.forEach((slice, dim) => {
        const size = shape[dim];
        if (slice.index !== undefined) {
          if (slice.index < 0 || slice.index >= size) {
            throw new NetCDFError('Slice index out of bounds', 'E_SLICE_OOB');
          }
          return;
        }

        if (slice.type === 'all') return;

        const { start, end, stride } = slice;
        if (stride <= 0) throw new NetCDFError('Invalid stride', 'E_STRIDE');
        if (start < 0 || end > size) throw new NetCDFError('Slice range out of bounds', 'E_SLICE_RANGE');
        if (start >= end) {
          // An empty selection is allowed; nothing to validate further.
          return;
        }
      });
    }

    const { _v } = this;
    const { _buffer, _header, _dataSource, _arrayBuffer } = this.reader;

    let data;

    // If we are in lazy mode (no full ArrayBuffer) and have a remote DataSource, fetch only the bytes we need
    const isRemote = _dataSource && typeof _dataSource.read === 'function' && typeof _dataSource.source === 'string';
    const haveFullBuffer = !!_arrayBuffer;

    if (!haveFullBuffer && isRemote && !_v.record) {
      // Currently only support non-record variables for range reads.
      const byteOffset = _v.offset;
      const byteLength = _v.size;
      const arrayBuffer = await _dataSource.read(byteOffset, byteLength);

      const tmpBuffer = new IOBuffer(arrayBuffer);
      tmpBuffer.setBigEndian();

      data = nonRecord(tmpBuffer, _v);
    } else {
      // Fallback to original in-memory buffer behaviour
      _buffer.seek(_v.offset);
      if (_v.record) {
        data = record(_buffer, _v, _header.recordDimension);
      } else {
        data = nonRecord(_buffer, _v);
      }
    }

    // Force type conversion if requested
    if (forceType) {
      // Map string names to constructors
      const typeMap = {
        int8: Int8Array,
        uint8: Uint8Array,
        int16: Int16Array,
        uint16: Uint16Array,
        int32: Int32Array,
        uint32: Uint32Array,
        float32: Float32Array,
        float64: Float64Array
      };
      const Ctor = typeMap[forceType.toLowerCase()];
      if (!Ctor) {
        throw new NetCDFError(`Unsupported force type: ${forceType}`, 'E_FORCE_TYPE');
      }
      data = new Ctor(data.buffer, data.byteOffset, data.length);
    }

    // Fast path: no slicing requested
    if (!hasSlice) {
      // Cache for zero-copy subarray feature when flat=true
      if (flat) this._cachedData = data;
      return flat ? data : Array.from(data);
    }

    // Apply slicing to obtain nested JS arrays
    const slices = normalizedSlices;
    const shape = _v.dimensions.map((idx) => this.reader._dimensions[idx].size);
    const sliced = sliceArray(data, shape, slices, 0);

    // If scalar result, return as-is (maintains previous semantics)
    if (!Array.isArray(sliced)) {
      return sliced;
    }

    if (!flat) {
      return sliced;
    }

    // Flatten nested array into same TypedArray constructor as source
    const flatArray = new data.constructor(countElements(sliced));
    flattenNested(sliced, flatArray);
    this._cachedData = flatArray;
    return flatArray;
  }

  /**
   * Return a zero-copy view into the variable's data.
   * @param {object} opts { start:number[], count:number[] }
   */
  subarray(opts) {
    const { start, count } = opts || {};
    if (!Array.isArray(start) || !Array.isArray(count)) {
      throw new NetCDFError('subarray requires {start:[], count:[]}', 'E_SUBARRAY_ARGS');
    }
    const shape = this.dimensions.map((d) => d.size);
    if (start.length !== shape.length || count.length !== shape.length) {
      throw new NetCDFError('start/count length mismatch', 'E_SUBARRAY_DIM');
    }

    // Ensure data cached
    if (!this._cachedData) {
      // synchronous use not allowed; instruct caller to read first
      throw new NetCDFError('Data not loaded; call read({flat:true}) first', 'E_SUBARRAY_NODATA');
    }

    // Compute linear offset in row-major order
    let stride = 1;
    let offset = 0;
    for (let i = shape.length - 1; i >= 0; i--) {
      offset += start[i] * stride;
      stride *= shape[i];
    }
    const totalElements = count.reduce((a, b) => a * b, 1);
    const bytesPerElem = num2bytes(str2num(this.type));
    const byteOffset = this._cachedData.byteOffset + offset * bytesPerElem;
    return new this._cachedData.constructor(this._cachedData.buffer, byteOffset, totalElements);
  }
}

// helper: count total number of leaf elements in nested arrays or scalars
function countElements(arr) {
  if (Array.isArray(arr) || ArrayBuffer.isView(arr)) {
    let sum = 0;
    for (const el of arr) sum += countElements(el);
    return sum;
  }
  return 1;
}

function flattenNested(source, dest) {
  let idx = 0;
  (function recurse(el) {
    // We treat both plain Arrays and TypedArrays as containers.
    const isContainer = Array.isArray(el) || ArrayBuffer.isView(el);
    if (isContainer) {
      for (const sub of el) recurse(sub);
    } else {
      dest[idx++] = el;
    }
  })(source);
  return dest;
}

class ModernNetCDFReader {
  constructor(arrayBuffer, parsed) {
    this._arrayBuffer = arrayBuffer;
    this._buffer = parsed.buffer;
    this._header = parsed.header;
    this._dimensions = this._header.dimensions;
    this._variables = this._header.variables;
  }

  /**
   * @param {string|ArrayBuffer} source  URL or ArrayBuffer
   * @param {object} [options]
   * @param {boolean} [options.lazy=false] If true and `source` is a URL, only the header is fetched. Variable data is fetched on demand via HTTP Range requests.
   */
  static async open(source, options = {}) {
    const { lazy = false } = options;
    const ds = new DataSource(source);

    let ab;
    if (lazy && typeof source === 'string') {
      // Attempt to fetch up to 4 MB for header. Typical NetCDF headers are small.
      const HEADER_BYTES = 4 * 1024 * 1024; // bytes to fetch for header
      ab = await ds.read(0, HEADER_BYTES);
    } else {
      ab = await ds.getArrayBuffer();
    }

    const parsed = parseFile(ab);
    const reader = new ModernNetCDFReader(ab, parsed);
    reader._dataSource = ds;

    // If lazy mode, clear stored full buffer to free memory.
    if (lazy) {
      reader._arrayBuffer = null;
    }
    return reader;
  }

  get dimensions() {
    return this._dimensions.map((d) => ({ name: d.name, size: d.size }));
  }

  // Deprecated: returns a map of { [name]: size }. Will be removed in v2.
  get dimensionMap() {
    const out = {};
    this._dimensions.forEach((d) => {
      out[d.name] = d.size;
    });
    return out;
  }

  get variables() {
    const out = {};
    this._variables.forEach((v) => {
      out[v.name] = new NetCDFVariable(this, v);
    });
    return out;
  }

  get globalAttributes() {
    return this._header.globalAttributes;
  }

  /**
   * Attempt to derive CRS / projection information from global or variable attributes.
   * Returns an EPSG code string (e.g., "EPSG:4326") or null.
   */
  getProjection() {
    // Check global attributes first
    for (const attr of this.globalAttributes) {
      if (typeof attr.value === 'string' && /EPSG:\d+/i.test(attr.value)) {
        const match = attr.value.match(/EPSG:\d+/i);
        if (match) return match[0].toUpperCase();
      }
    }
    // Search variable attributes for grid_mapping / spatial_ref
    for (const v of this._variables) {
      if (!v.attributes) continue;
      for (const a of v.attributes) {
        if (typeof a.value === 'string' && /EPSG:\d+/i.test(a.value)) {
          const m = a.value.match(/EPSG:\d+/i);
          if (m) return m[0].toUpperCase();
        }
      }
    }
    return null;
  }

  getVariable(name) {
    const v = this._variables.find((a) => a.name === name);
    if (!v) throw new NetCDFError('Variable not found', 'E_VAR_NOT_FOUND');
    return new NetCDFVariable(this, v);
  }

  async getData(variableName, sliceSpec) {
    const v = this.getVariable(variableName);
    return v.read(sliceSpec);
  }

  close() {
    this._arrayBuffer = null;
  }
}

// Internal helper to validate magic bytes
function parseFile(arrayBuffer) {
  // Detect HDF5/NetCDF-4 files and fail fast with a helpful error. The
  // NetCDF-4 format is HDF5 and begins with the eight-byte sequence
  // 0x89 0x48 0x44 0x46 0x0d 0x0a 0x1a 0x0a ("\x89HDF\r\n\x1a\n").
  if (arrayBuffer.byteLength >= 8) {
    const hdf5Magic = [0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a];
    const first8 = new Uint8Array(arrayBuffer, 0, 8);
    let isHdf5 = true;
    for (let i = 0; i < hdf5Magic.length; i++) {
      if (first8[i] !== hdf5Magic[i]) {
        isHdf5 = false;
        break;
      }
    }
    if (isHdf5) {
      throw new NetCDFError(
        'Unsupported NetCDF-4/HDF5 file detected (starts with HDF5 magic bytes). ' +
        'Please convert the file to classic NetCDF-3 with a tool like "nccopy -k classic".', 'E_HDF5');
    }
  }
  const buffer = new IOBuffer(arrayBuffer);
  buffer.setBigEndian();

  // Validate magic 'CDF'
  if (buffer.byteLength < 3) {
    throw new NetCDFError('Buffer too short to be a valid NetCDF file', 'E_TOO_SHORT');
  }
  const magic = buffer.readChars(3);
  if (magic !== 'CDF') {
    throw new NetCDFError('Not a valid NetCDF file: should start with CDF', 'E_MAGIC');
  }

  // Check version
  const version = buffer.readByte();
  if (version !== 1 && version !== 2) {
    throw new NetCDFError(`Unsupported NetCDF version: ${version}. Only classic (1) and 64-bit offset (2) formats are supported.`, 'E_VERSION');
  }

  const hdr = parseHeader(buffer, version);
  return { buffer, header: hdr };
}

// helper to extract a sub section of either TypedArray or Array
function viewSlice(arr, start, end) {
  return typeof arr.subarray === 'function' ? arr.subarray(start, end) : arr.slice(start, end);
}

function sliceArray(arr, shape, slices, dim = 0) {
  const isLeaf = dim === shape.length - 1;
  const slice = slices[dim];
  const dimSize = shape[dim];
  const stride = shape.slice(dim + 1).reduce((a, b) => a * b, 1);

  // Avoid expensive copies: TypedArrays are indexable like normal arrays. Keep them as-is.
  const array = arr;

  // Single index selection
  if (slice.index !== undefined) {
    if (slice.index < 0 || slice.index >= dimSize) {
      throw new NetCDFError('Slice index out of bounds', 'E_SLICE_OOB');
    }
    const view = viewSlice(array, slice.index * stride, (slice.index + 1) * stride);
    return isLeaf ? (view.length === 1 ? view[0] : view)
                  : sliceArray(view, shape, slices, dim + 1);
  }

  // Select all indices along this dimension
  if (slice.type === 'all') {
    if (isLeaf) {
      return array;
    }
    const result = new Array(dimSize);
    for (let i = 0; i < dimSize; i++) {
      const sub = viewSlice(array, i * stride, (i + 1) * stride);
      result[i] = sliceArray(sub, shape, slices, dim + 1);
    }
    return result;
  }

  // Range selection
  const { start, end, stride: st } = slice;
  if (st <= 0) throw new NetCDFError('Invalid stride', 'E_STRIDE');
  if (start < 0 || end > dimSize) throw new NetCDFError('Slice range out of bounds', 'E_SLICE_RANGE');
  if (start >= end) return isLeaf ? [] : [[]];

  const size = Math.ceil((end - start) / st);
  const result = new Array(size);
  let idx = 0;
  for (let i = start; i < end; i += st) {
    const sub = viewSlice(array, i * stride, (i + 1) * stride);
    result[idx++] = isLeaf ? sub[0] : sliceArray(sub, shape, slices, dim + 1);
  }
  return result;
}

module.exports = ModernNetCDFReader;