dasf-web/lib/map/model/NetcdfRasterSource.ts

Web frontend components for the data analytics software framework (DASF)

import { Extent, createEmpty, intersects } from 'ol/extent';
import ImageSource from 'ol/source/Image';
import ImageBase from 'ol/ImageBase';
import ImageState from 'ol/ImageState';
import { fromLonLat, transformExtent, ProjectionLike } from 'ol/proj.js';
import axios from 'axios';
import netcdfjs from 'netcdfjs';
import DateMap from '../../util/DateMap';
import NumericalParameter from './Parameter';
import Gradient, { Spectral } from '../../colors/Gradient';
import TemporalData from './TemporalData';
import DateRange from '../../dateSelection/daterangepicker/DateRange';
import Filter from './Filter';
import IHasFilter from './HasFilter';
import FilterThresholdCalculator from './FilterThresholdCalculator';
import HasParameterGradient from './HasParameterGradient';
import EpsgIO from '../utils/EpsgIO';
import { register } from 'ol/proj/proj4';
import proj4 from 'proj4';
import { ICancel } from 'typescript-observable/dist/interfaces/cancel';
import MathUtils from '../../util/MathUtils';
import ArrayUtils from '../../util/ArrayUtils';
import Histogram from '../../model/Histogram';
import b64a from 'base64-arraybuffer';
import VectorSource from 'ol/source/Vector';
import Geometry from 'ol/geom/Geometry';
import Point from 'ol/geom/Point';
import { Feature } from 'ol';
import MultiPoint from 'ol/geom/MultiPoint';
import MultiPolygon from 'ol/geom/MultiPolygon';
import Polygon from 'ol/geom/Polygon';

export type Raster = number[];

export class Coverage {
  public data: Raster;
  public param: NumericalParameter;
  public readonly date: Date;
  public readonly width: number;
  public readonly height: number;
  public readonly invertYAxis: boolean;

  public constructor(data: Raster, param: NumericalParameter, date: Date, width: number, height: number, invertYAxis: boolean) {
    this.data = data;
    this.param = param;
    this.date = date;
    this.width = width;
    this.height = height;
    this.invertYAxis = invertYAxis;
  }

  public getImage(gradient?: Gradient, filter?: Filter): HTMLCanvasElement {
    if (!gradient) {
      // no gradient given - create a temporary one
      gradient = this.createDefaultGradient();
    }

    return this.renderImage(gradient, filter);
  }

  private createDefaultGradient(): Gradient {
    let min: number = (this.param.hasMin() ? this.param.getMin() : MathUtils.min(this.data)) as number;
    let max: number = (this.param.hasMax() ? this.param.getMax() : MathUtils.max(this.data)) as number;

    // do we have a fill value?
    if (!this.param.hasFill()) {
      // no fill value - guess one
      if (min <= -2147483648) {
        // use min as no data value
        // replace all min values with NaN and find new min
        MathUtils.replace(min, Number.NaN, this.data);
        min = MathUtils.min(this.data);
        this.param.setFill(Number.NaN);
      }
    }

    if (!this.param.hasMin()) {
      this.param.setMin(min);
    }
    if (!this.param.hasMax()) {
      this.param.setMax(max);
    }
    return new Gradient(Spectral, min, max);
  }

  private renderImage(gradient: Gradient, filter?: Filter): HTMLCanvasElement {
    // initialize canvas with detected dimensions
    let canvas: HTMLCanvasElement = document.createElement('canvas');
    canvas.width = this.width;
    canvas.height = this.height;

    // get the rendering context to put the pixel values from the data set variables
    let ctx: CanvasRenderingContext2D | null = canvas.getContext('2d');
    if (ctx) {
      // diable antialising
      ctx.imageSmoothingEnabled = false;

      // init image and pixel array
      let image: ImageData = ctx.createImageData(this.width, this.height);
      let pixels: Uint8ClampedArray = image.data;

      let noDataValue = this.param.getFill();
      let hasNoData = this.param.hasFill();

      // put the raster data into the image pixels
      for (let y = 0; y < this.height; ++y) {
        for (let x = 0; x < this.width; ++x) {
          let dataIdx = y * this.width + x;
          let value = this.data[dataIdx];

          if (Number.isNaN(value) || (hasNoData && value === noDataValue) || (filter != null && !filter.filter(value))) {
            // no data - skip
            // let pix = [0, 0, 0, 255];
            // pixels.set(pix, idx * 4);
            continue;
          }

          let pixelIdx = dataIdx;
          if (this.invertYAxis) {
            pixelIdx = ((this.height - 1 - y) * this.width + x);
          }

          // get color from gradient and put pixel values
          let pix = gradient.getColorRGBA(value);
          pixels.set(pix, pixelIdx * 4);
        }
      }

      // write the image into the canvas context
      ctx.putImageData(image, 0, 0);

    } else {
      console.error(
        'unable to create canvas rendering context'
      );
    }

    return canvas;
  }

  public extractExtent ( extent: Extent, filter?: Filter ): number[] {
    const values: number[] = []

    let noDataValue = this.param.getFill();
      let hasNoData = this.param.hasFill();

      // put the raster data into the image pixels
      for (let y = extent[1]; y < extent[3]; ++y) {
        for (let x = extent[0]; x < extent[2]; ++x) {
          let dataIdx = y * this.width + x;
          let value = this.data[dataIdx];

          if (Number.isNaN(value) || (hasNoData && value === noDataValue) || (filter != null && !filter.filter(value))) {
            // no data - skip
            continue;
          }

          values.push(value)
        }
      }

    return values
  }

  public getHistogram(numBuckets: number, filter: Filter, sampleBounds: [number, number] = [NaN, NaN]): Histogram {
    let min = Number.NaN;

    if (!Number.isNaN(sampleBounds[0])) {
      min = sampleBounds[0];
    } else if (filter) {
      if (this.param.hasMin()) {
        // there is a filter and a known parameter minimum
        min = Math.max(filter.threshold[0], this.param.getMin());
      } else {
        min = filter.threshold[0];
      }
    } else if (this.param.hasMin()) {
      // no filter, but parameter min
      min = this.param.getMin();
    }


    if (Number.isNaN(min)) {
      // unknown min - determine
      min = MathUtils.min(this.data);
    }

    let max = Number.NaN;
    if (!Number.isNaN(sampleBounds[1])) {
      max = sampleBounds[1];
    } else if (filter) {
      if (this.param.hasMax()) {
        // there is a filter and a known parameter maximum
        max = Math.min(filter.threshold[1], this.param.getMax());
      } else {
        max = filter.max;
      }
    } else if (this.param.hasMax()) {
      // no filter, but parameter min
      max = this.param.getMax();
    }

    if (Number.isNaN(max)) {
      // unknown max - determine
      max = MathUtils.max(this.data);
    }

    return ArrayUtils.histogram(this.data, numBuckets, this.param.getFill(), min, max);
  }

  public toJson(filter?: Filter): object {

    let dataObject: object = {};
    dataObject["dimension"] = { width: this.width, height: this.height, inverted: this.invertYAxis };
    dataObject["date"] = this.date.toISOString();
    let data = this.data;

    if (filter) {
      // apply filter before encoding
      // set nodata
      let noData = this.param.hasFill() ? this.param.getFill() : Number.NaN;
      // clone array
      data = Object.assign([], this.data);
      // set filtered values to noData
      for (let i = 0; i < data.length; ++i) {
        let value = data[i];
        if (Number.isNaN(value) || value == noData) {
          // the value already is noData - ignore
          continue;
        } else if (!filter.filter(value)) {
          // this value does not satisfy the filter - set to noData
          data[i] = noData;
        }
      }
    }

    dataObject["data"] = b64a.encode(new Float32Array(data).buffer);

    return dataObject;
  }
}

class RenderedConverage extends ImageBase {

  private _canvas: HTMLCanvasElement | undefined;

  public constructor(extent, resolution) {
    super(extent, resolution, 1, ImageState.LOADED);
  }

  public getImage(): HTMLCanvasElement | undefined {
    return this._canvas;
  }

  public renderCoverage(coverage: Coverage, gradient: Gradient | undefined, filter?: Filter): void {
    // we need to sqeeze the coverage into its defined extent
    // therefore we 'draw' the coverage image into a new canvas having the same ratio as the extent
    let extent: Extent = this.getExtent();
    let extentRatio = (extent[2] - extent[0]) / (extent[3] - extent[1]);
    let canvasWidth = coverage.height * extentRatio;
    let canvasHeight = coverage.height;

    let renderedCanvas: HTMLCanvasElement = document.createElement('canvas');
    renderedCanvas.width = canvasWidth
    renderedCanvas.height = canvasHeight;

    let ctx: CanvasRenderingContext2D | null = renderedCanvas.getContext('2d');
    if (ctx) {
      // diable antialising
      ctx.imageSmoothingEnabled = false;
      ctx.drawImage(coverage.getImage(gradient, filter), 0, 0, canvasWidth, canvasHeight);
    }

    this._canvas = renderedCanvas;
  }

  public load(): void {
    // nothing to do here
  }

}

export default class NetcdfRasterSource extends ImageSource implements TemporalData, IHasFilter, HasParameterGradient {
  public static readonly DATA_PROP = 'data';
  public static readonly PARAMETER_PROP = 'parameter';

  private dateCoverageMap: DateMap<Map<string, Coverage>>;
  private activeDate: Date;
  private activeParameter: string;
  private dateRange: DateRange;
  private parameters: NumericalParameter[];
  private gradient: Gradient;
  private filter?: Filter;
  private extent: Extent;
  private resolution = 0;
  private coverageExtent: Extent;
  private projection: ProjectionLike;

  private static VALID_MAX = (Math.pow(2, 31) - 1);

  /**
   * internal rendered image
   */
  private _image: RenderedConverage;

  // TODO: add means to alter the shown date, paramerter and the used colorscale

  public constructor(staticSourceOptions: object) {
    super(staticSourceOptions);

    this.dateCoverageMap = staticSourceOptions['coverages'];
    this.activeDate = staticSourceOptions['date'] as Date;
    this.activeParameter = staticSourceOptions['parameter'] as string;
    this.parameters = staticSourceOptions['availableParameters'];
    this.dateRange = new DateRange(...this.dateCoverageMap.dateRange());
    this.extent = staticSourceOptions['imageExtent'];
    this.coverageExtent = staticSourceOptions['coverageExtent'];

    if (!this.coverageExtent) {
      // missing coverageExtent - use imageExtent as fallback
      this.coverageExtent = this.extent;
    }

    this.resolution = staticSourceOptions['resolution'];
    this.projection = staticSourceOptions['projection'];

    this._image = new RenderedConverage(this.coverageExtent, this.resolution);
    this.bootstrapFilter();
    this.update();
  }

  public static async create(options: object): Promise<NetcdfRasterSource> {
    return new NetcdfRasterSource(await NetcdfRasterSource.toStaticSourceOptions(options));
  }

  public static createFromArray(options: object): NetcdfRasterSource {
    return new NetcdfRasterSource(NetcdfRasterSource.loadFromArray(options));
  }

  private static async toStaticSourceOptions(options: object): Promise<object> {
    if (options.hasOwnProperty(NetcdfRasterSource.DATA_PROP)) {
      let data: object = options[NetcdfRasterSource.DATA_PROP];
      if (typeof data === 'string') {
        return NetcdfRasterSource.loadFromUrl(options);
      } else if (data instanceof ArrayBuffer) {
        return NetcdfRasterSource.loadFromBuffer(options);
      } else if (data instanceof Blob) {
        // (data as Blob).arrayBuffer()
        return NetcdfRasterSource.loadFromBlob(options);
      } else if (data instanceof Array) {
        return NetcdfRasterSource.loadFromArray(options);
      } else {
        console.warn('unsupported data option');
        console.warn(options);
      }
    } else {
      console.warn('missing mandatory data property in options');
    }

    return {};
  }

  private static loadFromArray(options: object): object {
    // check neseccary options
    if (!options.hasOwnProperty(NetcdfRasterSource.DATA_PROP)) {
      console.warn('missing :data: option property for array2raster_source');
      return {};
    }
    if (!options.hasOwnProperty(NetcdfRasterSource.PARAMETER_PROP)) {
      console.warn('missing :parameter: option property for array2raster_source');
      return {};
    }

    // extract options parameters        
    let data: Raster = options[NetcdfRasterSource.DATA_PROP];
    let parameter: NumericalParameter = options[NetcdfRasterSource.PARAMETER_PROP];
    let date: Date = options.hasOwnProperty('date') ? options['date'] : new Date();
    let extent: Extent = options['imageExtent'];
    let imgSize: number[] = options['imageSize'];
    let invertYAxis: boolean = options.hasOwnProperty('invertYAxis') ? options['invertYAxis'] : false;

    // initialize coverage
    let coverage: Coverage = new Coverage(data, parameter, date, imgSize[0], imgSize[1], invertYAxis);

    // initialize parameter map
    let parameterMap: Map<string, Coverage> = new Map<string, Coverage>();
    parameterMap.set(parameter.getName(), coverage);

    // initialize date coverage map
    let dateCoverageMap: DateMap<Map<string, Coverage>> = new DateMap<Map<string, Coverage>>();
    dateCoverageMap.set(date, parameterMap);

    // determine resolution
    let yResolution = (extent[3] - extent[1]) / imgSize[1];

    return {
      coverages: dateCoverageMap,
      date: date,
      parameter: parameter.getName(),
      availableParameters: [parameter],
      // [left, bottom, right, top]
      imageExtent: extent,
      resolution: yResolution,
      projection: 'EPSG:3857',
      imageSmoothing: false
    };
  }

  private static async loadFromUrl(options: object): Promise<object> {
    // TODO: load from url via axios, then load buffered netcdf
    const response = await axios.get(options[NetcdfRasterSource.DATA_PROP], { responseType: 'arraybuffer' });
    options[NetcdfRasterSource.DATA_PROP] = response.data;
    return NetcdfRasterSource.loadFromBuffer(options);
  }

  private static async loadFromBlob(options: object): Promise<object> {
    let buffer = await new Response(options[NetcdfRasterSource.DATA_PROP]).arrayBuffer();

    if (buffer) {
      options[NetcdfRasterSource.DATA_PROP] = buffer;
      return NetcdfRasterSource.loadFromBuffer(options);
    } else {
      console.warn('unable to read blob/file.');
      return {};
    }
  }

  private static async loadFromBuffer(options: object): Promise<object> {
    let reader: netcdfjs = new netcdfjs(options[NetcdfRasterSource.DATA_PROP]);

    // build the date coverage map
    let coverageBuild: object = NetcdfRasterSource.buildDateCoverageMap(reader);
    if (coverageBuild == undefined) {
      throw new Error('error building coverages from netcdf data');
    }

    let dateCoverageMap: DateMap<Map<string, Coverage>> = coverageBuild['map'];
    let parameters: NumericalParameter[] = coverageBuild['parameters'];
    if (!dateCoverageMap) {
      // no or unsupported data found
      return {};
    }

    let extent = await NetcdfRasterSource.extractExtent(reader);
    if (extent.length < 4) {
      // invalid extent
      return {};
    }

    let firstEntry = dateCoverageMap.entries().next().value;
    let firstParam = firstEntry[1].entries().next().value;
    let firstCoverage: Coverage = firstParam[1];
    let crs: string = 'crs' in extent ? extent['crs'] : 'EPSG:3857';
    let crsExtent: Extent = extent['crsExtent'];

    // calculate resoultion as the ratio of real world extent and image size
    let realworldHeight = extent[3] - extent[1];
    let yResolution = (realworldHeight / firstCoverage.height);

    return {
      coverages: dateCoverageMap,
      date: firstEntry[0],
      parameter: firstParam[0],
      availableParameters: parameters,
      // [left, bottom, right, top]
      imageExtent: extent,
      coverageExtent: crsExtent,
      resolution: yResolution,
      projection: crs,
      imageSmoothing: false
    };
  }

  private static async extractExtent(reader: netcdfjs): Promise<Extent> {
    console.log('extracting extent...');
    // first: check for 'geospatial min/max values'
    var latMin = reader.getAttribute('geospatial_lat_min');
    var latMax = reader.getAttribute('geospatial_lat_max');
    var lonMin = reader.getAttribute('geospatial_lon_min');
    var lonMax = reader.getAttribute('geospatial_lon_max');

    if (latMin !== null && latMax !== null && lonMin !== null && lonMax !== null) {
      // convert to map projection
      let minPos = fromLonLat([lonMin, latMin]);
      let maxPos = fromLonLat([lonMax, latMax]);

      return [minPos[0], minPos[1], maxPos[0], maxPos[1]];
    }

    // second: extract min/max values from lat/lon dim variables
    let lats: number[] = [];
    let lons: number[] = [];
    if (reader.dataVariableExists('lat')) {
      lats = reader.getDataVariable('lat');
    } else if (reader.dataVariableExists('latitude')) {
      lats = reader.getDataVariable('latitude');
    } else if (reader.dataVariableExists('Lat')) {
      lats = reader.getDataVariable('Lat');
    }
    if (reader.dataVariableExists('lon')) {
      lons = reader.getDataVariable('lon');
    } else if (reader.dataVariableExists('longitude')) {
      lons = reader.getDataVariable('longitude');
    } else if (reader.dataVariableExists('Long')) {
      lons = reader.getDataVariable('Long');
    } else if (reader.dataVariableExists('Lon')) {
      lons = reader.getDataVariable('Lon');
    }

    if (lats.length > 0 && lons.length > 0) {
      latMin = MathUtils.min(lats);
      latMax = MathUtils.max(lats);
      lonMin = MathUtils.min(lons);
      lonMax = MathUtils.max(lons);

      if (latMin !== undefined && latMax !== undefined && lonMin !== undefined && lonMax !== undefined) {
        // convert to map projection
        let minPos = fromLonLat([lonMin, latMin]);
        let maxPos = fromLonLat([lonMax, latMax]);

        // FIXME: deal with a raster that has only one pixel and therefore identical min/max for lat and lon
        // as a workaround we render a pixel roughly one degree in size, this size should be extracted from the meta data.
        const delta = 111100 / 2;

        if (minPos[0] == maxPos[0]) {
          minPos[0] -= delta
          maxPos[0] += delta
        }

        if (minPos[1] == maxPos[1]) {
          minPos[1] -= delta
          maxPos[1] += delta
        }

        console.log(minPos, maxPos);
        return [minPos[0], minPos[1], maxPos[0], maxPos[1]];
      } else {
        console.warn('unable to extract spatial bounds from netcdf');
        return createEmpty();
      }
    }

    // third check x/y and crs
    // check crs first
    let crs: string = NetcdfRasterSource.findCrsAttributeValue(reader.globalAttributes);
    if (crs === undefined || crs.length == 0) {
      console.warn('unable to extract spatial bounds from netcdf');
      return createEmpty();
    }

    let proj4def: string = '';
    if (crs.startsWith('+')) {
      // found proj4 def - use it right away
      proj4def = crs;
    } else {
      if(!isNaN(+crs)) {
        // number only - prepend epsg
        crs = 'epsg:'+ crs;
      }

      // propably epsg string - query EpsgIO for proj4 definition
      crs = crs.toUpperCase();
      proj4def = await EpsgIO.search(crs);
    }

    if (!proj4def || proj4def.length == 0) {
      throw new Error('unable to find projection parameters for crs: ' + crs);
    }
    // register proj4 definition
    proj4.defs(crs, proj4def);
    register(proj4);

    // we have a src crs
    let x: number[] = [];
    let y: number[] = [];
    if (reader.dataVariableExists('x')) {
      x = reader.getDataVariable('x') as number[];
    }
    if (reader.dataVariableExists('y')) {
      y = reader.getDataVariable('y') as number[];
    }

    // setUserProjection(crs);
    // let min = fromUserCoordinate([x[0], y[0]]);
    // let max = fromUserCoordinate([x[x.length-1], y[y.length-1]], crs);
    // clearUserProjection();
    // extent in view projection (epsg:3857)
    // minX = 1525338.53
    // minY = 6569752.93
    // maxX = 1547490.80
    // maxY = 6617676.68

    // let extent: Extent = [min[0], min[1], max[0], max[1]];
    let crsExtent: Extent = [x[0], y[0], x[x.length - 1], y[y.length - 1]];
    if (crsExtent[3] < crsExtent[1]) {
      // inverted y
      let temp = crsExtent[1];
      crsExtent[1] = crsExtent[3];
      crsExtent[3] = temp;
    }

    let extent: Extent;
    if (crs.toUpperCase() === 'EPSG:3857') {
      extent = crsExtent;

      // ensure we have square pixels
      const yRes = (extent[3] - extent[1]) / y.length;
      const ratioDif = 0.5 * ((extent[2] - extent[0]) - (yRes * x.length))
      extent[0] += ratioDif;
      extent[2] -= ratioDif;

      // for the corner pixels to cover the extent locations, we have to move it by half a pixel each
      extent[0] -= yRes/2;
      extent[2] += yRes/2;

      extent[1] -= yRes/2 * y.length / x.length;  // keep aspect ratio
      extent[3] += yRes/2 * y.length / x.length;  // keep aspect ratio
    } else {
      extent = transformExtent(crsExtent, crs, 'EPSG:3857')
    }

    // console.log('layer extent in 4326', transformExtent(crsExtent, crs, 'EPSG:4326'));
    // let extent: Extent = [1525338.53, 6569752.93, 1547490.80, 6617676.68];
    extent['crs'] = crs;
    extent['crsExtent'] = crsExtent;
    return extent;
    // console.warn('unable to extract spatial bounds from netcdf');
    // return createEmpty();


  }

  private static buildDateCoverageMap(reader: netcdfjs): object | undefined {
    let map: DateMap<Map<string, Coverage>> = new DateMap<Map<string, Coverage>>();

    // get the defined dimensions
    let dimensions: object = reader['dimensions'];
    if (!dimensions) {
      console.warn(
        'dataset does not provide <dimension> information - ignoring it'
      );
      return undefined;
    }

    // get lat/lon dimensions
    let width = 0;
    let height = 0;
    let timeDim = 0;
    let timeVarName = 'time';
    let xVarName = 'lon';
    let yVarName = 'lat';
    let projectedDataset = false;

    for (let dim of dimensions as object[]) {
      let name: string = dim['name'];
      let size: number = dim['size'];

      switch (name) {
        case 'x':
        case 'easting':
          projectedDataset = true;
          width = size;
          xVarName = name;
          break;
        case 'lon':
        case 'long':
        case 'longitude':
        case 'Lon':
        case 'Long':
          projectedDataset = false;
          width = size;
          xVarName = name;
          break;
        case 'y':
        case 'northing':
          projectedDataset = true;
          height = size;
          yVarName = name;
          break;
        case 'lat':
        case 'latitude':
        case 'Lat':
        case 'Latitude':
          projectedDataset = false;
          height = size;
          yVarName = name;
          break;
        case 'time':
        case 'ansi':
          timeDim = size;
          timeVarName = name;
          break;
        default:
          console.info("ignoring unknown dimension '" + name + "'");
          break;
      }
    }

    if (width == 0 || height == 0) {
      console.warn(
        'unable to detect lat/lon dimension size - ignoring data set'
      );
      return undefined;
    }

    let times: number[] = [0];
    if (timeDim || reader.dataVariableExists(timeVarName)) {
      times = reader.getDataVariable(timeVarName);
      timeDim = times.length;
    }

    let parameters: NumericalParameter[] = NetcdfRasterSource.getParameters(reader);
    let numValues = width * height;

    let timeUnitScale: number[] = NetcdfRasterSource.extractTimeUnitScale(reader, timeVarName);
    let invertYAxis: boolean = NetcdfRasterSource.isYAxisInverted(reader, yVarName);

    // check before hand, if there are global min/max values provided by the netcdf head
    let hasGlobalRange: Set<NumericalParameter> = new Set();
    for (let param of parameters) {
      if (param.hasMin() && param.hasMax()) {
        hasGlobalRange.add(param);
      }
    }


    for (let t = 0; t < times.length; ++t) {
      // decode date, by converting unix seconds to milliseconds
      let date: Date = new Date(timeUnitScale[0] + times[t] * timeUnitScale[1]);

      if (timeVarName == 'ansi' && Number.isInteger(times[t])) {
        date.setHours(0, 0, 0, 0);
      }

      // initialize the date map value
      let dateMap: Map<string, Coverage> = new Map();
      map.set(date, dateMap);

      // extract all coverages for this date
      for (let param of parameters) {
        // get the data array for this parameter
        let offset = t * numValues;
        let data = reader.getDataVariable(param.getName());
        let raster: Raster = null;

        let renderOption: string = "";
        for (let dimPos of param.getDimensions()) {
          let dimName: string = dimensions[dimPos]['name'];
          if (dimName == timeVarName) {
            dimName = 'time';
          } else if (dimName == xVarName) {
            dimName = 'lon';
          } else if (dimName == yVarName) {
            dimName = 'lat';
          } else {
            console.warn('ignoring unknown dimension for rendering', dimName);
          }
          renderOption = renderOption + dimName;
        }


        if (data.length === timeDim) {
          // this is an array of arrays - select the one for the given time t
          raster = data[t];
        } else {
          switch (renderOption) {
            case 'latlontime':
              raster = new Array<number>(numValues);
              for (let i = 0; i < numValues; ++i) {
                raster[i] = data[i * times.length + t];
              }
              break;
            case 'timelatlon':
              raster = data.slice(offset, offset + numValues);
              break;
            default:
              raster = data.slice(offset, offset + numValues);
              break;
          }
        }

        if (!Array.isArray(raster)) {
          raster = [raster];
        }

        // update min max ranges if missing
        if (!hasGlobalRange.has(param)) {
          let min = MathUtils.min(raster, param.getFill());

          if (min <= -NetcdfRasterSource.VALID_MAX) {
            // propably MIN_INT fill value - replace
            param.setFill(Number.NaN)
            MathUtils.replace(min, Number.NaN, raster);
            min = MathUtils.min(raster);
          }

          if (Number.isFinite(min)) {
            if (!param.hasMin() || min < param.getMin()) {
              param.setMin(min)
            }
          }
        }

        if (!hasGlobalRange.has(param)) {
          let max = MathUtils.max(raster);
          if (Number.isFinite(max)) {
            if (!param.hasMax() || max > param.getMax()) {
              param.setMax(max)
            }
          }
        }

        dateMap.set(param.getName(), new Coverage(raster, param, date, width, height, invertYAxis));
      }
    }

    return { 'map': map, 'parameters': parameters };
  }

  private static isYAxisInverted(reader: netcdfjs, yVarName: string): boolean {
    let lats = [];
    if (reader.dataVariableExists(yVarName)) {
      lats = reader.getDataVariable(yVarName);
    }

    if (lats.length == 0) {
      console.warn('unable to find latitude variable, assuming default y axis orientation');
      return false;
    }

    // we have an inverted axis if the latitude values are in ascending order
    return lats[0] < lats[lats.length - 1];
  }

  private static extractTimeUnitScale(reader: netcdfjs, timeVarName: string): number[] {
    if (timeVarName == 'ansi') {
      // ansi time is days since 1601
      let offset = new Date('12/31/1600').getTime();
      let scale = 1000 * 60 * 60 * 24;
      return [offset, scale];
    }

    if ('header' in reader && 'variables' in reader['header']) {
      // find 'time' variable
      for (let variable of reader['header']['variables']) {
        if ('name' in variable && variable['name'] === timeVarName && 'attributes' in variable) {
          // find units attribute
          for (let attribute of variable['attributes']) {
            if ('name' in attribute && attribute['name'] === 'units' && 'value' in attribute) {
              // found time units attribute - parse it
              let value: string = attribute['value'];
              let split: string[] = value.split(' since ');

              let date: Date = new Date(split[1]);
              let offset = Date.UTC(date.getFullYear(), date.getMonth(), date.getDate(),
                date.getHours(), date.getMinutes(), date.getSeconds());
              let scale = 1000;

              switch (split[0]) {
                case 'months':
                  scale = 1000 * 60 * 60 * 24 * 29;
                case 'days':
                  scale = 1000 * 60 * 60 * 24;
                  break;
                case 'hours':
                  scale = 1000 * 60 * 60;
                  break;
                case 'minutes':
                  scale = 1000 * 60;
                  break;
                case 'seconds':
                  scale = 1000;
                  break;
                default:
                  console.warn('unable to parse time scale from units - assuming senconds');
                  scale = 1000;
                  break;
              }

              return [offset, scale];
            }
          }
        }
      }

    }
    console.warn('unable to extract time unit definition, assuming seconds since 1970')
    return [0, 1000];
  }

  private static getParameters(reader: netcdfjs): NumericalParameter[] {
    let parameters: NumericalParameter[] = [];

    for (let parameter of reader['header']['variables']) {
      if (parameter['dimensions'] && parameter['dimensions']['length'] < 2) {
        // this is no spatial parameter, maybe one of the axis like lat/lon/time - skip
        continue;
      }

      let name: string = parameter['name'];

      if (name === 'time' || name === 'ansi' || name === 'lat' || name === 'lon' || name === 'latitude' || name === 'longitude' || name === 'easting' || name === 'northing' || name.endsWith('_bnds')) {
        // axis parameter - skip
        continue;
      }

      let min: number | undefined = NetcdfRasterSource.findMinAttributeValue(parameter);
      let max: number | undefined = NetcdfRasterSource.findMaxAttributeValue(parameter);
      let fill: number | undefined = NetcdfRasterSource.findFillAttributeValue(parameter);
      let dimensions: number[] = parameter['dimensions'];

      parameters.push(new NumericalParameter(name, min, max, fill, dimensions))
    }

    return parameters;
  }

  private static findMinAttributeValue(parameter: object): number | undefined {
    // check for 'min'
    let min: string | number | undefined = NetcdfRasterSource.findAttributeValue(parameter, 'min');
    if (min !== undefined) {
      // found
      return Number(min);
    }
    // not found, check for 'valid_min'
    min = NetcdfRasterSource.findAttributeValue(parameter, 'valid_min');
    if (min !== undefined) {
      if (Number(min) < -NetcdfRasterSource.VALID_MAX) {
        // unlikely
        return undefined;
      }

      // found
      return Number(min);
    }

    // not found at all
    return undefined;
  }

  private static findMaxAttributeValue(parameter: object): number | undefined {
    // check for 'max'
    let max: string | number | undefined = NetcdfRasterSource.findAttributeValue(parameter, 'max');
    if (max !== undefined) {
      // found
      return Number(max);
    }
    // not found, check for 'valid_max'
    max = NetcdfRasterSource.findAttributeValue(parameter, 'valid_max');
    if (max !== undefined) {
      if (Number(max) > NetcdfRasterSource.VALID_MAX) {
        // unlikely
        return undefined;
      }

      // found
      return Number(max);
    }

    // not found at all
    return undefined;
  }

  private static findFillAttributeValue(parameter: object): number | undefined {
    // check for '_FillValue'
    let fill: string | number | undefined = NetcdfRasterSource.findAttributeValue(parameter, '_FillValue');
    if (fill !== undefined) {
      // found
      return Number(fill);
    }
    // not found, check for 'missing_value'
    fill = NetcdfRasterSource.findAttributeValue(parameter, 'missing_value');
    if (fill !== undefined) {
      // found
      return Number(fill);
    }
    // not found, check for '_NoData'
    fill = NetcdfRasterSource.findAttributeValue(parameter, '_NoData');
    if (fill !== undefined) {
      // found
      return Number(fill);
    }
    // not found, check for 'nodata_value'
    fill = NetcdfRasterSource.findAttributeValue(parameter, 'nodata_value');
    if (fill !== undefined) {
      // found
      return Number(fill);
    }

    // not found at all
    return undefined;
  }

  private static findAttributeValue(parameter: object, attributeName: string): string | number | undefined {
    for (let attr of parameter['attributes']) {
      if (attr['name'] === attributeName) {
        return attr['value'];
      }
    }

    return undefined;
  }

  private static findCrsAttributeValue(globalAttributes: object[]): string {
    for (let attr of globalAttributes) {
      if (attr['name'].toString().toUpperCase() === 'CRS') {
        return attr['value'];
      }
    }

    return undefined;
  }

  public splitToSingleParameterSources(): NetcdfRasterSource[] {
    let sources: NetcdfRasterSource[] = [];
    for (let parameter of this.parameters) {
      sources.push(this.cloneParameterSource(parameter));
    }

    return sources;
  }

  public cloneParameterSource(parameter: NumericalParameter): NetcdfRasterSource {
    // clone coverageMap
    let newCovMap = new DateMap<Map<string, Coverage>>();

    for (let date of this.dateCoverageMap.keys()) {
      let paraMap = this.dateCoverageMap.get(date);
      if (paraMap && paraMap.has(parameter.getName())) {
        let newParaMap = new Map<string, Coverage>();
        newParaMap.set(parameter.getName(), paraMap.get(parameter.getName()));
        newCovMap.set(date, newParaMap);
      }
    }

    let firstEntry = newCovMap.entries().next().value;
    let firstParam = firstEntry[1].entries().next().value;

    let opts = {
      coverages: newCovMap,
      date: firstEntry[0],
      parameter: firstParam[0],
      availableParameters: [parameter],
      imageExtent: this.extent,
      coverageExtent: this.coverageExtent,
      resolution: this.resolution,
      projection: this.projection
    }

    return new NetcdfRasterSource(opts);
  }

  /**
   * Vectorizes this raster source. A vector source for the active date is generated.
   * 
   * @param geometryType 
   * resulting geometry, possible values are:
   * - 'pixel-points': single point geometry per raster pixel, 
   * - 'pixel-squares': single squares per raster pixel (only works with 'epsg:3857' as raster crs)
   * @param groupingProperty 
   * pixel geometries are grouped into MultiGeometry features, 
   * there will be as many features as there are distinct property values. 
   * This discards all properties except for the given grouping property.
   * @returns 
   */
  public vectorize(geometryType: string = 'pixel-points', groupingByParameterName?: string, targetCrs: ProjectionLike = 'epsg:3857'): VectorSource {

    if (groupingByParameterName) {
      // check if we have this parameter
      const param = this.getAvailableParameters().find((p: NumericalParameter) => p.getName() === groupingByParameterName)
      if (param === undefined) {
        throw new Error('unknown grouping parameter name');
      }
    }

    // create geometries get a coverage first
    const cov: Coverage = this.getCoverage()
    const e: Extent = this.extent;
    const width = cov.width;
    const height = cov.height;
    const xDif = (e[2] - e[0]) / width;
    const yDif = (e[3] - e[1]) / height;
    const xOff = 0.5 * xDif;
    const yOff = 0.5 * yDif;

    const geometries: Geometry[] = [];

    for (let rasterY = 0; rasterY < height; ++rasterY) {
      const geometryY = e[1] + rasterY * yDif;
      for (let rasterX = 0; rasterX < width; ++rasterX) {
        const geometryX = e[0] + rasterX * xDif;

        // create geometry
        switch (geometryType) {
          case 'pixel-points':
            geometries.push(new Point([geometryX + xOff, geometryY + yOff]))
            break;
          case 'pixel-squares':
            geometries.push(new Polygon([[[geometryX, geometryY], [geometryX + xDif, geometryY], [geometryX + xDif, geometryY + yDif], [geometryX, geometryY + yDif], [geometryX, geometryY]]]));
            break;
        }
      }
    }

    // transform geometries to target projection
    if (this.projection.toString().localeCompare(targetCrs.toString(), undefined, { sensitivity: 'accent' }) !== 0) {
      geometries.forEach((geom) => geom.transform(this.projection, targetCrs));
    }

    // convert geometries into features
    const features: Feature[] = [];
    if (groupingByParameterName) {
      const raster: Raster = this.getCoverage(this.activeDate, groupingByParameterName).data;

      const valueMap: Map<number, Geometry[]> = new Map()
      for (let y = 0; y < height; ++y) {
        for (let x = 0; x < width; ++x) {
          const value: number = raster[y * width + x];
          let valueGeometries: Geometry[];
          if (valueMap.has(value)) {
            valueGeometries = valueMap.get(value);
          } else {
            valueGeometries = [];
            valueMap.set(value, valueGeometries);
          }

          valueGeometries.push(geometries[y * width + x]);
        }
      }

      // now that all geometries are grouped we can create multi geometry features for each value
      for (let [key, value] of valueMap.entries()) {
        let geom: Geometry;
        switch (geometryType) {
          case 'pixel-points':
            geom = new MultiPoint([]);
            value.forEach((point: Geometry) => (geom as MultiPoint).appendPoint(point as Point))
            break;
          case 'pixel-squares':
            geom = new MultiPolygon(value as Polygon[]);
            break;
        }

        const feature: Feature = new Feature(geom);
        feature.set(groupingByParameterName, key);
        features.push(feature);
      }
    } else {
      // no grouping - iterate over all parameter coverages and append parameter values to each geometry feature
      // convert geometries to features first
      geometries.forEach((geom) => features.push(new Feature(geom)))

      for (let parameter of this.getAvailableParameters()) {
        const raster: Raster = this.getCoverage(this.activeDate, parameter.getName()).data;
        for (let y = 0; y < height; ++y) {
          for (let x = 0; x < width; ++x) {
            features[y * width + x].set(parameter.getName(), raster[y * width + x]);
            // features[y*(width+1) + x].set(parameter.getName(), raster[y*width + x]);
          }
        }
      }
    }

    return new VectorSource({
      features: features,
    })
  }

  public getExtent(): Extent {
    return this.extent;
  }

  public getDate(): Date {
    return this.activeDate;
  }

  public setDate(date: Date): void {
    if (date === undefined) {
      return;
    }

    if (this.dateCoverageMap.has(date)) {
      // this is a valid date we have data for
      this.activeDate = date;
    } else {
      // this is not a valid date we have data for
      // find closest valid date
      let minDelta = Number.MAX_VALUE;
      let closestValidDate: Date = this.activeDate;
      for (let validDate of this.dateCoverageMap.keys()) {
        let delta = Math.abs(date.getTime() - validDate.getTime());
        if (delta < minDelta) {
          minDelta = delta;
          closestValidDate = validDate;
        }
      }
      this.activeDate = closestValidDate;
    }

    if (this.filter && !this.filter.useTimeseries) {
      // the date changed and the filter is bound to single rasters
      this.bootstrapFilter();
    }

    this.update();
  }

  public getDateRange(): DateRange {
    return this.dateRange;
  }

  public getAllowedDates(): Date[] {
    return Array.from(this.dateCoverageMap.keys());
  }

  public getAvailableParameters(): NumericalParameter[] {
    return this.parameters;
  }

  public getGradient(): Gradient | undefined {
    return this.gradient;
  }

  private gradientRegistration: ICancel = null;

  public setGradient(gradient: Gradient): void {
    this.gradient = gradient;

    if (this.gradientRegistration) {
      this.gradientRegistration.cancel();
    }

    this.gradientRegistration = this.gradient.on('changed', () => {
      // gradient changed - update raster
      this.update();
    })

    this.update();
  }

  public getActiveParameter(): NumericalParameter {
    return this.parameters.find((param) => { return param.getName() == this.activeParameter });
  }

  private bootstrapFilter(): void {
    // if (this.filter && this.filter.attribute == this.activeParameter) {
    //     return;
    // }

    let activeParam = this.getActiveParameter();
    if (activeParam == undefined) {
      return;
    }

    // create filter
    let filter: Filter;
    if (this.filter) {
      // we already have a filter - get and update
      filter = this.filter;
    } else {
      // no filter yet - create one
      filter = new Filter();
    }

    // initialize threshold calculator
    let filterCalculator: FilterThresholdCalculator = new FilterThresholdCalculator();

    // collect rasters of the active parameter to calculate thresholds
    let rasterList: Raster[] = [];
    if (filter.useTimeseries) {
      for (let date of this.getAllowedDates()) {
        rasterList.push(this.getCoverage(date, this.activeParameter).data);
      }
    } else {
      rasterList.push(this.getCoverage().data);
    }

    filterCalculator.determineStatisticalThresholds(rasterList, activeParam.getFill());

    if (!filter.isProtected) {
      filter.min = filterCalculator.getMin();
      filter.max = filterCalculator.getMax();
    }
    filter.median = filterCalculator.getMedian();
    filter.quartileQ1 = filterCalculator.getQuartileQ1();
    filter.quartileQ3 = filterCalculator.getQuartileQ3();
    filter.attribute = this.activeParameter;
    filter.setThreshold(filter.min, filter.max);

    if (!this.filter) {
      this.setFilter(filter);
    }
  }

  public setParameter(parameter: string): void {
    this.activeParameter = parameter;
    this.bootstrapFilter()
    this.update();
  }

  public setParameterAndGradient(parameter: string, gradient: Gradient): void {
    this.activeParameter = parameter;
    this.bootstrapFilter();
    this.setGradient(gradient);
  }


  public getFilter(): Filter | undefined {
    return this.filter;
  }

  private filterObserverRegistration: ICancel[] = [];

  public setFilter(filter: Filter): void {
    this.filter = filter;

    while (this.filterObserverRegistration.length > 0) {
      this.filterObserverRegistration.pop().cancel();
    }

    if (filter) {
      // the filter thresholds changed - update the rendered coverage
      this.filterObserverRegistration.push(filter.on('changed:threshold', () => {
        this.update();
      }
      ));
      // the filter use timeseries property changed - update thresholds
      this.filterObserverRegistration.push(filter.on('changed:useTimeseries', () => {
        this.bootstrapFilter();
      }
      ));
    }

    this.update();
  }

  public getImageInternal(extent): RenderedConverage | null {
    if (intersects(extent, (this._image as ImageBase).getExtent())) {
      return this._image;
    }
    return null;
  }

  public getHistogram(numBuckets: number, sampleBounds: [number, number] = [NaN, NaN]): Histogram {
    let coverage = this.getCoverage();
    if (coverage) {
      return coverage.getHistogram(numBuckets, this.filter, sampleBounds);
    }

    return undefined;
  }

  public getHistograms(numBuckets: number, sampleBounds: [number, number] = [NaN, NaN]): Histogram[] {
    let hists: Histogram[] = [];

    for (let date of this.getAllowedDates()) {
      let coverage = this.getCoverage(date);
      if (coverage) {
        hists.push(coverage.getHistogram(numBuckets, this.filter, sampleBounds));
      }
    }

    return hists;
  }

  public update(): void {
    let coverage = this.getCoverage();
    if (coverage) {
      this._image.renderCoverage(coverage, this.gradient, this.filter);
      this.changed();
    }
  }

  public getCoverage(date?: Date, parameter?: string): Coverage {
    let coverageMap = this.dateCoverageMap.get(date ? date : this.activeDate);
    if (coverageMap) {
      return coverageMap.get(parameter ? parameter : this.activeParameter);
    }

    return undefined;
  }

  public toJson(activeDateOnly: boolean = false, activeParameterOnly: boolean = false, filtered: boolean = false): object {

    let netcdfObject: object = {};
    // always return epsg:4326 projection
    netcdfObject["crs"] = 'EPSG:4326';
    netcdfObject["extent"] = transformExtent(this.extent, this.projection, 'EPSG:4326');

    let params: NumericalParameter[] = activeParameterOnly ? [this.getActiveParameter()] : this.getAvailableParameters();
    let dates: Date[] = activeDateOnly ? [this.activeDate] : this.getAllowedDates();
    let filter: Filter = filtered ? this.getFilter() : undefined;

    let parameters: object[] = [];
    for (let p of params) {
      let parameter: object = {
        name: p.getName(),
        fill: p.hasFill() ? p.getFill() : Number.NaN,
        min: p.getMin(),
        max: p.getMax()
      }

      let coverages: object[] = [];
      for (let d of dates) {
        let coverage = this.getCoverage(d, p.getName());
        if (coverage) {
          coverages.push(coverage.toJson(filter));
        }
      }

      parameter['coverages'] = coverages;
      parameters.push(parameter);
    }

    netcdfObject["parameters"] = parameters;

    return netcdfObject;
  }


  public extractValues ( extent: Extent ): DateMap<number[]> {
    const values: DateMap<number[]> = new DateMap()

    // check whether the given extent is relevant at all
    if ( !intersects(extent, this.extent) ) {
      // the requested extent is outside of this raster source - return empty map
      return values
    }

    // determine pixel coordinates to extract
    // identify coverage dimensions
    const width = this.getCoverage().width
    const height = this.getCoverage().height
    const invertYAxis = this.getCoverage().invertYAxis

    // convert to coverage projection
    const coverageExtractExtent = transformExtent(extent, 'EPSG:3857', this.projection)
    // limit the extraction extent to the coverage extent
    coverageExtractExtent[0] = Math.max(coverageExtractExtent[0], this.coverageExtent[0])
    coverageExtractExtent[1] = Math.max(coverageExtractExtent[1], this.coverageExtent[1])
    coverageExtractExtent[2] = Math.min(coverageExtractExtent[2], this.coverageExtent[2])
    coverageExtractExtent[3] = Math.min(coverageExtractExtent[3], this.coverageExtent[3])
    // transform to pixel coordinates via resolution
    const xRes = (this.coverageExtent[2] - this.coverageExtent[0]) / width
    const yRes = (this.coverageExtent[3] - this.coverageExtent[1]) / height

    const minX = Math.floor((coverageExtractExtent[0] - this.coverageExtent[0]) / xRes)
    const maxX = Math.ceil((coverageExtractExtent[2] - this.coverageExtent[0]) / xRes)
    let minY = Math.floor(height - ((coverageExtractExtent[3] - this.coverageExtent[1]) / yRes))
    let maxY = Math.ceil(height - ((coverageExtractExtent[1] - this.coverageExtent[1]) / yRes))
    if ( invertYAxis ) {
      minY = Math.floor((coverageExtractExtent[1] - this.coverageExtent[1]) / yRes)
      maxY = Math.floor((coverageExtractExtent[3] - this.coverageExtent[1]) / yRes)
    }
    
    const pixelExtent: Extent = [minX, minY, maxX, maxY]

    // loop over all coverages and extract pixel values for pixel extent
    this.getAllowedDates().forEach((date) => {
      const coverage: Coverage = this.getCoverage(date)
      const coverageValues: number[] = coverage.extractExtent(pixelExtent, this.filter)

      values.set(date, coverageValues)
    })

    return values
  }

}