dasf-web/lib/dataDiscovery/EsgfDataArchive.ts

Web frontend components for the data analytics software framework (DASF)

import { DataArchive, QueryProperty, Dataset, EnsembleItem } from './DataArchive';
import axios from 'axios';
import * as xmljs from 'xml-js';

export class EsgfDataset extends EnsembleItem implements Dataset {
    public static readonly DEFAULT_HIST_DATES: [Date, Date] = [new Date("1950-01-01"), new Date("2005-12-31")];
    public static readonly DEFAULT_RCP_DATES: [Date, Date] = [new Date("2006-01-01"), new Date("2100-12-31")];
    
    private title: string;
    private url: string;
    private startDate: Date;
    private endDate: Date;

    // "cordex.output.EUR-11.MPI-CSC.MPI-M-MPI-ESM-LR.historical.r1i1p1.REMO2009.v1.day.pr"
    public readonly project: string;
    public readonly product: string;
    public readonly domain: string;
    public readonly institute: string;
    public readonly drivingModel: string;
    public readonly experiment: string;
    public readonly ensemble: string;
    public readonly regionalModel: string;
    public readonly downscaling: string;
    public readonly timeFrequency: string;
    public readonly variable: string;



    constructor(title: string, url: string, start?: Date, end?: Date) {
        super();

        this.title = title;
        this.url = url;

        const titleSplit = title.split('.');
        this.project = titleSplit[0];
        this.product = titleSplit[1];
        this.domain = titleSplit[2];
        this.institute = titleSplit[3];
        this.drivingModel = titleSplit[4];
        this.experiment = titleSplit[5];
        this.ensemble = titleSplit[6];
        this.regionalModel = titleSplit[7];
        this.downscaling = titleSplit[8];
        this.timeFrequency = titleSplit[9];
        this.variable = titleSplit[10];

        if(start == undefined || start == null) {
            this.startDate = this.experiment == 'historical' ? EsgfDataset.DEFAULT_HIST_DATES[0] : EsgfDataset.DEFAULT_RCP_DATES[0];
        } else  {
            this.startDate = start;
        }
        
        if(end == undefined || end == null) {
            this.endDate = this.experiment == 'historical' ? EsgfDataset.DEFAULT_HIST_DATES[1] : EsgfDataset.DEFAULT_RCP_DATES[1];
        } else  {
            this.endDate = end;
        }
    }

    public getName(): string {
        return this.title;
    }

    public getUrl(): string {
        return this.url;
    }

    public getStartDate(): Date {
        return this.startDate;
    }

    public getEndDate(): Date {
        return this.endDate;
    }

    public getLabel(): string {
        return this.experiment;
    }

    public getTitle(): string {
        return this.url;
    }
}

export class EsgfDataArchive implements DataArchive {

    private static readonly QUERRIED_FIELDS = ['id', 'title', 'type', 'datetime_start', 'datetime_stop'];
    
    constructor(private dataNode: string, private propertyDefaults: Map<string, string[]> = new Map()) {
    }

    public async getQueryProperties(supportedProperties: string[] = ['*']): Promise<QueryProperty[]> {
        let props: QueryProperty[] = [];

        // fetch supported properties and their values via facet query
        let facetQueryResponse = await axios.get('http://' + this.dataNode + '/esg-search/search?limit=0&facets=' + supportedProperties.join(','));
        let facetQueryData = xmljs.xml2js(facetQueryResponse.data, { compact: true });
        let response = facetQueryData['response'];
        if (response) {
            // get 'facet_count' attribute
            let facets = this.getArrayAttribute(response, 'facet_counts');
            // get 'facet_fields' attribute
            facets = this.getArrayAttribute(facets, 'facet_fields');
            // get facet field array
            let facetFields: any[] = this.getLstArray(facets);

            // iterate over all facets and values
            if (facetFields) {
                for (let facet of facetFields) {
                    props.push(this.compileFullQueryProperty(facet));
                }
            }

            this.setDefaults(props);
        }

        return props;
    }


    public async queryDatasets(props: QueryProperty[]): Promise<Dataset[]> {
        // esg-search/search?
        let q = 'http://' + this.dataNode + '/esg-search/search?limit=1000&' + this.toQueryString(props) + "&fields=" + EsgfDataArchive.QUERRIED_FIELDS.join(',');
        // console.log(q);
        let datasetQueryResponse = await axios.get(q);
        // console.log(datasetQueryResponse.data);

        let dsQueryData = xmljs.xml2js(datasetQueryResponse.data, { compact: true });
        let doc = dsQueryData['response']['result'];

        let numResults = this.extractAttribute(doc, 'numFound');

        if (numResults == 0) {
            return [];
        }

        // there are results - continue
        doc = doc['doc'];
        this.checkFormat(doc, "Missing 'doc' attribute");

        let datasets: Dataset[] = [];

        for (let datasetEntry of doc) {
            datasets.push(this.compileDataset(datasetEntry));
        }
        // console.log(doc);

        return datasets;
    }

    private compileDataset(datasetEntry: any): EsgfDataset {
        // extract name
        let str = datasetEntry['str'];
        this.checkFormat(str, 'Invalid dataset format');
        if (!Array.isArray(str)) {
            str = [str];
        }

        let id = 'Unknown';
        let title = 'Unknown';

        for (let strItem of str) {
            let fieldName = this.extractAttribute(strItem, 'name');
            switch (fieldName) {
                case 'title':
                    title = strItem['_text'];
                    break;
                case 'id':
                    id = strItem['_text']
                    break;
                default:
                    console.warn('[EsgfDataArchive] ignored dataset string field: ' + fieldName);
            }
        }

        this.checkFormat(id, 'invalid dataset format');
        this.checkFormat(title, 'invalid dataset format');

        // extracting the url from the dataset meta data gives us a thredds catalog url which is deprecated
        // instead we build our own url based on the solr rest api that will give us the dataset file listing
        const url = 'http://' + this.dataNode + '/esg-search/search?type=File&limit=1000&format=application%2Fsolr%2Bjson&dataset_id=' + id;
        // e.g. https://esgf-data.dkrz.de/esg-search/search?type=File&dataset_id=cordex-reklies.output.EUR-11.GERICS.CCCma-CanESM2.historical.r1i1p1.REMO2015.v1.day.pr.v20170329|esgf1.dkrz.de&format=application%2Fsolr%2Bjson

        // extract date ranges
        let start: Date | undefined = undefined;
        let end: Date | undefined = undefined;
        const dates = datasetEntry['date'];

        if(dates != undefined && dates != null && Array.isArray(dates)) {
            for(let date of dates) {
                let fieldName = this.extractAttribute(date, 'name');
                switch(fieldName) {
                    case 'datetime_start':
                        start = new Date(date['_text']);
                        break;
                    case 'datetime_stop':
                        end = new Date(date['_text']);
                        break;
                    default:
                        console.warn('ignored date field: ' + fieldName);
                }
            }
        }

        console.log([title, url]);

        // compile and return dataset object
        return new EsgfDataset(title, url, start, end);
    }

    private compileFullQueryProperty(facet: any): QueryProperty {
        let p = new QueryProperty(this.extractAttribute(facet, 'name'));
        let values = facet['int'];
        if (values) {
            for (let value of values) {
                p.addValue(this.extractAttribute(value, 'name'));
            }
        }

        return p;
    }

    private extractAttribute(element: any, attributeName: string) {
        let name = element['_attributes'][attributeName];
        this.checkFormat(name, 'No ' + attributeName + ' attribute found');
        return name;
    }

    private checkFormat(element: any, errmsg: string) {
        if (!element) {
            throw new Error('Unsupported ESGF facet or dataset query response received: ' + errmsg);
        }
    }

    private getArrayAttribute(lst: any, name: string): any {
        // check for lst array
        if (!Array.isArray(lst)) {
            lst = this.getLstArray(lst);
        }

        let element = this.getLstAttribute(lst, name);
        this.checkFormat(element, name + " element not found");
        return element;
    }

    private getLstArray(obj: any): any[] {
        let array: any[] = obj['lst'];
        this.checkFormat(array, 'lst element not found');
        return array;
    }

    private getLstAttribute(lst: any, name: string): any {
        if (lst) {
            // find element with name attribute
            for (let e of lst) {
                if (this.extractAttribute(e, 'name') === name) {
                    return e;
                }
            }
        }
    }



    private toQueryString(props: QueryProperty[]): string {
        let facets: string[] = [];
        for (let qp of props) {
            if (qp.hasSelected()) {
                for (let item of qp.selected) {
                    facets.push(qp.name + "=" + item);
                }
            }
        }

        return facets.join("&");
    }

    private setDefaults(props: QueryProperty[]): void {
        for (let p of props) {
            if (this.propertyDefaults.has(p.name)) {
                for (let selected of this.propertyDefaults.get(p.name)) {
                    p.addSelected(selected);
                }
            }
        }
    }
}