dasf-web/lib/dataAcquisition/ThreddsCatalogCrawler.ts

Web frontend components for the data analytics software framework (DASF)

import CatalogCrawler from './CatalogCrawler';
import axios from 'axios';
import * as xmljs from 'xml-js';

export default class ThreddsCatalogCrawler implements CatalogCrawler {
    private readonly catalogFile = "catalog.xml";

    async startCrawling(basePath: string, recursive: boolean, datasetSuffix: string, onDatasetDiscovered: (datasetPath: string) => void, onFinish?: () => void): Promise<void> {

        let catalogUrl: string;
        if (basePath.endsWith(this.catalogFile)) {
            catalogUrl = basePath;
            // remove catalog file from base path for recursive crawling
            basePath = basePath.substr(0, basePath.length - this.catalogFile.length);
        } else {
            if (!basePath.endsWith("/")) {
                basePath += "/";
            }

            catalogUrl = basePath + this.catalogFile;
        }

        const response = await axios.get(catalogUrl);

        let catalog = xmljs.xml2js(response.data, { compact: true });
        catalog = catalog["thredds:catalog"];
        if (catalog) {
            // we have a catalog - get datasets
            let datasets = catalog["thredds:dataset"];

            if (recursive) {
                // get catalog references
                let subCatalogs = datasets["thredds:catalogRef"];

                if (subCatalogs) {
                    // there are more catalogs to crawl
                    subCatalogs.forEach(e => {
                        let catalogName = e._attributes['xlink:title'];
                        this.startCrawling(basePath + catalogName, recursive, datasetSuffix, onDatasetDiscovered);
                    });
                }
            }

            // crawl actual datasets
            datasets = datasets['thredds:dataset'];
            if (datasets) {
                if (!Array.isArray(datasets)) {
                    datasets = [datasets];
                }

                datasets.forEach(e => {
                    let datasetName = e._attributes['name'];
                    if (datasetName && datasetName.endsWith(datasetSuffix)) {
                        onDatasetDiscovered(basePath + datasetName);
                    }
                });
            }
        }

        if (onFinish) {
            onFinish();
        }
    }

}