factbook/lib/utils/get-country-data.js

Serves as an independent data scraping module, complete with ontology and full scraping ability for the CIA World Factbook site

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
var cheerio = require("cheerio");
var rp = require("request-promise");
var constants_1 = require("../constants/constants");
var globalStore_1 = require("../constants/globalStore");
var data_getters_1 = require("../scrapers/data-getters");
var country_to_id_1 = require("./country-to-id");
var numberOfScrapers = Object.keys(data_getters_1.dataScrapers).length;
function getCountryData(country, url) {
    if (country && url) {
        return rp(url, { timeout: constants_1.consts.BASE.DATA_REQUEST_TIMEOUT })
            .then(function (html) {
            var $ = cheerio.load(html);
            var countryId = country_to_id_1.countryToId(country.dataCode);
            data_getters_1.dataScrapers.getArea($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 1 / numberOfScrapers);
            data_getters_1.dataScrapers.getBackground($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 2 / numberOfScrapers);
            data_getters_1.dataScrapers.getBorderMapImg($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 3 / numberOfScrapers);
            data_getters_1.dataScrapers.getBorders($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 4 / numberOfScrapers);
            data_getters_1.dataScrapers.getClimate($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 5 / numberOfScrapers);
            data_getters_1.dataScrapers.getCoastLength($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 6 / numberOfScrapers);
            data_getters_1.dataScrapers.getElevation($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 7 / numberOfScrapers);
            data_getters_1.dataScrapers.getFlag($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 8 / numberOfScrapers);
            data_getters_1.dataScrapers.getGeographicNotes($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 9 / numberOfScrapers);
            data_getters_1.dataScrapers.getGeography($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 10 / numberOfScrapers);
            data_getters_1.dataScrapers.getGeographyCoordinates($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 11 / numberOfScrapers);
            data_getters_1.dataScrapers.getIrrigatedLand($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 12 / numberOfScrapers);
            data_getters_1.dataScrapers.getLandUses($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 13 / numberOfScrapers);
            data_getters_1.dataScrapers.getMaritimeClaims($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 14 / numberOfScrapers);
            data_getters_1.dataScrapers.getNaturalHazard($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 15 / numberOfScrapers);
            data_getters_1.dataScrapers.getNaturalResources($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 16 / numberOfScrapers);
            data_getters_1.dataScrapers.getPopDist($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 17 / numberOfScrapers);
            data_getters_1.dataScrapers.getRegionMapImg($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 18 / numberOfScrapers);
            data_getters_1.dataScrapers.getSupplementalImages($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 19 / numberOfScrapers);
            data_getters_1.dataScrapers.getTerrains($, country.name, countryId);
            globalStore_1.store.progressLogger(country.name, 20 / numberOfScrapers);
            globalStore_1.store.debugLogger("Data scrape for " + country.name + " is complete");
        })
            .catch(function (err) {
            globalStore_1.store.failedCountries.push(country);
            var errMsg = new Date().toISOString() + "\n\nIndividual country query failed:  " + country.name + "\n" + url + "\n" + err.toString().trim() + "\n\n";
            globalStore_1.store.errorLogger(errMsg);
        });
    }
    else {
        return new Promise(function (resolve) {
            globalStore_1.store.errorLogger(new Date().toISOString() + "\n\nFailure to scrape data for " + country.name + " at \n" + url + "\n\n");
            resolve();
        }).then(function () { });
    }
}
exports.getCountryData = getCountryData;
;