@democracy-deutschland/bt-named-polls/distribution/scraper.js

scraper for the german "Bundestag" (Namentliche Abstimmungen)

'use strict';

var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };

var _Browser = require('./Browser');

var _Browser2 = _interopRequireDefault(_Browser);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; } /* eslint-disable max-len */
/* eslint-disable no-throw-literal */

process.setMaxListeners(Infinity);

class Scraper {
  constructor() {
    var _this = this;

    this.options = {
      startId: 1,
      onData: false,
      onFinish: false
    };
    this.urls = {
      get: ({ id }) => `https://www.bundestag.de/parlament/plenum/abstimmung/abstimmung?id=${id}`,
      getPolls: ({ offset }) => `https://www.bundestag.de/ajax/filterlist/de/parlament/plenum/abstimmung/484422-484422/?noFilterSet=true&offset=${offset}`,
      start: null
    };
    this.browser = null;
    this.scrapedWeeks = [];
    this.dataPromises = [];
    this.lastTitle = '';
    this.pollIds = [];
    this.getAvailablePolls = _asyncToGenerator(function* () {
      const {
        browser: { browser }
      } = _this;
      let offset = 0;
      let results = true;
      while (results) {
        const { body } = yield browser.request({
          uri: _this.urls.getPolls({ offset })
        });
        const polls = body.match(/<a href="\/parlament\/plenum\/abstimmung\/abstimmung\?id=(\d*)"/g);
        if (polls) {
          _this.pollIds = [..._this.pollIds, ...polls.map(function (poll) {
            return parseInt(poll.match(/(\d+)/)[1], 10);
          })];

          offset += polls.length;
        } else {
          results = false;
        }
      }
      _this.pollIds = [...new Set(_this.pollIds)].filter(function (pollId) {
        return pollId >= parseInt(_this.options.startId, 10);
      });
    });

    this.getClosestPollId = id => {
      if (Math.max(this.pollIds < id)) {
        return false;
      }
      const closest = this.pollIds.reduce((prev, pollId) => {
        const diff = pollId - id;
        if (diff >= 0 && (prev === false || pollId <= prev)) {
          return pollId;
        }
        return prev;
      }, false);

      const index = this.pollIds.indexOf(closest);
      if (index > -1) {
        this.pollIds.splice(index, 1);
      }
      return closest;
    };

    this.getPolls = (() => {
      var _ref2 = _asyncToGenerator(function* ({ id }) {
        let curId = _this.getClosestPollId(id);
        if (!curId) {
          throw new Error('Poll id is to high!');
        }
        const {
          browser: { browser }
        } = _this;
        let { body } = yield browser.request({
          uri: _this.urls.get({ id: curId })
        });
        let { title, date, documents } = browser.getHead(body);
        let errorCount = 0;

        while (curId && errorCount < 5) {
          _this.lastTitle = title;
          ({ body } = yield browser.request({
            uri: _this.urls.get({ id: curId })
          }));
          try {
            ({ title, date, documents } = browser.getHead(body));
            const voteResults = browser.getPartyVotes(body);

            const resultData = {
              id: curId,
              title,
              date,
              documents,
              voteResults
            };
            if (_this.options.onData) {
              _this.dataPromises = [..._this.dataPromises, _this.options.onData(resultData)];
            }
            errorCount = 0;
          } catch (error) {
            errorCount += 1;
          }

          curId = _this.getClosestPollId(curId);
        }
      });

      return function (_x) {
        return _ref2.apply(this, arguments);
      };
    })();

    this.createNewBrowser = (() => {
      var _ref3 = _asyncToGenerator(function* ({ browserObject } = {}) {
        if (browserObject) {
          delete browserObject.browser; // eslint-disable-line
        }
        const browser = new _Browser2.default();
        yield browser.initialize(_this.urls.start);
        return {
          browser,
          used: false,
          scraped: 0,
          errors: 0
        };
      });

      return function () {
        return _ref3.apply(this, arguments);
      };
    })();
  }

  scrape(options) {
    var _this2 = this;

    return _asyncToGenerator(function* () {
      _this2.options = _extends({}, _this2.options, options);

      _this2.urls.start = _this2.urls.get({ id: _this2.options.startId });

      _this2.browser = yield _this2.createNewBrowser();

      yield _this2.getAvailablePolls();

      yield _this2.getPolls({ id: parseInt(_this2.options.startId, 10) });
      if (_this2.options.onFinish) {
        yield Promise.all(_this2.dataPromises);
        yield _this2.options.onFinish({ length: _this2.dataPromises.length });
      }
    })();
  }

}

module.exports = Scraper;