robots-txt-parser/src/robots.js

A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.

const get = require('./get.js');
const parser = require('./parser.js');
const util = require('./util.js');

const DFLT_OPTS = {
  userAgent: '*',
  allowOnNeutral: true,
  timeout: 10000,
};

class Robots {
  constructor(opts = {}) {
    this.robotsCache = {};
    this.opts = {
      userAgent: opts.userAgent ? opts.userAgent.toLowerCase() : DFLT_OPTS.userAgent.toLowerCase(),
      allowOnNeutral: opts.allowOnNeutral ? opts.allowOnNeutral : DFLT_OPTS.allowOnNeutral,
      timeout: opts.timeout ? opts.timeout : DFLT_OPTS.timeout,
    };
  }

  getRecordsForAgent() {
    const key = this.active;
    const domainBots = this.robotsCache[key] || {};
    const ourBotInBots = this.opts.userAgent in domainBots;
    const otherBots = '*' in domainBots;
    if (ourBotInBots) {
      return domainBots[this.opts.userAgent];
    } else if (otherBots) {
      return domainBots['*'];
    }
    return false;
  }

  canVisit(url, botGroup) {
    const allow = util.applyRecords(url, botGroup.allow);
    const disallow = util.applyRecords(url, botGroup.disallow);
    const noAllows = allow.numApply === 0 && disallow.numApply > 0;
    const noDisallows = allow.numApply > 0 && disallow.numApply === 0;

    // No rules for allow or disallow apply, therefore full allow.
    if (noAllows && noDisallows) {
      return true;
    }

    if (noDisallows || (allow.maxSpecificity > disallow.maxSpecificity)) {
      return true;
    } else if (noAllows || (allow.maxSpecificity < disallow.maxSpecificity)) {
      return false;
    }
    return this.opts.allowOnNeutral;
  }

  parseRobots(url, string) {
    const formattedLink = util.formatLink(url);
    this.robotsCache[formattedLink] = parser(string);
    this.active = formattedLink;
  }

  fetch(link) {
    const formattedLink = util.formatLink(link);
    const robotsLink = `${formattedLink}/robots.txt`;
    return get(robotsLink, this.opts.timeout)
      .then((data) => {
        this.robotsCache[formattedLink] = parser(data);
        this.active = link;
        return this.robotsCache[formattedLink];
      });
  }

  isCached(domain) {
    return util.formatLink(domain) in this.robotsCache;
  }

  useRobotsFor(url, callback) {
    const link = util.formatLink(url);

    if (this.isCached(link)) {
      this.active = link;
      if (util.isFunction(callback)) {
        return callback();
      }
      return Promise.resolve();
    }

    const fetch = this.fetch(url);

    if (util.isFunction(callback)) {
      return fetch.then(callback);
    }

    return fetch;
  }

  canCrawl(url, callback) {
    if (this.isCached(url)) {
      const crawlable = this.canCrawlSync(url);
      if (util.isFunction(callback)) {
        return callback(crawlable);
      }
      return Promise.resolve(crawlable);
    }

    return this.fetch(url).then(() => {
      const crawlable = this.canCrawlSync(url);
      if (util.isFunction(callback)) {
        return callback(crawlable);
      }
      return crawlable;
    });
  }

  canCrawlSync(url) {
    // Get the parsed robotsCache.txt for this domain.
    const botGroup = this.getRecordsForAgent();
    // Conditional allow, our bot or the * user agent is in the robotsCache.txt
    if (botGroup) {
      return this.canVisit(url, botGroup);
    }
    // Robots txt exists doesn't have any rules for our bot or *, therefore full allow.
    return true;
  }

  getSitemaps(callback) {
    const sitemaps = this.getSitemapsSync();
    if (util.isFunction(callback)) {
      return callback(sitemaps);
    }
    return Promise.resolve(sitemaps);
  }

  getSitemapsSync() {
    const botRecords = this.robotsCache[this.active];
    return botRecords ? botRecords.sitemaps : [];
  }

  getCrawlDelay(callback) {
    const crawlDelay = this.getCrawlDelaySync();
    if (util.isFunction(callback)) {
      return callback(crawlDelay);
    }
    return Promise.resolve(crawlDelay);
  }

  getCrawlDelaySync() {
    const botRecords = this.getRecordsForAgent();
    return botRecords ? botRecords.crawlDelay : 0;
  }

  getCrawlableLinks(linkArray, callback) {
    const crawlableLinks = this.getCrawlableLinksSync(linkArray);
    if (util.isFunction(callback)) {
      return callback(crawlableLinks);
    }
    return Promise.resolve(crawlableLinks);
  }

  /*
   * Takes an array of links and returns an array of links which are crawlable
   * for the given domain.
   */
  getCrawlableLinksSync(linkArray) {
    const links = linkArray instanceof Array ? linkArray : [linkArray];
    const crawlableLinks = [];
    const botGroup = this.getRecordsForAgent();
    if (botGroup) {
      for (let i = 0; i < links.length; i += 1) {
        if (this.canVisit(links[i], botGroup)) {
          crawlableLinks.push(links[i]);
        }
      }
    }
    return crawlableLinks;
  }

  getPreferredHost(callback) {
    const host = this.getPreferredHostSync();
    if (util.isFunction(callback)) {
      callback(host);
    }
    return Promise.resolve(host);
  }

  getPreferredHostSync() {
    const botRecords = this.robotsCache[this.active] || {};
    return botRecords.host;
  }

  setUserAgent(agent) {
    this.opts.userAgent = agent.toLowerCase();
  }

  setAllowOnNeutral(allow) {
    this.opts.allowOnNeutral = allow;
  }

  clearCache() {
    this.robotsCache = {};
  }
}

module.exports = Robots;