robots-txt-parser/src/get.js

A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.

const get = require('simple-get');

function getRobots(url, timeout) {
  return new Promise((resolve, reject) => {
    get.concat({ method: 'GET', url, timeout }, (error, response, body) => {
      if (!error) {
        resolve(body.toString());
      } else {
        reject(error);
      }
    });
  });
}

module.exports = getRobots;