robots-txt-parser/src/get.js

A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.

const get = require('simple-get');

const getRobots = (url, timeout) => new Promise((resolve, reject) => {
  get.concat({ method: 'GET', url, timeout }, (error, response, body) => {
    if (!error) {
      resolve(body.toString());
    } else {
      reject(error);
    }
  });
});

module.exports = getRobots;