@benchristel/supercrawler/package.json

A web crawler. Supercrawler automatically crawls websites. Define custom handlers to parse content. Obeys robots.txt, rate limits and concurrency limits.

{
  "name": "@benchristel/supercrawler",
  "description": "A web crawler. Supercrawler automatically crawls websites. Define custom handlers to parse content. Obeys robots.txt, rate limits and concurrency limits.",
  "version": "0.16.0",
  "homepage": "https://github.com/brendonboshell/supercrawler",
  "author": "Brendon Boshell <brendonboshell@gmail.com>",
  "license": "Apache-2.0",
  "repository": {
    "type": "git",
    "url": "git+https://github.com/benchristel/supercrawler.git"
  },
  "bugs": {
    "url": "https://github.com/brendonboshell/supercrawler/issues"
  },
  "keywords": [
    "crawler",
    "spider",
    "supercrawler"
  ],
  "scripts": {
    "test": "mocha --recursive",
    "coverage": "istanbul cover _mocha -- -R spec --recursive"
  },
  "main": "./lib/index.js",
  "devDependencies": {
    "chai": "^3.5.0",
    "istanbul": "^0.4.4",
    "mocha": "^2.5.3",
    "proxyquire": "^1.7.10",
    "sinon": "^1.17.4"
  },
  "dependencies": {
    "bluebird": "^3.4.1",
    "cheerio": "^0.19.0",
    "ioredis": "^2.3.0",
    "mime-types": "^2.1.11",
    "node-cache": "^3.2.1",
    "request": "^2.73.0",
    "robots-parser": "brendonboshell/robots-parser#master",
    "sequelize": "^3.23.4",
    "sqlite3": "^3.1.4",
    "underscore": "^1.8.3"
  }
}