robots-txt-component/instances.json

Lightweight robots.txt parsing component without any external dependencies for Node.js.

{"*":{"rules":[{"directive":"disallow","specificity":7,"path":{}},{"directive":"allow","specificity":13,"path":{}},{"directive":"allow","specificity":14,"path":{}},{"directive":"allow","specificity":22,"path":{}},{"directive":"disallow","specificity":5,"path":{}},{"directive":"disallow","specificity":7,"path":{}},{"directive":"disallow","specificity":12,"path":{}},{"directive":"disallow","specificity":2,"path":{}},{"directive":"allow","specificity":5,"path":{}},{"directive":"disallow","specificity":7,"path":{}},{"directive":"allow","specificity":18,"path":{}},{"directive":"disallow","specificity":19,"path":{}},{"directive":"allow","specificity":13,"path":{}},{"directive":"allow","specificity":11,"path":{}},{"directive":"disallow","specificity":7,"path":{}},{"directive":"disallow","specificity":3,"path":{}},{"directive":"disallow","specificity":12,"path":{}},{"directive":"disallow","specificity":9,"path":{}},{"directive":"disallow","specificity":8,"path":{}},{"directive":"disallow","specificity":3,"path":{}},{"directive":"disallow","specificity":3,"path":{}},{"directive":"allow","specificity":10,"path":{}},{"directive":"disallow","specificity":5,"path":{}},{"directive":"disallow","specificity":6,"path":{}},{"directive":"disallow","specificity":12,"path":{}},{"directive":"disallow","specificity":7,"path":{}},{"directive":"disallow","specificity":8,"path":{}},{"directive":"disallow","specificity":14,"path":{}},{"directive":"disallow","specificity":1,"path":{}}],"crawlDelay":0}}