robots-txt-parser
Version:
A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.
23 lines (21 loc) • 506 B
JavaScript
module.exports = `
User-agent: *
Disallow: /cart/*
Disallow: /wardrobe/*
Disallow: /myaccount/*
Disallow: /opinions*
Disallow: /api/recos/reco
Disallow: /api/catalog/logs
Disallow: /api/pdp/sizereco
Disallow: /api/shop-the-look/looks
Disallow: /api/pdp/partner
Disallow: /api/cmag
Disallow: /api/navigation/wishlist-count
Disallow: /api/navigation/cart-count
Disallow: /api/catalog/logs
Disallow: /api/catalog/seo*
Disallow: /api/t/i
Disallow: /api/rr
User-agent: Screaming Frog SEO Spider
Disallow: /
`;