UNPKG

robots-txt-parser

Version:

A lightweight robots.txt parser for Node.js with support for wildcards, caching and promises.

github.com/ChrisAkroyd/robots-txt-parser

ChrisAkroyd/robots-txt-parser

66 lines (61 loc) • 1.9 kB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
module.exports = ` # version: d167f71fcf4277403ce3b7118a1fad5d25a41310 # HTTPS www.bbc.co.uk User-agent: * Sitemap: https://www.bbc.co.uk/sitemap.xml Sitemap: https://www.bbc.co.uk/sitemaps/https-index-uk-archive.xml Sitemap: https://www.bbc.co.uk/sitemaps/https-index-uk-news.xml Sitemap: https://www.bbc.co.uk/food/sitemap.xml Sitemap: https://www.bbc.co.uk/bitesize/sitemap/sitemapindex.xml Sitemap: https://www.bbc.co.uk/teach/sitemap/sitemapindex.xml Sitemap: https://www.bbc.co.uk/sitemaps/https-index-uk-archive_video.xml Sitemap: https://www.bbc.co.uk/sitemaps/https-index-uk-video.xml Sitemap: https://www.bbc.co.uk/sitemaps/sitemap-uk-ws-topics.xml Sitemap: https://www.bbc.co.uk/sport/sitemap.xml Disallow: /cbbc/search$ Disallow: /cbbc/search/ Disallow: /cbbc/search? Disallow: /cbeebies/search$ Disallow: /cbeebies/search/ Disallow: /cbeebies/search? Disallow: /chwilio/ Disallow: /chwilio$ Disallow: /chwilio? Disallow: /iplayer/bigscreen/ Disallow: /iplayer/cbbc/episodes/ Disallow: /iplayer/cbbc/search Disallow: /iplayer/cbeebies/episodes/ Disallow: /iplayer/cbeebies/search Disallow: /iplayer/search Disallow: /indepthtoolkit/smallprox$ Disallow: /indepthtoolkit/smallprox/ Disallow: /modules/musicnav/language/ Disallow: /news/0 Disallow: /radio/aod/ Disallow: /radio/aod$ Disallow: /radio/imda Disallow: /radio/player/ Disallow: /radio/player$ Disallow: /search/ Disallow: /search$ Disallow: /search? Disallow: /sport/videos/* Disallow: /sounds/player/ Disallow: /sounds/player$ Disallow: /ugc$ Disallow: /ugc/ Disallow: /ugcsupport$ Disallow: /ugcsupport/ Disallow: /userinfo/ Disallow: /userinfo Disallow: /food/favourites Disallow: /food/menus/*/shopping-list Disallow: /food/recipes/*/shopping-list Disallow: /food/search*?* Disallow: /sounds/search$ Disallow: /sounds/search/ Disallow: /sounds/search? Disallow: /ws/includes User-agent: magpie-crawler Disallow: / `;