woothee/woothee/testsets/crawler.yaml

User-Agent string parser (js implementation)

# Yahoo
- target: 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)'
  name: Yahoo! Slurp
  os: UNKNOWN
  category: crawler

- target: 'Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)'
  name: Yahoo! Slurp
  os: UNKNOWN
  category: crawler

- target: 'Y!J-BRO/YFSJ crawler (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html; YahooFeedSeekerJp/2.0; users 0; views 248)'
  name: Yahoo! Japan
  os: UNKNOWN
  category: crawler

- target: 'Y!J-BRP/YFSBJ crawler (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html; YahooFeedSeekerBetaJp/2.0; users 0; views 80)'
  name: Yahoo! Japan
  os: UNKNOWN
  category: crawler

- target: 'Y!J-BRJ/YATS crawler (http://listing.yahoo.co.jp/support/faq/int/other/other_001.html)'
  name: Yahoo! Japan
  os: UNKNOWN
  category: crawler

- target: 'Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)'
  name: Yahoo! Japan
  os: UNKNOWN
  category: crawler

- target: 'Y!J-BSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/blog-search/)'
  name: Yahoo! Japan
  os: UNKNOWN
  category: crawler

- target: 'Mozilla/5.0 (Linux; U; Android 2.3.3; ja-jp; sdk Build/GRI34; Y!J-BRZ/YATSHA crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1'
  name: Yahoo! Japan
  os: UNKNOWN
  category: crawler

- target: 'Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)'
  name: Yahoo! Japan
  os: UNKNOWN
  category: crawler

- target: 'Yahoo Pipes 2.0'
  name: Yahoo! Pipes
  os: UNKNOWN
  category: crawler

# Baidu
- target: 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)'
  name: Baiduspider
  os: UNKNOWN
  category: crawler

- target: 'Baiduspider+(+http://www.baidu.jp/spider/)'
  name: Baiduspider
  os: UNKNOWN
  category: crawler

- target: 'Baiduspider-image+(+http://www.baidu.com/search/spider.htm)'
  name: Baiduspider
  os: UNKNOWN
  category: crawler

# MSN/Bing
- target: 'msnbot/1.1 (+http://search.msn.com/msnbot.htm)'
  name: msnbot
  os: UNKNOWN
  category: crawler

- target: 'msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)'
  name: msnbot
  os: UNKNOWN
  category: crawler

- target: 'msnbot/2.0b (+http://search.msn.com/msnbot.htm)._'
  name: msnbot
  os: UNKNOWN
  category: crawler

- target: '"msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)'
  name: msnbot
  os: UNKNOWN
  category: crawler

- target: 'msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)'
  name: msnbot
  os: UNKNOWN
  category: crawler

- target: 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)'
  name: bingbot
  os: UNKNOWN
  category: crawler

- target: 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b'
  name: BingPreview
  os: UNKNOWN
  category: crawler

# Naver
- target: 'Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)'
  name: Naver Yeti
  os: UNKNOWN
  category: crawler

- target: 'Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/support/robots.html)'
  name: Naver Yeti
  os: UNKNOWN
  category: crawler

- target: 'Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)'
  name: Naver Yeti
  os: UNKNOWN
  category: crawler

# Unknown Chinese bot
- target: 'Mozilla/3.0 (compatible; Indy Library)'
  name: Indy Library
  os: UNKNOWN
  category: crawler

# Apple
- target: 'Apple-PubSub/65.28'
  name: Apple iCloud
  os: UNKNOWN
  category: crawler

# salesforce marketing radian6
- target: 'R6_CommentReader(www.radian6.com/crawler)'
  name: salesforce radian6
  os: UNKNOWN
  category: crawler

- target: 'R6_FeedFetcher(www.radian6.com/crawler)'
  name: salesforce radian6
  os: UNKNOWN
  category: crawler

# Genieo
- target: 'Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)'
  name: Genieo Web Filter
  os: UNKNOWN
  category: crawler

# topsy
- target: 'Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8'
  name: topsy Butterfly
  os: UNKNOWN
  category: crawler

# SeoMoz rogerbot
- target: 'rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)'
  name: SeoMoz rogerbot
  os: UNKNOWN
  category: crawler

- target: 'rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)'
  name: SeoMoz rogerbot
  os: UNKNOWN
  category: crawler

# ahref
- target: 'Mozilla/5.0 (compatible; AhrefsBot/4.0; +http://ahrefs.com/robot/)'
  name: ahref AhrefsBot
  os: UNKNOWN
  category: crawler

# Hatena
- target: 'Hatena Antenna/0.5 (http://a.hatena.ne.jp/help)'
  name: Hatena
  os: UNKNOWN
  category: crawler

- target: 'Hatena Pagetitle Agent/1.0'
  name: Hatena
  os: UNKNOWN
  category: crawler

- target: 'Hatena Diary RSS Module (http://d.hatena.ne.jp/help#rssmodule)'
  name: Hatena
  os: UNKNOWN
  category: crawler

# Goo
- target: 'ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)'
  name: goo
  os: UNKNOWN
  category: crawler

- target: 'DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)'
  name: goo
  os: UNKNOWN
  category: crawler

- target: 'gooblogsearch/2.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)'
  name: goo
  os: UNKNOWN
  category: crawler

# livedoor
- target: 'livedoor FeedFetcher/0.01 (http://reader.livedoor.com/; 999 subscribers)'
  name: livedoor FeedFetcher
  os: UNKNOWN
  category: crawler

- target: 'Fastladder FeedFetcher/0.01 (http://fastladder.com/; 27 subscribers)'
  name: livedoor FeedFetcher
  os: UNKNOWN
  category: crawler

# twitter
- target: 'Twitterbot/1.0'
  name: twitter
  os: UNKNOWN
  category: crawler

# facebook
- target: 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
  name: facebook
  os: UNKNOWN
  category: crawler

# mixi
- target: 'mixi-check/1.0 (http://mixi.jp/)'
  name: mixi
  os: UNKNOWN
  category: crawler

- target: 'mixi-news-crawler/1.00 (http://mixi.jp/)'
  name: mixi
  os: UNKNOWN
  category: crawler

- target: 'mixi-crawler/2.00 (http://mixi.jp/)'
  name: mixi
  os: UNKNOWN
  category: crawler

# trendictionbot
- target: 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11'
  name: trendiction
  os: UNKNOWN
  category: crawler