#
# Economist.com robots.txt
#
# Created MS 29 May 2001 Full disallow
# Amended MS 27 Jul 2001 Allow directories
# 21/07/2003 SR 90011    Allow Google but nothing else
# 16/10/2003 SR 12236   Give adsense premium access
# 19/07/2005 RWS 15237  Allow wayback
# 11/08/2005 RWS 15376  Allow Yahoo
# 21/08/2006 JBC 17953  Allow MSNBOT, MSNBOT-NEWS
# 18/04/2007 KA1 19695	Allow googlebot-pm
# 30/09/2008 LT	 22899  Add Semantic Proxy to the list of allowed user agents
#
User-agent: Mediapartners-Google* 
Disallow: 

User-agent: OpenCalaisSemanticProxy
Allow: /
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/

User-agent: googlebot
Allow: /
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/

User-agent: ia_archiver
Allow: /
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/

User-agent: Slurp
Allow: /
Crawl-delay: 60
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/

User-agent: MSNBOT
Allow: /
Crawl-delay: 60
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/

User-agent: MSNBOT-NEWS
Allow: /
Crawl-delay: 60
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/

User-agent: googlebot-pm
Allow: /
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/

User-agent: Teoma
Allow: /
Disallow: /search/
Disallow: /members/
Disallow: /subscriptions/
Disallow: /admin/