User-agent: Googlebot-Mobile
Disallow: /

User-agent: AdsBot-Google
Disallow: /

User-agent: Yahoo Pipes 2.0
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: asterias
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-Agent: Oracle Secure Enterprise Search
Disallow: /

# User-agent: Baiduspider
# Disallow: /

# User-agent: Yandex
# Disallow: /

User-Agent: Mail.RU_Bot
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: NerdyBot
Disallow: /

User-agent: 008
Disallow: /	

User-agent: ccbot
Disallow: /	

User-agent: discobot
Disallow: /	

User-agent: OmegaSeek
Disallow: /	

User-agent: discoverybot
Disallow: / 

User-agent: MJ12bot
Disallow: /

User-agent: wotbox
Disallow: /

User-agent: yacy
Disallow: /

User-agent: Twitterbot
Disallow:

User-agent: Blekkobot
Disallow: /

User-agent: Abonti
Disallow: /

user-agent: CrystalSemanticsBot
Disallow: / 

User-agent: *
Disallow: /etext
Disallow: /dirs/etext
Disallow: /dirs/1
Disallow: /dirs/2
Disallow: /dirs/3
Disallow: /dirs/4
Disallow: /dirs/5
Disallow: /dirs/6
Disallow: /dirs/7
Disallow: /dirs/8
Disallow: /dirs/9
Disallow: /catalog/world/	# old search engine
Disallow: /ebooks/search        # new search engine
Disallow: /ebooks/send/         # dropbox et al.
Disallow: /ratelimiter
Disallow: /zipcat
Disallow: /addresses/		# don't remove, ask webmaster
Disallow: /author/
Disallow: /internal/		# private
Disallow: /etext/magnet         # against brain damaged robots
Disallow: /cache/               # on-the-fly generated files
Disallow: /cdproject/
Disallow: /w/			# wiki edit pages etc.
Disallow: /wiki/Special:Search
Disallow: /wiki/Special:Random

Crawl-delay: 5

Sitemap: http://www.gutenberg.org/ebooks/sitemaps/