# Fichero robots.txt by Tito Pelon
#
# Se basa en una lista estudiada durante tres años.
#
# Las carpetas están accesibles por todos los robots excepto por estos, que he comprobado que son dañinos o
# por lo menos, muy pesados.

User-agent: MJ12bot
Disallow: /

User-agent: CFNetwork
Disallow: /

User-agent: Java/1.4.1_04
Disallow: /

User-agent: Java/1.5.0_06
Disallow: /

User-agent: Java/1.6.0_03
Disallow: /

User-agent: Java/1.5.0_04
Disallow: /

User-agent: Java/1.6.0_02
Disallow: /

User-agent: Java/1.6.0_07
Disallow: /

User-agent: Netcraft
Disallow: / 

User-agent: Baiduspider
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: robot
Disallow: /

User-agent: spider
Disallow: /

User-agent: bot/
Disallow: /

User-agent: bot-
Disallow: /

User-agent: HTTrack off-line browser
Disallow: /

User-agent: MSRBot
Disallow: /

User-agent: StackRambler
Disallow: /

User-agent: Wells Search
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: spider
Disallow: /

User-agent: NG 2.0
Disallow: /

User-agent: crawl
Disallow: /

User-agent: DotBot/1.1
Disallow: /

User-agent: Twiceler-0.9
Disallow: /

User-agent: panscient.com
Disallow: /

User-agent: YoudaoBot/1.0
Disallow: /

User-agent: Yanga WorldSearch Bot v1.1/beta
Disallow: /

User-agent: MLBot
Disallow: /

User-agent: vortex
Disallow: /

User-agent: Technoratibot
Disallow: /

User-agent: MSNBot
Disallow: /

User-agent: MSNBot-media
Disallow: /

# Denegamos acceso a todos los robots a las carpetas bofh, config y recursos
User-agent: *
Disallow: /bofh/
Disallow: /config/
Disallow: /recursos/