# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:

User-agent: Bingbot
Crawl-delay: 10

User-agent: ia_archiver
Disallow: /
Crawl-delay: 10

User-agent: msnbot
Crawl-delay: 10

User-agent: Naverbot
Crawl-delay: 10

User-agent: seznambot
Crawl-delay: 10

User-agent: Yeti
Crawl-delay: 10


User-agent: Gigabot
Crawl-delay: 10

User-agent: Scrubby
Crawl-delay: 10

User-agent: Robozilla
Crawl-delay: 10

User-agent: Nutch
Crawl-delay: 10

User-agent: YahooSeeker/M1A1-R2D2
Crawl-delay: 10

User-agent: MSNBOT_Mobile
Crawl-delay: 10

User-agent: yahoo-mmcrawler
Crawl-delay: 10

User-agent: psbot
Crawl-delay: 10

User-agent: asterias
Crawl-delay: 10

User-agent: yahoo-blogs/v3.9
Crawl-delay: 10

User-agent: IsraBot
Crawl-delay: 10

User-agent: *
Disallow: /de
Disallow: /en
Disallow: /fr
Disallow: /pt
Disallow: /qu
Disallow: /new_social_contact
Disallow: /*/comments
Disallow: /*?page=
Disallow: /*&page=

User-agent: ArchitextSpider
User-agent: Baiduspider
User-agent: Scooter
User-agent: Slurp
User-agent: Teoma
User-agent: Yandex
Disallow: /

# SemrushBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
Disallow: /

# allow only important bots
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Googlebot-Image
User-agent: Mediapartners-Google
User-agent: Yahoo-Blogs
User-agent: Yahoo-MMCrawler
User-agent: msnbot
User-agent: msnbot-media
User-agent: SemrushBot-SA
Allow: /

sitemap: http://proexpansion.com/sitemap.xml