User-agent: ia_archiver
Disallow: /

# This pair is intended to exclude the internet archive from keeping old versions of pages.

User-agent: *
Disallow: /
Disallow: /grouphub/
Disallow: /cgi-bin/
Disallow: /index_files/
Disallow: /grouphub_res/

# These lines are intended to help exclude all web crawlers from secure or FTP part of site

# The line Disallow: / should be removed when we load the final website

# For more info on robot files see http://www.searchengineworld.com/robots/robots_tutorial.htm

#Or here: http://www.robotstxt.org/

# Check the file code here: http://tool.motoricerca.info/robots-checker.phtml