#User-agent: *
#Crawl-delay: 300

#User-agent: Twitterbot
#Disallow: