User-agent: * Disallow: /awstats/ Disallow: /search.php Disallow: /search_exp.php Disallow: /munin/ Disallow: /opensearch.xml #Disallow: /site/ #Disallow: /opensearch_exp.xml #Crawl-delay: 600 #Request-rate: 1/10 # maximum rate is one page every 10 seconds #Visit-time: 0200-0645