diff --git a/www/robots.txt b/www/robots.txt index 26f4f54..5acf224 100644 --- a/www/robots.txt +++ b/www/robots.txt @@ -5,6 +5,10 @@ User-agent: * Disallow: CVS Disallow: .svn +# Be gentle +User-agent: * +Crawl-delay: 5 + # Prevent excessive search engine hits Disallow: /cgi-bin/trac.cgi Disallow: /log