From 7a74c16779f745eed3e2c464b1eb360abeeefe51 Mon Sep 17 00:00:00 2001
From: sam <sam@92316355-f0b4-4df1-b90c-862c8a59935f>
Date: Wed, 14 Aug 2013 10:12:39 +0000
Subject: [PATCH] web: ask crawler bots to hit us less aggressively.

git-svn-id: file:///srv/caca.zoy.org/var/lib/svn/web/trunk@4868 92316355-f0b4-4df1-b90c-862c8a59935f
---
 www/robots.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/www/robots.txt b/www/robots.txt
index 26f4f54..5acf224 100644
--- a/www/robots.txt
+++ b/www/robots.txt
@@ -5,6 +5,10 @@ User-agent: *
 Disallow: CVS
 Disallow: .svn
 
+# Be gentle
+User-agent: *
+Crawl-delay: 5
+
 # Prevent excessive search engine hits
 Disallow: /cgi-bin/trac.cgi
 Disallow: /log