|
|
@@ -9,6 +9,10 @@ Disallow: .svn |
|
|
|
Disallow: /cgi-bin/trac.cgi |
|
|
|
Disallow: /log |
|
|
|
|
|
|
|
# Don’t crawl git repos |
|
|
|
Disallow: /git/*.git/* |
|
|
|
Disallow: /git/*.git.broken/* |
|
|
|
|
|
|
|
# "This robot collects content from the Internet for the sole purpose of |
|
|
|
# helping educational institutions prevent plagiarism. [...] we compare |
|
|
|
# student papers against the content we find on the Internet to see if we |
|
|
|