From fa3aecb7e779de3ee9094d4b8ec8ff0db67bad98 Mon Sep 17 00:00:00 2001 From: Nicolas Vigier Date: Wed, 14 Aug 2013 19:53:49 +0000 Subject: viewvc/robots.txt: ban AhrefsBot It doesn't respect Crawl-delay, or disallowed URLs. So try to disallow everything for them. --- modules/viewvc/files/robots.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'modules/viewvc') diff --git a/modules/viewvc/files/robots.txt b/modules/viewvc/files/robots.txt index b377dfc8..1c08ba9e 100644 --- a/modules/viewvc/files/robots.txt +++ b/modules/viewvc/files/robots.txt @@ -2,7 +2,6 @@ User-agent: Googlebot User-agent: Baiduspider User-agent: bingbot User-agent: YandexBot -User-agent: AhrefsBot User-agent: Mail.RU_Bot User-agent: MJ12bot Disallow: /*/tags/ @@ -17,5 +16,7 @@ Disallow: *?pathrev=* Disallow: *?*&view=log* Disallow: *view=log&* Disallow: *diff_format=* +User-agent: AhrefsBot +Disallow: / User-agent: * Crawl-delay: 30 -- cgit v1.2.1