diff options
| -rw-r--r-- | modules/viewvc/files/robots.txt | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/modules/viewvc/files/robots.txt b/modules/viewvc/files/robots.txt index b377dfc8..1c08ba9e 100644 --- a/modules/viewvc/files/robots.txt +++ b/modules/viewvc/files/robots.txt @@ -2,7 +2,6 @@ User-agent: Googlebot User-agent: Baiduspider User-agent: bingbot User-agent: YandexBot -User-agent: AhrefsBot User-agent: Mail.RU_Bot User-agent: MJ12bot Disallow: /*/tags/ @@ -17,5 +16,7 @@ Disallow: *?pathrev=* Disallow: *?*&view=log* Disallow: *view=log&* Disallow: *diff_format=* +User-agent: AhrefsBot +Disallow: / User-agent: * Crawl-delay: 30 |