diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/viewvc/files/robots.txt | 4 |
1 files changed, 4 insertions, 0 deletions
diff --git a/modules/viewvc/files/robots.txt b/modules/viewvc/files/robots.txt index 3e0fc5c5..416168b6 100644 --- a/modules/viewvc/files/robots.txt +++ b/modules/viewvc/files/robots.txt @@ -6,6 +6,8 @@ User-agent: Mail.RU_Bot User-agent: MJ12bot User-agent: ClaudeBot User-agent: Amazonbot +User-agent: PetalBot +User-agent: Bytespider Disallow: /*/tags/ Disallow: *?view=annotate* Disallow: *?annotate=* @@ -20,5 +22,7 @@ Disallow: *view=log&* Disallow: *diff_format=* User-agent: AhrefsBot Disallow: / +User-agent: Sogou web spider +Disallow: / User-agent: * Crawl-delay: 30 |