From 222574e4822bdd63488c0acb8e51e379170458e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jani=20V=C3=A4limaa?= <wally@mageia.org>
Date: Mon, 17 Jun 2024 18:53:54 +0300
Subject: svnweb: add more bots to robots.txt

---
 modules/viewvc/files/robots.txt | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'modules/viewvc')

diff --git a/modules/viewvc/files/robots.txt b/modules/viewvc/files/robots.txt
index 3e0fc5c5..416168b6 100644
--- a/modules/viewvc/files/robots.txt
+++ b/modules/viewvc/files/robots.txt
@@ -6,6 +6,8 @@ User-agent: Mail.RU_Bot
 User-agent: MJ12bot
 User-agent: ClaudeBot
 User-agent: Amazonbot
+User-agent: PetalBot
+User-agent: Bytespider
 Disallow: /*/tags/
 Disallow: *?view=annotate*
 Disallow: *?annotate=*
@@ -20,5 +22,7 @@ Disallow: *view=log&*
 Disallow: *diff_format=*
 User-agent: AhrefsBot
 Disallow: /
+User-agent: Sogou web spider
+Disallow: /
 User-agent: *
 Crawl-delay: 30
-- 
cgit v1.2.1