1 files changed, 29 insertions, 0 deletions
diff --git a/modules/viewvc/files/robots.txt b/modules/viewvc/files/robots.txt
new file mode 100644
index 00000000..dbb13834
--- /dev/null
+++ b/modules/viewvc/files/robots.txt
@@ -0,0 +1,29 @@
+User-agent: Googlebot
+User-agent: Baiduspider
+User-agent: bingbot
+User-agent: YandexBot
+User-agent: Mail.RU_Bot
+User-agent: MJ12bot
+User-agent: ClaudeBot
+User-agent: Amazonbot
+User-agent: PetalBot
+User-agent: Bytespider
+User-agent: facebookexternalhit
+Disallow: /*/tags/
+Disallow: *?view=annotate*
+Disallow: *?annotate=*
+Disallow: *?view=diff*
+Disallow: *?r1=*
+Disallow: *sortby=*
+Disallow: *sortdir=*
+Disallow: *?revision=*&view=markup&*
+Disallow: *pathrev=*
+Disallow: *?*&view=log*
+Disallow: *view=log&*
+Disallow: *diff_format=*
+User-agent: AhrefsBot
+Disallow: /
+User-agent: Sogou web spider
+Disallow: /
+User-agent: *
+Crawl-delay: 30