aboutsummaryrefslogtreecommitdiffstats
path: root/robots.txt
diff options
context:
space:
mode:
authorRomain d'Alverny <rda@mageia.org>2012-08-03 10:29:28 +0000
committerRomain d'Alverny <rda@mageia.org>2012-08-03 10:29:28 +0000
commit8be0fec9cb8dd4c231bdff73018b736df8bd2db6 (patch)
tree429f58aee9bdf1f3bd6174af19fa52faee098d1b /robots.txt
parent8423d019a82e8eb63c6b969e80a39207a647f4d5 (diff)
downloadwww-8be0fec9cb8dd4c231bdff73018b736df8bd2db6.tar
www-8be0fec9cb8dd4c231bdff73018b736df8bd2db6.tar.gz
www-8be0fec9cb8dd4c231bdff73018b736df8bd2db6.tar.bz2
www-8be0fec9cb8dd4c231bdff73018b736df8bd2db6.tar.xz
www-8be0fec9cb8dd4c231bdff73018b736df8bd2db6.zip
add a bot
Diffstat (limited to 'robots.txt')
-rw-r--r--robots.txt9
1 files changed, 3 insertions, 6 deletions
diff --git a/robots.txt b/robots.txt
index dbc4707f9..2a72f5bcf 100644
--- a/robots.txt
+++ b/robots.txt
@@ -7,23 +7,20 @@
User-Agent: *
Disallow:
-
User-Agent: bingbot
-Disallow: /*/downloads/dl.php*
Disallow: /*/downloads/get/*
User-Agent: Googlebot
-Disallow: /*/downloads/dl.php*
Disallow: /*/downloads/get/*
User-Agent: msnbot
-Disallow: /*/downloads/dl.php*
Disallow: /*/downloads/get/*
User-Agent: Ezooms
-Disallow: /*/downloads/dl.php*
Disallow: /*/downloads/get/*
User-Agent: MJ12bot
-Disallow: /*/downloads/dl.php*
+Disallow: /*/downloads/get/*
+
+User-Agent: GSLFbot
Disallow: /*/downloads/get/*