From 8be0fec9cb8dd4c231bdff73018b736df8bd2db6 Mon Sep 17 00:00:00 2001 From: Romain d'Alverny Date: Fri, 3 Aug 2012 10:29:28 +0000 Subject: add a bot --- robots.txt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'robots.txt') diff --git a/robots.txt b/robots.txt index dbc4707f9..2a72f5bcf 100644 --- a/robots.txt +++ b/robots.txt @@ -7,23 +7,20 @@ User-Agent: * Disallow: - User-Agent: bingbot -Disallow: /*/downloads/dl.php* Disallow: /*/downloads/get/* User-Agent: Googlebot -Disallow: /*/downloads/dl.php* Disallow: /*/downloads/get/* User-Agent: msnbot -Disallow: /*/downloads/dl.php* Disallow: /*/downloads/get/* User-Agent: Ezooms -Disallow: /*/downloads/dl.php* Disallow: /*/downloads/get/* User-Agent: MJ12bot -Disallow: /*/downloads/dl.php* +Disallow: /*/downloads/get/* + +User-Agent: GSLFbot Disallow: /*/downloads/get/* -- cgit v1.2.1