diff --git a/server/src/client/app/public/robots.txt b/server/src/client/app/public/robots.txt index df371334..8671b559 100644 --- a/server/src/client/app/public/robots.txt +++ b/server/src/client/app/public/robots.txt @@ -1,35 +1,3 @@ -User-agent: Googlebot -Disallow: -User-agent: googlebot-image -Disallow: -User-agent: googlebot-mobile -Disallow: -User-agent: MSNBot -Disallow: -User-agent: Slurp -Disallow: -User-agent: Teoma -Disallow: -User-agent: Gigabot -Disallow: -User-agent: Robozilla -Disallow: -User-agent: Nutch -Disallow: -User-agent: ia_archiver -Disallow: -User-agent: baiduspider -Disallow: -User-agent: naverbot -Disallow: -User-agent: yeti -Disallow: -User-agent: yahoo-mmcrawler -Disallow: -User-agent: psbot -Disallow: -User-agent: yahoo-blogs/v3.9 -Disallow: User-agent: * -Disallow: +Disallow: /data/ Disallow: /cgi-bin/