diff --git a/assets/robots.txt b/assets/robots.txt index 600b8f4..479ebcb 100644 --- a/assets/robots.txt +++ b/assets/robots.txt @@ -1,5 +1,56 @@ User-Agent: * Sitemap: https://ezrizhu.com/sitemap.xml Disallow: /health + +# Dark Visitors robots.txt + +# AI Assistant +# https://darkvisitors.com/agents/chatgpt-user + +User-agent: ChatGPT-User +Disallow: /blog/ + +# AI Assistant +# https://darkvisitors.com/agents/cohere-ai + +User-agent: cohere-ai +Disallow: /blog/ + +# AI Data Scraper +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai +Disallow: /blog/ + +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot +Disallow: /blog/ + +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot + +User-agent: FacebookBot +Disallow: /blog/ + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: /blog/ + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + User-agent: GPTBot Disallow: /blog/ + +# AI Data Scraper +# https://darkvisitors.com/agents/omgilibot + +User-agent: omgilibot +Disallow: /blog/ + +User-agent: omgili +Disallow: /blog/