diff --git a/homepage/robots.txt b/homepage/robots.txt index 8e8a381..a7f87f8 100644 --- a/homepage/robots.txt +++ b/homepage/robots.txt @@ -29,6 +29,8 @@ User-agent: psbot Disallow: / User-agent: yahoo-blogs/v3.9 Disallow: / +User-agent: SemrushBot +Disallow: / User-agent: * Disallow: Crawl-delay: 10