From 1c00b3138fa47a30f4cf8bf1f1a42a247c928c46 Mon Sep 17 00:00:00 2001 From: "[Harper Innes]" <[harper@whisper.cat]> Date: Mon, 16 Oct 2023 01:48:20 +1100 Subject: [PATCH] added semrush bot to the robots.txt file --- homepage/robots.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/homepage/robots.txt b/homepage/robots.txt index 8e8a381..a7f87f8 100644 --- a/homepage/robots.txt +++ b/homepage/robots.txt @@ -29,6 +29,8 @@ User-agent: psbot Disallow: / User-agent: yahoo-blogs/v3.9 Disallow: / +User-agent: SemrushBot +Disallow: / User-agent: * Disallow: Crawl-delay: 10