From 944f45619c4c00392defe71a0757760bbdb01465 Mon Sep 17 00:00:00 2001
From: "[Harper Innes]" <[harper@whisper.cat]>
Date: Mon, 16 Oct 2023 01:41:15 +1100
Subject: [PATCH] added robots.txt to main domain

---
 homepage/robots.txt | 57 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 homepage/robots.txt

diff --git a/homepage/robots.txt b/homepage/robots.txt
new file mode 100644
index 0000000..8e8a381
--- /dev/null
+++ b/homepage/robots.txt
@@ -0,0 +1,57 @@
+robots.txt custom made for whisper.cat
+User-agent: Googlebot
+Disallow: 
+User-agent: googlebot-image
+Disallow: 
+User-agent: googlebot-mobile
+Disallow: 
+User-agent: MSNBot
+Disallow: /
+User-agent: Slurp
+Disallow: /
+User-agent: Gigabot
+Disallow: /
+User-agent: Robozilla
+Disallow: /
+User-agent: Nutch
+Disallow: /
+User-agent: ia_archiver
+Disallow: /
+User-agent: baiduspider
+Disallow: /
+User-agent: naverbot
+Disallow: /
+User-agent: yeti
+Disallow: /
+User-agent: yahoo-mmcrawler
+Disallow: /
+User-agent: psbot
+Disallow: /
+User-agent: yahoo-blogs/v3.9
+Disallow: /
+User-agent: *
+Disallow: 
+Crawl-delay: 10
+Disallow: /not-found/
+Disallow: /admin
+Disallow: /login
+Disallow: /ipfs
+Disallow: /upload
+Disallow: /cgi-bin
+Disallow: /wp-admin
+Disallow: /wp-includes
+Disallow: /wp-content/plugins
+Disallow: /wp-content/cache
+Disallow: /wp-content/themes
+Disallow: /trackback
+Disallow: /feed
+Disallow: /comments
+Disallow: /category/*/*
+Disallow: */trackback
+Disallow: */feed
+Disallow: */comments
+Disallow: /*?*
+Disallow: /*?
+Disallow: /wp-includes
+Disallow: /trackback
+Disallow: /comments