2023-10-16 01:41:15 +11:00
|
|
|
robots.txt custom made for whisper.cat
|
|
|
|
|
User-agent: Googlebot
|
|
|
|
|
Disallow:
|
|
|
|
|
User-agent: googlebot-image
|
|
|
|
|
Disallow:
|
|
|
|
|
User-agent: googlebot-mobile
|
|
|
|
|
Disallow:
|
|
|
|
|
User-agent: MSNBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: Slurp
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: Gigabot
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: Robozilla
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: Nutch
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: ia_archiver
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: baiduspider
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: naverbot
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: yeti
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: yahoo-mmcrawler
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: psbot
|
|
|
|
|
Disallow: /
|
|
|
|
|
User-agent: yahoo-blogs/v3.9
|
|
|
|
|
Disallow: /
|
2023-10-16 01:48:20 +11:00
|
|
|
User-agent: SemrushBot
|
|
|
|
|
Disallow: /
|
2023-10-16 01:41:15 +11:00
|
|
|
User-agent: *
|
|
|
|
|
Disallow:
|
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
Disallow: /not-found/
|
|
|
|
|
Disallow: /admin
|
|
|
|
|
Disallow: /login
|
|
|
|
|
Disallow: /ipfs
|
|
|
|
|
Disallow: /upload
|
|
|
|
|
Disallow: /cgi-bin
|
|
|
|
|
Disallow: /wp-admin
|
|
|
|
|
Disallow: /wp-includes
|
|
|
|
|
Disallow: /wp-content/plugins
|
|
|
|
|
Disallow: /wp-content/cache
|
|
|
|
|
Disallow: /wp-content/themes
|
|
|
|
|
Disallow: /trackback
|
|
|
|
|
Disallow: /feed
|
|
|
|
|
Disallow: /comments
|
|
|
|
|
Disallow: /category/*/*
|
|
|
|
|
Disallow: */trackback
|
|
|
|
|
Disallow: */feed
|
|
|
|
|
Disallow: */comments
|
|
|
|
|
Disallow: /*?*
|
|
|
|
|
Disallow: /*?
|
|
|
|
|
Disallow: /wp-includes
|
|
|
|
|
Disallow: /trackback
|
|
|
|
|
Disallow: /comments
|
