added robots.txt to main domain
This commit is contained in:
parent
078f334a66
commit
944f45619c
57
homepage/robots.txt
Normal file
57
homepage/robots.txt
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
robots.txt custom made for whisper.cat
|
||||||
|
User-agent: Googlebot
|
||||||
|
Disallow:
|
||||||
|
User-agent: googlebot-image
|
||||||
|
Disallow:
|
||||||
|
User-agent: googlebot-mobile
|
||||||
|
Disallow:
|
||||||
|
User-agent: MSNBot
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Slurp
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Gigabot
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Robozilla
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Nutch
|
||||||
|
Disallow: /
|
||||||
|
User-agent: ia_archiver
|
||||||
|
Disallow: /
|
||||||
|
User-agent: baiduspider
|
||||||
|
Disallow: /
|
||||||
|
User-agent: naverbot
|
||||||
|
Disallow: /
|
||||||
|
User-agent: yeti
|
||||||
|
Disallow: /
|
||||||
|
User-agent: yahoo-mmcrawler
|
||||||
|
Disallow: /
|
||||||
|
User-agent: psbot
|
||||||
|
Disallow: /
|
||||||
|
User-agent: yahoo-blogs/v3.9
|
||||||
|
Disallow: /
|
||||||
|
User-agent: *
|
||||||
|
Disallow:
|
||||||
|
Crawl-delay: 10
|
||||||
|
Disallow: /not-found/
|
||||||
|
Disallow: /admin
|
||||||
|
Disallow: /login
|
||||||
|
Disallow: /ipfs
|
||||||
|
Disallow: /upload
|
||||||
|
Disallow: /cgi-bin
|
||||||
|
Disallow: /wp-admin
|
||||||
|
Disallow: /wp-includes
|
||||||
|
Disallow: /wp-content/plugins
|
||||||
|
Disallow: /wp-content/cache
|
||||||
|
Disallow: /wp-content/themes
|
||||||
|
Disallow: /trackback
|
||||||
|
Disallow: /feed
|
||||||
|
Disallow: /comments
|
||||||
|
Disallow: /category/*/*
|
||||||
|
Disallow: */trackback
|
||||||
|
Disallow: */feed
|
||||||
|
Disallow: */comments
|
||||||
|
Disallow: /*?*
|
||||||
|
Disallow: /*?
|
||||||
|
Disallow: /wp-includes
|
||||||
|
Disallow: /trackback
|
||||||
|
Disallow: /comments
|
Loading…
x
Reference in New Issue
Block a user