Sitemap: https://mthree.com/sitemap.xml

# Yahoo!
User-agent: Slurp
Crawl-delay: 20
Allow: /

# Bing
User-agent: msnbot/2.0b
Crawl-Delay: 100
Allow: /

# Google
User-agent: Googlebot
Allow: /

# Sensis
User-agent: Sensis Web Crawler
Allow: /

# Sensis.com.au
User-agent: Sensis.com.au Web Crawler
Allow: /

# All other agents will not spider
User-agent: *
Allow: /

user-agent: sitebot
disallow: /

User-agent: Yandex
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: trovitBot
Disallow: /

User-agent: Jooblebot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Yeti
Allow: /

User-agent: rogerbot
Crawl-delay: 10