# Allow whitelisted bots
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Mobile
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-News
Disallow:

User-agent: Googlebot-Video
Disallow:

User-agent: AlexaCrawler
Disallow:

User-agent: Bingbot
Disallow:

User-agent: BingPreview
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Yahoo! Slurp
Disallow:

User-agent: SemrushBot
Disallow:

User-agent: Screaming Frog SEO Spider
Disallow:

User-agent: SearchAtlas
Disallow:

User-agent: Otto
Disallow:

User-agent: KoalaLinks
Disallow:

User-agent: facebot
Disallow:

User-agent: Facebook External Hit
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: ChatGPT
Disallow:

User-agent: GPTbot
Disallow:

User-agent: CHATPGPT-USER
Disallow:

User-agent: OPENAI
Disallow:

User-agent: Applebot
Disallow:

User-agent: Claudebot
Disallow:

User-agent: Pinterestbot
Disallow:

User-agent: Instagram
Disallow:

User-agent: Msnbot
Disallow:

# Block all others and apply crawl delay
User-agent: *
Disallow: /
Crawl-delay: 10

# Sitemap
Sitemap: https://comprehensive-urology.com/sitemap_index.xml