# https://www.robotstxt.org/robotstxt.html
# Quorlin - the smart contract language of the Kortana blockchain.
# Every crawler is welcome: the entire site is public documentation.

User-agent: *
Allow: /
Disallow: /_build/
Disallow: /_server/
Disallow: /api/

# Major global search engines
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Storebot-Google
User-agent: Google-InspectionTool
User-agent: AdsBot-Google
User-agent: Bingbot
User-agent: msnbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Applebot
Allow: /
Disallow: /_build/
Disallow: /_server/
Disallow: /api/

# Regional search engines
User-agent: Baiduspider
User-agent: YandexBot
User-agent: YandexImages
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: 360Spider
User-agent: Yeti
User-agent: SeznamBot
User-agent: Qwantify
User-agent: Exabot
User-agent: Naverbot
User-agent: Daumoa
User-agent: coccocbot-web
Allow: /
Disallow: /_build/
Disallow: /_server/
Disallow: /api/

# Social / messaging link unfurlers
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: redditbot
User-agent: Pinterestbot
User-agent: SkypeUriPreview
User-agent: vkShare
Allow: /
Disallow: /_build/
Disallow: /_server/
Disallow: /api/

# AI answer engines and assistants
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
User-agent: YouBot
User-agent: DeepSeekBot
User-agent: MistralAI-User
Allow: /
Disallow: /_build/
Disallow: /_server/
Disallow: /api/

# Archives and independent indexes
User-agent: ia_archiver
User-agent: archive.org_bot
User-agent: CCBot
User-agent: MojeekBot
User-agent: MarginaliaBot
User-agent: Bravebot
Allow: /
Disallow: /_build/
Disallow: /_server/
Disallow: /api/

# Politeness limit for high-volume crawlers
User-agent: Bytespider
User-agent: PetalBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: AhrefsBot
Crawl-delay: 10

Host: quorlin.dev

Sitemap: https://quorlin.dev/sitemap.xml
Sitemap: https://quorlin.dev/sitemap-pages.xml
Sitemap: https://quorlin.dev/sitemap-docs.xml