# Allow all crawlers
User-agent: *
Disallow: /admin
Disallow: /checkout
Disallow: /search
Disallow: /api

# Crawl-delay for all bots
Crawl-delay: 1

# Special rules for major search engines
User-agent: Googlebot
Disallow: /admin
Disallow: /checkout
Disallow: /search
Disallow: /api
Crawl-delay: 0

User-agent: Bingbot
Disallow: /admin
Disallow: /checkout
Disallow: /search
Disallow: /api
Crawl-delay: 1

# Link to sitemap - added dynamically during prerender

# Sitemap with absolute URL
Sitemap: https://www.webarch.io/sitemap.xml