frontend/public/robots.txt

# Block all crawlers for /accounts
User-agent: *
Disallow: /accounts

# Allow all crawlers
User-agent: *
Allow: /

# Delay between requests
Crawl-delay: 5

Sitemap: https://smart-retrieval-demo.vercel.app/sitemap.xml