# Robots.txt for www.citedy.com
# Last updated: 2026-02-20

# ============================================================
# SITEMAPS
# ============================================================
Sitemap: https://www.citedy.com/sitemap.xml

# ============================================================
# DEFAULT RULES FOR ALL BOTS
# ============================================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /static/
Disallow: /auth/
Disallow: /account/
Disallow: /billing/
Disallow: /dashboard/
Disallow: /test/
Disallow: /test-analytics/
Disallow: /demo/
Disallow: /update-password/
Disallow: /reset-password/
Disallow: /monitoring
Disallow: /js9c/
Crawl-delay: 1

# Allow specific Next.js resources
Allow: /_next/static/
Allow: /_next/image/
Allow: /api/og/
Allow: /api/roast/badge/

# ============================================================
# AI & LLM CRAWLERS
# ============================================================
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Perplexity
Allow: /

User-agent: Google-Extended
Allow: /

# ============================================================
# BLOCKED BOTS (Aggressive Scrapers)
# ============================================================
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# ============================================================
# REFERENCE TO AI CONFIGURATION
# ============================================================
# See additional LLM access and metadata rules:
# https://www.citedy.com/ai.txt
# https://www.citedy.com/llms.txt
# https://www.citedy.com/llms-full.txt

# LLM-specific index for AI assistants (llmstxt.org standard)
# LLMs-Full: https://www.citedy.com/llms-full.txt