# Tender84 robots.txt
# Policy: search=yes | ai-search=yes | ai-train=no
# If you see Cloudflare "Managed content" Disallow blocks above this
# section, disable Cloudflare AI Crawl Control / managed robots.txt.

User-agent: *
Allow: /
Disallow: /account/
Disallow: /my-account/
Disallow: /manager/
Disallow: /api/
Disallow: /login/
Disallow: /sql/
Disallow: /logs/
Disallow: /data/detail_cache/
Disallow: /dev/
Disallow: /xdev/
Disallow: /t84-v5/
Disallow: /v5_staging/

# AI Search / Answer crawlers - ALLOWED (discoverability)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: GoogleProducer
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

# Pure training scrapers - BLOCKED (not free training data)
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: ImagesiftBot
Disallow: /

Content-Signal: search=yes,ai-train=no,ai-search=yes,ai-input=yes

# TDM: AI search indexing allowed; bulk training disallowed
TDM: 1
TDM-Reservation: 0

# AI discovery endpoints
#   https://tender84.com/llms.txt
#   https://tender84.com/llms-full.txt
#   https://tender84.com/mcp
#   https://tender84.com/api-docs

Sitemap: https://tender84.com/sitemap.xml
LLMs: https://tender84.com/llms.txt