# atomfoundry.dev: robots.txt
# Atom Foundry measures how AI sees, trusts, and recommends digital merchants.
# We WANT AI to read everything: parametric memory (training) and live retrieval
# both drive recommendations. Every reputable crawler is welcome to read the site.
#
# Exceptions, off-limits to all bots:
# - /scan is the free-scan tool. Letting crawlers hit it would trigger real
#   scans and burn the system.
# - /audit, /audit-processing, /audit-in-progress, and /report are private,
#   token-gated pages delivered to a single paying customer. They are for
#   that person only, never for indexing or crawling.
# - /ai-traffic and /human-traffic are the founder's own internal traffic
#   dashboards, not customer-facing, never for indexing or crawling.
# (Robots groups do NOT inherit, so the same rules are repeated for named AI bots.)

# --- AI answer engines, assistants, and training crawlers (explicit opt-in) ---
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Bytespider
Allow: /
Disallow: /scan
Disallow: /audit
Disallow: /audit-processing
Disallow: /audit-in-progress
Disallow: /report
Disallow: /ai-traffic
Disallow: /ai-traffic-cs
Disallow: /human-traffic
Disallow: /*?search=
Disallow: /*?filter=
Disallow: /*?q=
Disallow: /*?sort=

# --- Everyone else (classic search + all other bots) ---
User-agent: *
Allow: /
Disallow: /scan
Disallow: /audit
Disallow: /audit-processing
Disallow: /audit-in-progress
Disallow: /report
Disallow: /ai-traffic
Disallow: /ai-traffic-cs
Disallow: /human-traffic
Disallow: /*?search=
Disallow: /*?filter=
Disallow: /*?q=
Disallow: /*?sort=

Sitemap: https://atomfoundry.dev/sitemap.xml