# MerchantWords robots policy.
#
# Philosophy: allow search engines (they drive qualified traffic) and
# live-retrieval AI bots that cite sources (they also drive clicks).
# Block training-only AI crawlers — our product IS keyword/ASIN data,
# and we're not interested in having it laundered into model weights
# with no referral in return.
# Exception (2026-07-14): Google-Extended is ALLOWED — it gates Gemini-app /
# Vertex grounding+citation (better cited by us than by a competitor), and it
# never controlled Google AI Overviews anyway (those use the Search index).

# ---------- Training crawlers: explicitly disallowed ----------
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# (Google-Extended intentionally NOT disallowed — allowed for Gemini citation; see header.)

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ---------- Everyone else: allowed with standard carve-outs ----------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /my-account
Disallow: /login
Disallow: /signup
Disallow: /activate/
Disallow: /new-password/
Disallow: /bulk-search
Disallow: /tracker

Sitemap: https://www.merchantwords.com/sitemap.xml