# The /api/ rules below are split rather than a blanket Disallow: /api/.
#
# This site publishes an agent-addressable API on purpose: /.well-known/api-catalog
# advertises the read endpoints, /openapi.json documents them, the Agent Skill
# tells agents to fetch them, and every HTML page carries an RFC 8288 Link
# header pointing at the catalog. A blanket disallow meant a well-behaved agent
# followed that whole trail and then refused every endpoint at the end of it —
# including the /api/oembed link injected into each page's head, so link
# unfurling was blocked too. The two documents cannot both be right; this one
# now matches what the site actually invites.
#
# robots.txt has no inheritance — a named user-agent block replaces the
# wildcard one rather than extending it — so the Allow list is repeated in each
# block that should have it. Everything that writes, sends, bills or
# administers stays disallowed by the trailing Disallow: /api/.
#
# Content policy: search=yes, ai-input=yes, ai-train=no.
#
# Stated as a comment rather than as the `Content-Signal:` directive it used to
# be. That directive is a Cloudflare proposal, not part of RFC 9309, so Bing
# Webmaster Tools reported it as an invalid line — a standing error in the one
# tool we read this file's health from, which would mask a real fault later.
# It bought nothing to offset that: Google has said it has no effect on any of
# their crawlers, and Bing does not recognise it. Nothing here was ever an
# enforcement mechanism anyway — AI crawler policy is enforced in Cloudflare AI
# Crawl Control, which is where it belongs. Revisit if the directive is
# standardised and the major crawlers start honouring it.

User-agent: *
Allow: /
Allow: /api/cities
Allow: /api/cities-census
Allow: /api/city/
Allow: /api/facts/
Allow: /api/weather/
Allow: /api/search
Allow: /api/archive/
Allow: /api/territories
Allow: /api/oembed
Allow: /api/embed/
Allow: /api/pricing
Allow: /api/listing-quote
Allow: /api/listing-order
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /confirm
Disallow: /unsubscribe
Disallow: /api/

User-agent: Googlebot-News
Allow: /
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

Sitemap: https://mailerdot.com/sitemap.xml
Sitemap: https://mailerdot.com/news-sitemap.xml

# LLM and AI crawlers — welcome
User-agent: GPTBot
Allow: /
Allow: /api/cities
Allow: /api/cities-census
Allow: /api/city/
Allow: /api/facts/
Allow: /api/weather/
Allow: /api/search
Allow: /api/archive/
Allow: /api/territories
Allow: /api/oembed
Allow: /api/embed/
Allow: /api/pricing
Allow: /api/listing-quote
Allow: /api/listing-order
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Allow: /api/cities
Allow: /api/cities-census
Allow: /api/city/
Allow: /api/facts/
Allow: /api/weather/
Allow: /api/search
Allow: /api/archive/
Allow: /api/territories
Allow: /api/oembed
Allow: /api/embed/
Allow: /api/pricing
Allow: /api/listing-quote
Allow: /api/listing-order
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

User-agent: Claude-Web
Allow: /
Allow: /api/cities
Allow: /api/cities-census
Allow: /api/city/
Allow: /api/facts/
Allow: /api/weather/
Allow: /api/search
Allow: /api/archive/
Allow: /api/territories
Allow: /api/oembed
Allow: /api/embed/
Allow: /api/pricing
Allow: /api/listing-quote
Allow: /api/listing-order
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

User-agent: Anthropic-AI
Allow: /
Allow: /api/cities
Allow: /api/cities-census
Allow: /api/city/
Allow: /api/facts/
Allow: /api/weather/
Allow: /api/search
Allow: /api/archive/
Allow: /api/territories
Allow: /api/oembed
Allow: /api/embed/
Allow: /api/pricing
Allow: /api/listing-quote
Allow: /api/listing-order
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Allow: /api/cities
Allow: /api/cities-census
Allow: /api/city/
Allow: /api/facts/
Allow: /api/weather/
Allow: /api/search
Allow: /api/archive/
Allow: /api/territories
Allow: /api/oembed
Allow: /api/embed/
Allow: /api/pricing
Allow: /api/listing-quote
Allow: /api/listing-order
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin
Disallow: /operators
Disallow: /account
Disallow: /api/

# LLM-readable information
# See https://mailerdot.com/llms.txt for a structured overview
# See https://mailerdot.com/llms-full.txt for comprehensive documentation
# See https://mailerdot.com/developers for the API, MCP server, and error reference
# See https://mailerdot.com/openapi.json for the OpenAPI 3.1 specification