# ============================================================================
# danamic.org / robots.txt
# ============================================================================
#
# TAG ARCHIVES ARE DISALLOWED  (2026-08-13)
#   Measured over two days of logs:
#     - 13,409 bot requests vs 227 non-bot  (98.3% bot traffic)
#     - 11 referrals from Google search, total
#     - 6,098 of 7,965 tags (76.6%) contain exactly ONE post
#   They consumed a large share of a 2-vCPU origin that renders ~2 pages/sec
#   while returning essentially no search traffic.
#
# CRAWL-DELAY  (2026-08-13)
#   The origin is 2 vCPUs and renders roughly 2 uncached pages/sec. In one
#   2h17m window, verified Amazonbot alone took 6,442s of origin CPU across 424
#   source IPs -- 52% of ALL origin time -- averaging 4.3s per article, most of
#   which was the crawler queueing against its own concurrency. Crawl-delay
#   paces the sweep to something the box can absorb.
#
#   NOTHING IS BLOCKED. The full archive is still crawlable; it just gets
#   fetched more slowly. Values are seconds between requests.
#
# ----------------------------------------------------------------------------
# IMPORTANT IF YOU EDIT THIS FILE
# ----------------------------------------------------------------------------
#   A crawler obeys exactly ONE User-agent group -- the most specific one that
#   matches its token -- and ignores every other group, INCLUDING "*". Each
#   named block below therefore REPEATS "Disallow: /tag/". Deleting that line
#   from a named block does not fall back to the "*" block; it silently
#   re-opens tag archives to that one crawler.
#
#   Googlebot is deliberately absent. It ignores Crawl-delay entirely, and
#   adding a named Googlebot block would make it stop reading the "*" block --
#   re-opening /tag/ to the one crawler that actually sends us traffic. Adjust
#   Google via crawl rate settings in Search Console instead.
# ============================================================================

# ----------------------------------------------------------------------------
# CONTENT SIGNALS  (2026-09-02)   https://contentsignals.org
# ----------------------------------------------------------------------------
#   danamic.org WANTS to be indexed and cited by search engines and AI
#   assistants (ChatGPT, Claude, Gemini, Copilot, Perplexity). "search=yes"
#   grants indexing + linking/excerpting; "ai-input=yes" grants use as
#   grounding/RAG input for AI answers that cite us. "ai-train" is deliberately
#   unset (neither granted nor restricted). Because a crawler reads only ONE
#   User-agent group, the Content-Signal line is repeated in every group below.
# ============================================================================

User-agent: *
Disallow: /tag/
Content-Signal: search=yes, ai-input=yes

# --- heaviest consumer: 52% of all origin CPU ------------------------------
User-agent: Amazonbot
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

# --- SEO/backlink crawlers: high volume, no traffic returned ----------------
User-agent: AhrefsBot
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

User-agent: SemrushBot
Disallow: /tag/
Crawl-delay: 20
Content-Signal: search=yes, ai-input=yes

User-agent: MJ12bot
Disallow: /tag/
Crawl-delay: 20
Content-Signal: search=yes, ai-input=yes

User-agent: DotBot
Disallow: /tag/
Crawl-delay: 20
Content-Signal: search=yes, ai-input=yes

User-agent: PetalBot
Disallow: /tag/
Crawl-delay: 20
Content-Signal: search=yes, ai-input=yes

# --- search engines that honour Crawl-delay ---------------------------------
# bingbot gets a gentler value: it returns real search traffic.
User-agent: bingbot
Disallow: /tag/
Crawl-delay: 5
Content-Signal: search=yes, ai-input=yes

User-agent: Baiduspider
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

User-agent: YandexBot
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

# --- Meta's indexer ---------------------------------------------------------
User-agent: meta-externalagent
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

User-agent: meta-webindexer
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

# --- AI crawlers: allowed to read the archive, just paced -------------------
User-agent: GPTBot
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

User-agent: ClaudeBot
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

User-agent: PerplexityBot
Disallow: /tag/
Crawl-delay: 10
Content-Signal: search=yes, ai-input=yes

User-agent: Bytespider
Disallow: /tag/
Crawl-delay: 20
Content-Signal: search=yes, ai-input=yes

User-agent: CCBot
Disallow: /tag/
Crawl-delay: 20
Content-Signal: search=yes, ai-input=yes

Sitemap: https://danamic.org/sitemap_index.xml