# ─────────────────────────────────────────────────────────────────
# STNews · robots.txt
# Real-time stablecoin & crypto news · Bedrock WordPress
# Goal: maximize crawl budget for fresh stories; allow AI citation.
# Updated: 2026-05-25
# ─────────────────────────────────────────────────────────────────

# ── Sitemaps (news sitemap first = freshness priority signal) ──
Sitemap: https://stnews.live/news-sitemap.xml
Sitemap: https://stnews.live/sitemap_index.xml
Sitemap: https://stnews.live/sitemap-coins.xml
Sitemap: https://stnews.live/post-sitemap.xml

# ── Universal policy (search engines + AI citation bots inherit) ──
User-agent: *

# Render-critical paths (allow before deeper blocks)
Allow: /app/uploads/
Allow: /app/themes/stnews/dist/
Allow: /app/themes/stnews/assets/
Allow: /wp/wp-admin/admin-ajax.php

# WP admin + source code
Disallow: /wp/wp-admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /app/plugins/
Disallow: /app/mu-plugins/
Disallow: /app/themes/stnews/node_modules/
Disallow: /app/themes/stnews/src/
Disallow: /app/themes/stnews/vendor/

# Crawl-budget noise (internal search, tracking params, replies)
Disallow: /?s=
Disallow: /*?s=
Disallow: /*&s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?utm_term=
Disallow: /*?utm_content=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?msclkid=
Disallow: /*?ref=
Disallow: /*?source=
Disallow: /*?share=
Disallow: /*?amp=
Disallow: /*?print=
Disallow: /*&utm_
Disallow: /*&fbclid=
Disallow: /*&gclid=

# Trackbacks & comment feeds
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /comments/feed/
Disallow: /*/feed/atom/

# ── Blocked scrapers (no traffic value, just resource consumption) ──
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: webmeup-crawler
Disallow: /

User-agent: NPbot
Disallow: /

# ── Throttle commercial SEO crawlers (Google + AI bots unthrottled) ──
User-agent: AwarioRssBot
Crawl-delay: 30

User-agent: AwarioSmartBot
Crawl-delay: 30

# ─── INTERNATIONAL SEARCH ENGINES (Phase 1 global rank) ───
# Allow all major non-Google engines full crawl access
User-agent: YandexBot
Allow: /
Crawl-delay: 1

User-agent: YandexImages
Allow: /

User-agent: YandexNews
Allow: /

User-agent: YandexMetrika
Allow: /

User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

User-agent: NaverBot
Allow: /

User-agent: Yeti
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Baiduspider-news
Allow: /

User-agent: Sogou web spider
Allow: /

User-agent: Sogou inst spider
Allow: /

User-agent: Applebot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

# ─── Additional sitemaps ───
Sitemap: https://stnews.live/page-sitemap.xml
Sitemap: https://stnews.live/category-sitemap.xml
Sitemap: https://stnews.live/author-sitemap.xml

# ===== SEO tools (explicit allow — needed for site audits + backlink intel) =====
# Ahrefs (Site Audit + Backlinks + Rank Tracker)
User-agent: AhrefsBot
Allow: /

User-agent: AhrefsSiteAudit
Allow: /

# Semrush (Site Audit + Backlink Analytics + Position Tracking)
User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

User-agent: SemrushBot-BA
Allow: /

User-agent: SemrushBot-SI
Allow: /

User-agent: SiteAuditBot
Allow: /

# Majestic
User-agent: MJ12bot
Allow: /

# Moz (Link Explorer + Pro Site Audit)
User-agent: rogerbot
Allow: /

User-agent: DotBot
Allow: /

User-agent: dotbot
Allow: /

# DataForSEO (powers many SEO platforms)
User-agent: DataForSeoBot
Allow: /

# Screaming Frog
User-agent: ScreamingFrogSEOSpider
Allow: /

# Sitebulb
User-agent: SitebulbBot
Allow: /

# BacklinkChecker
User-agent: BLEXBot
Allow: /

# Pinterest (social discovery)
User-agent: Pinterestbot
Allow: /

# ===== AI assistants / answer engines (explicit opt-in for citation) =====
# The policy header above states the goal is to allow AI citation, but no AI
# crawler was ever named — they only inherited "User-agent: *". Naming them makes
# the opt-in explicit. CCBot is deliberately NOT listed here: it stays blocked in
# the "Blocked scrapers" section above, which is an existing, intentional choice.

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

# DuckDuckGo assistant
User-agent: DuckAssistBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /