# FollowUP CRM - Robots.txt
# https://followup.agency
# Last updated: July 18, 2026

# ===========================================
# SITEMAP LOCATIONS
# ===========================================
Sitemap: https://followup.agency/sitemap.xml

# ===========================================
# LLMs.txt - AI Training & Context Files
# ===========================================
# These files provide structured information about FollowUP CRM
# for AI models and language learning systems.
#
# llms.txt - Concise summary for quick context:
#   https://followup.agency/llms.txt
# llms-full.txt - Comprehensive documentation for detailed training:
#   https://followup.agency/llms-full.txt

# ===========================================
# DEFAULT RULES - Allow all crawlers
# ===========================================
# All crawlers - search engines, social previews and AI crawlers
# (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, CCBot etc.) -
# are welcome and governed by this "*" group. Per RFC 9309 a bot
# without its own named group falls back to these rules, so the
# Disallow list below applies to every welcome bot.
User-agent: *
Allow: /
Allow: /en/

# Block sensitive paths for all crawlers
Disallow: /api/
Disallow: /admin/
Disallow: /private/
# Next.js assets (JS/CSS/hashed media incl. blog og-images) must stay crawlable
Allow: /_next/static/
Disallow: /_*
Disallow: /*.json$
Disallow: /*?*utm_*
Disallow: /*?*ref=*
Disallow: /*?*source=*
Disallow: /*?*fbclid=*
Disallow: /*?*gclid=*

# Block duplicate content paths
Disallow: /*?*page=*
Disallow: /*?*sort=*
Disallow: /*?*filter=*

# ===========================================
# BLOCK HARMFUL BOTS
# ===========================================

User-agent: AhrefsBot-User
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: WebCopier
Disallow: /