# =============================================================================
#  ____  ____  ____  ____  ____  ____
# |  _ \     /\    / ___|____|  ____|  ____
# |  | |    /  \  | |  _ |__  | |__  |  _ \
# |  |_|   / /\ \ | |_| || |  |  __| | |_) |
# |____/  /_/  \_\ \____|____|_|    |____/
#
# robots.txt — Enterprise-Grade Configuration
# Version: 2.0 | Updated: May 2026
# Domain: dagzer.com / www.dagzer.com
# Docs: https://developers.google.com/search/docs/crawling-indexing/robots/intro
# RFC:  https://www.rfc-editor.org/rfc/rfc9309
# =============================================================================
#
# POLICY SUMMARY:
#   - All legitimate bots:          ALLOWED (full site)
#   - AI / LLM training crawlers:   ALLOWED (full site)
#   - SEO & analytics tools:        ALLOWED (full site, /api/ excluded)
#   - Social preview crawlers:      ALLOWED (full site)
#   - Archive / research bots:      ALLOWED (full site)
#   - API endpoints:                BLOCKED (all bots)
#   - Admin paths:                  BLOCKED (all bots)
#   - Preview/draft params:         BLOCKED (all bots)
#   - Malicious / spam bots:        BLOCKED
#
# =============================================================================


# =============================================================================
# SECTION 1 — GLOBAL DEFAULT
# Applies to every crawler not explicitly named below.
# "Allow: /" with minimal disallows is the most permissive safe default.
# =============================================================================

User-agent: *
Allow: /

# Internal / private paths — block all bots
Disallow: /api/
Disallow: /api/*
Disallow: /admin/
Disallow: /admin/*
Disallow: /_next/
Disallow: /dashboard/
Disallow: /internal/
Disallow: /private/
Disallow: /staging/
Disallow: /preview/
Disallow: /draft/
Disallow: /cdn-cgi/
Disallow: /tmp/
Disallow: /cache/
Disallow: /.well-known/
Disallow: /wp-admin/
Disallow: /wp-login.php

# Block noisy query parameters (duplicate content, preview modes)
Disallow: /*?preview=
Disallow: /*?draft=
Disallow: /*?token=
Disallow: /*?ref=
Disallow: /*?utm_source=
Disallow: /*?sessionid=

# Block raw data/config files
Disallow: /*.json
Disallow: /*.xml$
Disallow: /*.config
Disallow: /*.env
Disallow: /*.log

# Sitemap declarations — both www and non-www for maximum compatibility
# Primary sitemap (contains all pages with image/video metadata)
Sitemap: https://www.dagzer.com/sitemap.xml
Sitemap: https://dagzer.com/sitemap.xml

# Sitemap index (lists all sitemap files)
Sitemap: https://www.dagzer.com/sitemap-index.xml
Sitemap: https://dagzer.com/sitemap-index.xml

# Image sitemap (helps Google Images index logos and screenshots)
Sitemap: https://www.dagzer.com/sitemap-images.xml
Sitemap: https://dagzer.com/sitemap-images.xml

# Subdomain sitemaps
Sitemap: https://docs.dagzer.com/sitemap.xml
Sitemap: https://api.dagzer.com/sitemap.xml
Sitemap: https://status.dagzer.com/sitemap.xml


# =============================================================================
# SECTION 2 — GOOGLE CRAWLERS
# Full access to all Google bots for maximum search visibility.
# Ref: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
# =============================================================================

# Main web crawler
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/

# Image indexing
User-agent: Googlebot-Image
Allow: /

# Google News
User-agent: Googlebot-News
Allow: /

# Video indexing
User-agent: Googlebot-Video
Allow: /

# Shopping / product indexing
User-agent: Storebot-Google
Allow: /

# Google Inspect / testing tool
User-agent: Google-InspectionTool
Allow: /

# AdsBot (required explicit listing — not covered by wildcard)
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

# Google Read Aloud
User-agent: Google-Read-Aloud
Allow: /

# Google Site Verification
User-agent: Google-Site-Verification
Allow: /


# =============================================================================
# SECTION 3 — MICROSOFT / BING CRAWLERS
# =============================================================================

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: MSNBot
Allow: /

User-agent: MSNBot-Media
Allow: /

User-agent: AdIdxBot
Allow: /

User-agent: BingPreview
Allow: /

# Microsoft Copilot AI (Bing-powered)
User-agent: Copilot
Allow: /


# =============================================================================
# SECTION 4 — OTHER SEARCH ENGINES
# =============================================================================

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

User-agent: DuckDuckGo-Favicons-Bot
Allow: /

# Yahoo (uses Slurp)
User-agent: Slurp
Allow: /

# Baidu (China)
User-agent: Baiduspider
Allow: /

User-agent: Baiduspider-image
Allow: /

User-agent: Baiduspider-video
Allow: /

User-agent: Baiduspider-news
Allow: /

# Yandex (Russia)
User-agent: Yandex
Allow: /

User-agent: YandexBot
Allow: /

User-agent: YandexImages
Allow: /

User-agent: YandexVideo
Allow: /

User-agent: YandexNews
Allow: /

User-agent: YandexMobileBot
Allow: /

# Naver (South Korea)
User-agent: Yeti
Allow: /

# Seznam (Czech Republic)
User-agent: SeznamBot
Allow: /

# Sogou (China)
User-agent: Sogou web spider
Allow: /

User-agent: Sogou inst spider
Allow: /

# Ecosia (Germany, privacy-first)
User-agent: Ecosia
Allow: /

# Brave Search
User-agent: BraveBot
Allow: /

# Mojeek (independent UK index)
User-agent: MojeekBot
Allow: /

# Qwant (France)
User-agent: Qwantify
Allow: /

# Petal Search / Huawei
User-agent: PetalBot
Allow: /

# Exabot (France)
User-agent: Exabot
Allow: /

# Applebot (Siri, Spotlight, Safari)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /


# =============================================================================
# SECTION 5 — AI / LLM TRAINING & INFERENCE CRAWLERS
# Allowing all known AI crawlers for maximum AI search visibility.
# This is intentional — Dagzer wants to appear in AI-generated answers.
# =============================================================================

# --- OpenAI / ChatGPT ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# --- Anthropic / Claude ---
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# --- Google AI (Gemini, Bard, Search Generative Experience) ---
User-agent: Google-Extended
Allow: /

User-agent: Gemini-Web
Allow: /

# --- Perplexity AI ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- You.com ---
User-agent: YouBot
Allow: /

# --- Cohere ---
User-agent: cohere-ai
Allow: /

# --- Meta AI ---
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# --- Amazon / AWS Bedrock ---
User-agent: AmazonBot
Allow: /

User-agent: Amazonbot
Allow: /

# --- Apple AI (Apple Intelligence) ---
User-agent: Applebot-Extended
Allow: /

# --- Mistral AI ---
User-agent: MistralBot
Allow: /

# --- xAI / Grok ---
User-agent: xAI
Allow: /

User-agent: Grok
Allow: /

# --- Diffbot (AI structured data extraction) ---
User-agent: Diffbot
Allow: /

# --- Common Crawl (used by many LLMs for training) ---
User-agent: CCBot
Allow: /

# --- Bytedance / Doubao ---
User-agent: Bytespider
Allow: /

# --- Webz.io (AI data provider) ---
User-agent: Webzio-Extended
Allow: /

# --- Brightbot ---
User-agent: Brightbot
Allow: /

# --- Timpibot ---
User-agent: Timpibot
Allow: /

# --- DataForSeoBot ---
User-agent: DataForSeoBot
Allow: /

# --- AI2Bot (Allen Institute for AI) ---
User-agent: AI2Bot
Allow: /

# --- Omgili / Webz ---
User-agent: omgilibot
Allow: /

# --- iaskspider ---
User-agent: iaskspider
Allow: /

# --- Kangaroo (Indian AI) ---
User-agent: KangarooBot
Allow: /


# =============================================================================
# SECTION 6 — SOCIAL MEDIA & PREVIEW CRAWLERS
# Allow all social platforms to render rich link previews.
# =============================================================================

# Facebook / Meta
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: FacebookBot
Allow: /

# Instagram (Meta)
User-agent: Instagram
Allow: /

# Twitter / X
User-agent: Twitterbot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Pinterest
User-agent: Pinterest
Allow: /

User-agent: PinterestBot
Allow: /

# WhatsApp (Meta)
User-agent: WhatsApp
Allow: /

# Telegram
User-agent: TelegramBot
Allow: /

# Slack (link unfurling)
User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

# Discord (link previews)
User-agent: Discordbot
Allow: /

# Snapchat
User-agent: Snapchat
Allow: /

# Reddit
User-agent: redditbot
Allow: /

# YouTube (link resolution)
User-agent: YouTubeBot
Allow: /

# Line (Asia messaging app)
User-agent: Line
Allow: /

# Viber
User-agent: Viber
Allow: /

# TikTok
User-agent: Bytedance
Allow: /

# Notion (link previews)
User-agent: NotionBot
Allow: /

# Skype / Teams preview
User-agent: SkypeUriPreview
Allow: /


# =============================================================================
# SECTION 7 — SEO TOOLS & SITE MONITORING
# Allow SEO tools full access except /api/ endpoints.
# =============================================================================

User-agent: AhrefsBot
Allow: /
Disallow: /api/

User-agent: AhrefsSiteAudit
Allow: /
Disallow: /api/

User-agent: SemrushBot
Allow: /
Disallow: /api/

User-agent: SemrushBot-SA
Allow: /
Disallow: /api/

User-agent: rogerbot
Allow: /
Disallow: /api/

User-agent: dotbot
Allow: /
Disallow: /api/

User-agent: MJ12bot
Allow: /
Disallow: /api/

User-agent: BLEXBot
Allow: /
Disallow: /api/

User-agent: SEOkicks-Robot
Allow: /

User-agent: seokicks
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

User-agent: SiteAuditBot
Allow: /

User-agent: DatadogSynthetics
Allow: /

User-agent: GTmetrix
Allow: /

User-agent: PTST
Allow: /

User-agent: pingdom
Allow: /

User-agent: UptimeRobot
Allow: /

User-agent: StatusCake
Allow: /

User-agent: Zabbix
Allow: /

# Majestic (backlink checker)
User-agent: MagesticSEO
Allow: /

User-agent: MJ12bot
Allow: /

# Moz
User-agent: rogerbot
Allow: /

User-agent: Exabot
Allow: /

# Sitebulb
User-agent: Sitebulb
Allow: /

# Lumar / DeepCrawl
User-agent: Deepcrawl
Allow: /

# OnCrawl
User-agent: OnCrawlBot
Allow: /

# Conductor
User-agent: ConductorCrawler
Allow: /

# BrightEdge
User-agent: BrightEdge
Allow: /


# =============================================================================
# SECTION 8 — WEB PERFORMANCE & CDN CRAWLERS
# =============================================================================

# Cloudflare
User-agent: Cloudflare-SSLDetector
Allow: /

User-agent: Cloudflare-AlwaysOnline
Allow: /

# Fastly
User-agent: Fastly-Healthcheck
Allow: /

# Vercel
User-agent: vercel-screenshot
Allow: /

# Netlify
User-agent: Netlify
Allow: /


# =============================================================================
# SECTION 9 — DEVELOPER TOOLS & API PLATFORMS
# =============================================================================

# Postman (API testing)
User-agent: PostmanRuntime
Allow: /

# RapidAPI
User-agent: RapidAPI
Allow: /

# AWS health checks
User-agent: ELB-HealthChecker
Allow: /

User-agent: AWS-Support-TechTools
Allow: /

# Generic health check agents
User-agent: HealthCheck
Allow: /

User-agent: health_check
Allow: /


# =============================================================================
# SECTION 10 — ARCHIVE & RESEARCH CRAWLERS
# =============================================================================

# Internet Archive / Wayback Machine
User-agent: archive.org_bot
Allow: /

User-agent: ia_archiver
Allow: /

User-agent: Wayback Machine
Allow: /

# Common Crawl Foundation
User-agent: CCBot
Allow: /

# EU Web Archive
User-agent: heritrix
Allow: /

# Academic / research
User-agent: Mediapartners-Google
Allow: /


# =============================================================================
# SECTION 11 — SECURITY & ACCESSIBILITY SCANNERS
# Allow legitimate security and a11y tools.
# =============================================================================

User-agent: Detectify
Allow: /

User-agent: ImplisenseBot
Allow: /

User-agent: SiteCheckerBotCrawler
Allow: /

User-agent: W3C_Validator
Allow: /

User-agent: W3C-checklink
Allow: /

User-agent: W3C_CSS_Validator
Allow: /

# Lighthouse (Google PageSpeed)
User-agent: Chrome-Lighthouse
Allow: /


# =============================================================================
# SECTION 12 — KNOWN MALICIOUS & SPAM BOTS — BLOCKED
# These bots scrape for spam, email harvesting, or malicious purposes.
# =============================================================================

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: Harvest
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Wget
Disallow: /

User-agent: libwww
Disallow: /

User-agent: lwp-trivial
Disallow: /

User-agent: curl
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: scrapy
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: spbot
Disallow: /

User-agent: zgrab
Disallow: /

User-agent: masscan
Disallow: /

User-agent: nikto
Disallow: /

User-agent: sqlmap
Disallow: /

User-agent: BlackWidow
Disallow: /

User-agent: Zeus
Disallow: /

User-agent: Zeus Link Scout
Disallow: /

User-agent: backdoorbot
Disallow: /

User-agent: Bad-Neighborhood
Disallow: /

User-agent: Flunky
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: looksmart
Disallow: /

User-agent: WebAuto
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebEnhancer
Disallow: /

User-agent: WebFetch
Disallow: /

User-agent: WebGo IS
Disallow: /

User-agent: WebLeacher
Disallow: /

User-agent: WebMirror
Disallow: /

User-agent: WebRipper
Disallow: /

User-agent: Webster Pro
Disallow: /

User-agent: WebSauger
Disallow: /

User-agent: Website eXtractor
Disallow: /

User-agent: Website Quester
Disallow: /

User-agent: Wells Search II
Disallow: /

User-agent: WEP Search
Disallow: /

User-agent: WISENutbot
Disallow: /

User-agent: WWWOFFLE
Disallow: /

User-agent: Xaldon WebSpider
Disallow: /

User-agent: Xenu
Disallow: /


# =============================================================================
# END OF FILE
# =============================================================================
# Last updated: May 2026
# Maintainer: admin@dagzer.com
# Applies to: https://dagzer.com/ and https://www.dagzer.com/
# Validate at: https://search.google.com/search-console
# =============================================================================