# NuAura.Ai — robots.txt
# Updated: June 2026
# Strategy: allow all AI search/citation crawlers for maximum AI visibility
# (B2B SaaS — all content is public, no gated/paywall pages to protect)

# -------------------------------------------------------
# Traditional search engines
# -------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /
# Bing powers Microsoft Copilot — important for B2B AI search

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# -------------------------------------------------------
# OpenAI
# GPTBot = training crawler
# OAI-SearchBot = ChatGPT search index (cite-driving)
# ChatGPT-User = real-time fetch when a user asks ChatGPT
# -------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /
# Required to appear in ChatGPT search answers

User-agent: ChatGPT-User
Allow: /
# Real-time fetch — drives actual referral traffic

# -------------------------------------------------------
# Anthropic / Claude
# ClaudeBot = model training
# Claude-SearchBot = Claude search index (cite-driving)
# Claude-User = real-time fetch at user request
# Each is independent — must be listed separately
# -------------------------------------------------------
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# -------------------------------------------------------
# Perplexity
# -------------------------------------------------------
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# -------------------------------------------------------
# Google AI (Gemini / AI Overviews)
# Google-Extended controls Gemini training independently
# from Googlebot — allowing both keeps full Google coverage
# -------------------------------------------------------
User-agent: Google-Extended
Allow: /

# -------------------------------------------------------
# Microsoft / Copilot
# -------------------------------------------------------
User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

# -------------------------------------------------------
# You.com
# -------------------------------------------------------
User-agent: YouBot
Allow: /

# -------------------------------------------------------
# Apple
# -------------------------------------------------------
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# -------------------------------------------------------
# Common Crawl — used to train many open-source LLMs
# Optional: allow so NuAura.Ai appears in open training sets
# -------------------------------------------------------
User-agent: CCBot
Allow: /

# -------------------------------------------------------
# Block non-compliant / abusive crawlers
# Bytespider (ByteDance) has a documented history of
# ignoring robots.txt — block here as a signal;
# add a WAF rule for hard enforcement if needed
# -------------------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

# -------------------------------------------------------
# Default: allow everything else
# -------------------------------------------------------
User-agent: *
Allow: /

Sitemap: https://nuaura.ai/sitemap.xml