# navNote Website Robots.txt
# Updated: April 2026
# Site: https://www.navnote.ai
# HQ: 5550 Hellyer Ave, San Jose, CA 95138, USA
# Contact: contact@navNote.ai
#
# Note: each User-agent block is independent per the robots.txt spec.
# Every block that grants broad access MUST repeat the same Disallow
# rules so private routes stay private. The private set is:
#   /invest, /admin, /api/, /src/, /.git/
#   /pitch-deck     investor deck slides. The files must stay deployed
#                   because the in-dashboard demo slideshow imports them,
#                   so they are blocked here rather than deleted.
#   /walkthrough.md partner-only integration doc, no longer published
# The deck is additionally served with X-Robots-Tag: noindex in
# firebase.json, because the broad "Allow: /*.png" rule below would
# otherwise out-specify this Disallow under longest-match resolution.

# -------------------------------------------------------------------
# Default policy for every well-behaved bot.
# -------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /contact
Allow: /team
Allow: /about
Allow: /careers
Allow: /navigator
Allow: /privacy-policy
Allow: /terms-of-use
Allow: /email-communications
Allow: /cookie-settings
Allow: /account-policy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /updates
Allow: /official-profiles.json
Allow: /updates-feed.json
Allow: /structured-data.json
Allow: /person-esmaeil-mousavi.json
Allow: /person-niklas-kennedy.json
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /src/
Disallow: /.git/
Disallow: /.bolt/
Disallow: /node_modules/
Disallow: /api/
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.svg
Allow: /*.webp
Allow: /*.gif

# -------------------------------------------------------------------
# AI / LLM crawlers - explicitly allowed for public pages,
# explicitly blocked from private routes.
# -------------------------------------------------------------------

# OpenAI - ChatGPT / SearchGPT
User-agent: GPTBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: ChatGPT-User
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: OAI-SearchBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Anthropic - Claude
User-agent: ClaudeBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: Claude-Web
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: anthropic-ai
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: Perplexity-User
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Google AI Overviews / Gemini (separate from Googlebot)
User-agent: Google-Extended
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Meta - Meta AI / Llama
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: FacebookBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Apple Intelligence / Siri
User-agent: Applebot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: Applebot-Extended
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Amazon - Alexa / Q
User-agent: Amazonbot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Common Crawl (used by many LLMs as training data)
User-agent: CCBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# ByteDance - Doubao
User-agent: Bytespider
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# You.com
User-agent: YouBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# DuckDuckGo Assist
User-agent: DuckAssistBot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# Mistral / Diffbot / Timpi
User-agent: MistralAI-User
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: Diffbot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

User-agent: Timpibot
Allow: /
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/
Disallow: /src/
Disallow: /.git/

# -------------------------------------------------------------------
# Heavy third-party crawlers - be nice to our servers.
# -------------------------------------------------------------------
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/

User-agent: MJ12bot
Crawl-delay: 10
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/

User-agent: DotBot
Crawl-delay: 10
Disallow: /invest
Disallow: /admin
Disallow: /pitch-deck
Disallow: /walkthrough.md
Disallow: /api/

# -------------------------------------------------------------------
# AI / LLM Context Files (the new "robots.txt for LLMs" standard)
#   - https://www.navnote.ai/llms.txt       (concise, link-style overview)
#   - https://www.navnote.ai/llms-full.txt  (long-form context document)
# These are the canonical AI-readable description of navNote. Crawlers
# from OpenAI, Anthropic, Perplexity, Google, etc. that follow the
# llms.txt convention will look for these at /llms.txt by default.
# -------------------------------------------------------------------

# -------------------------------------------------------------------
# Sitemaps
# -------------------------------------------------------------------
Sitemap: https://www.navnote.ai/sitemap.xml
Sitemap: https://www.navnote.ai/image-sitemap.xml