# Content Signals (https://contentsignals.org/)
# Declares how this site's content may be used, independent of crawl permission.
#   search    = appear in search results
#   ai-input  = be retrieved at answer time (RAG / grounding)
#   ai-train  = be used to train a model
# Yes to search and answer-time retrieval, no to training.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# The API catalog is the one machine-readable discovery document this site
# publishes. It has no file extension, so the blanket "Disallow: /*.json$"
# further down does not reach it, but the explicit Allow keeps that true if the
# catalog ever moves to a .json path.
Allow: /.well-known/

# Google and Bing specific optimizations
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Security and System Protection
Disallow: /_astro/
Disallow: /_image/
Disallow: /admin/
Disallow: /draft/
Disallow: /tmp/
Disallow: /components/
Disallow: /*.json$
Disallow: /*.config$
Disallow: /*.env$

# Prevent duplicate content from tracking parameters
Disallow: /*?*utm_*
Disallow: /*?*fbclid=*
Disallow: /*?*gclid=*

# Public content, one entry per public route family
Allow: /aircraft-maintenance/
Allow: /aopa-available-now-for-flight-school/
Allow: /ap-school/
Allow: /army-credentialing-assistance/
Allow: /aviation-guides/
Allow: /aviation-guides/*
Allow: /aviation-services/
Allow: /becoming-a-pilot/
Allow: /best-flight-school-international-students/
Allow: /certified-flight-instructor-course/
Allow: /commercial-pilot-training/
Allow: /comparisons/
Allow: /comparisons/*
Allow: /contact-us/
Allow: /designated-pilot-examiner/
Allow: /discovering-aviation/
Allow: /discovery-flight-gift-certificates/
Allow: /enroll-with-us/
Allow: /faa-knowledge-testing-center/
Allow: /flight-school/
Allow: /flight-school/*
Allow: /flight-school-financial-aid/
Allow: /edly-loan-program/
Allow: /flight-school-international-students/
Allow: /flight-school-mission-values/
Allow: /frequently-asked-questions/
Allow: /frequently-asked-questions/*
Allow: /greenville-downtown-airport/
Allow: /ground-school-courses/
Allow: /instrument-rating/
Allow: /multi-engine-rating/
Allow: /multiengine-training/
Allow: /multiengine-training/*
Allow: /news-tips/
Allow: /news-tips/*
Allow: /pilot-training/
Allow: /pilot-training-in-greenville-sc/
Allow: /pilot-training-in-spartanburg-sc/
Allow: /privacy-policy/
Allow: /private-pilot-training/
Allow: /rotary-fixed-wing-transition/
Allow: /sc-flight-instructors/
Allow: /sc-flight-instructors/*
Allow: /terms-of-service/
Allow: /university-partnerships/
Allow: /ccbc-partnership/
Allow: /veterans-benefits/

# Asset indexing
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$

# ==============================
# AI crawlers
# ==============================
# Split by purpose, to match the Content-Signal above.
#
# Training crawlers are blocked: they bulk-collect pages to train models and
# send back no traffic. Answer-time crawlers are allowed: they fetch a page
# because a user asked something right now, and they cite the source. For a
# lead-gen site those citations are the whole point, so blocking them costs
# real traffic.

# --- Training / bulk collection: blocked (ai-train=no) ---
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# --- Answer-time retrieval and AI search: allowed (ai-input=yes, search=yes) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Block SEO Spiders
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Sitemaps and Host
Sitemap: https://usaeroflight.net/sitemap-index.xml
Sitemap: https://usaeroflight.net/rss.xml
Host: usaeroflight.net