# Robots.txt for Yell STL - St. Louis Events & Restaurant Guide

User-agent: *
Allow: /

# High priority pages for crawling
Allow: /calendar
Allow: /finder
Allow: /recommendations
Allow: /cheap-eats
Allow: /stories
Allow: /careers
Allow: /about

# Neighborhood pages for local SEO
Allow: /neighborhoods/
Allow: /events/

# RSS feed for content discovery
Allow: /rss.xml

# Admin areas - restrict crawling but allow access
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /static/
Disallow: /sw.js

# Allow specific admin login page for SEO
Allow: /admin/login

# Sitemap location
Sitemap: https://yellstl.com/sitemap.xml

# RSS feed location
Sitemap: https://yellstl.com/rss.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Specific instructions for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 2

# Social media crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: InstagramBot
Allow: /

User-agent: WhatsApp
Allow: /

# News and content aggregators
User-agent: Applebot
Allow: /

User-agent: ia_archiver
Allow: /

# Block unwanted bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MauiBot
Disallow: /

# Allow search engines to crawl job postings
User-agent: *
Allow: /careers

# Enhanced crawling for local content
User-agent: *
Allow: /influencer-tracker
Allow: /data-import

# Local business directories
User-agent: YelpBot
Allow: /

User-agent: TripAdvisorBot
Allow: /

# Performance monitoring bots
User-agent: GTmetrix
Allow: /

User-agent: PageSpeed
Allow: /