User-agent: *
Allow: /
Allow: /sitemap.xml
Allow: /assets/
Allow: /images/
Allow: /uploads/
Allow: /public/

# Disallow admin and sensitive paths
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /password/
Disallow: /cron/
Disallow: /api/
Disallow: /*.php$
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*search=
Disallow: /tmp/
Disallow: /vendor/
Disallow: /.env
Disallow: /.git
Disallow: /storage/
Disallow: /bootstrap/
Disallow: /config/
Disallow: /database/

# Allow Googlebot specific access
User-agent: Googlebot
Allow: /*.js$
Allow: /*.css$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$

# Allow Googlebot Image specific access
User-agent: Googlebot-Image
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$

# Allow Bingbot specific access
User-agent: Bingbot
Allow: /*.js$
Allow: /*.css$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$

# Crawl delay for heavy bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# Sitemaps
Sitemap: https://s3.pragnews.com/sitemap.xml
Sitemap: https://s3.pragnews.com/sitemap_news.xml
Sitemap: https://s3.pragnews.com/sitemap_images.xml
Sitemap: https://s3.pragnews.com/sitemap_videos.xml

# Host
Host: https://s3.pragnews.com