# robots.txt for fbargainsgalore.co.uk
# Last updated: 2025-09-07

# Default rules for all crawlers
User-agent: *
Allow: /

# Crawl delay to prevent server overload (1 second)
Crawl-delay: 1

# === SECURITY & PRIVACY BLOCKS ===
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-content/uploads/wpo/
Disallow: /cart/
Disallow: /checkout/
Disallow: /customer/
Disallow: /account/
Disallow: /my-account/
Disallow: /login
Disallow: /register
Disallow: /password-reset
Disallow: /wishlist/
Disallow: /compare/
Disallow: /track-order/
Disallow: /order-tracking/
Disallow: /returns/
Disallow: /refunds/
Disallow: /ajax/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /private/
Disallow: /tmp/
Disallow: /temp/
Disallow: /backup/
Disallow: /error/
Disallow: /404/
Disallow: /500/

# === CONVERSION & THANK YOU PAGES ===
Disallow: /thank-you/
Disallow: /order-received/
Disallow: /order-confirmation/
Disallow: /payment-success/
Disallow: /payment-failed/

# === SEARCH & FILTER PARAMETERS ===
Disallow: /search
Disallow: /search/
Disallow: /*?q=*
Disallow: /*?search=*
Disallow: /*?s=*
Disallow: /*?query=*
Disallow: /*?keyword=*

# === DUPLICATE CONTENT PREVENTION ===
# Sorting and filtering parameters
Disallow: /*?sort=*
Disallow: /*?order=*
Disallow: /*?orderby=*
Disallow: /*?filter=*
Disallow: /*?price=*
Disallow: /*?color=*
Disallow: /*?size=*
Disallow: /*?brand=*

# Pagination parameters
Disallow: /*?page=*
Disallow: /*?p=*
Disallow: /*?paged=*
Disallow: /*&page=*
Disallow: /*&p=*

# View parameters
Disallow: /*?view=*
Disallow: /*?display=*
Disallow: /*?layout=*

# Tracking parameters
Disallow: /*?utm_*
Disallow: /*&utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?campaign=*

# Session and cache parameters
Disallow: /*?session=*
Disallow: /*?sid=*
Disallow: /*?cache=*
Disallow: /*?v=*
Disallow: /*?ver=*
Disallow: /*?version=*

# === FILE TYPE RESTRICTIONS ===
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.xls$
Disallow: /*.xlsx$
Disallow: /*.ppt$
Disallow: /*.pptx$
Disallow: /*.zip$
Disallow: /*.rar$
Disallow: /*.tar$
Disallow: /*.gz$
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$

# === EXPLICITLY ALLOW IMPORTANT CONTENT ===
# Essential assets
Allow: /css/
Allow: /js/
Allow: /javascript/
Allow: /images/
Allow: /img/
Allow: /media/
Allow: /fonts/
Allow: /static/
Allow: /assets/
Allow: /uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /wp-includes/

# Important pages and sections
Allow: /products/
Allow: /product/
Allow: /categories/
Allow: /category/
Allow: /collections/
Allow: /collections/*
Allow: /brands/
Allow: /brand/
Allow: /sale/
Allow: /deals/
Allow: /offers/
Allow: /clearance/
Allow: /new-arrivals/
Allow: /blog/
Allow: /news/
Allow: /about/
Allow: /contact/
Allow: /shipping/
Allow: /delivery/
Allow: /size-guide/
Allow: /faq/
Allow: /privacy-policy/
Allow: /terms-conditions/
Allow: /cookie-policy/

# === SPECIFIC BOT CONFIGURATIONS ===

# Google bots - more permissive
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
Allow: /

# Bing bot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Facebook bot (for social sharing)
User-agent: facebookexternalhit
Allow: /

# Twitter bot (for social sharing)
User-agent: Twitterbot
Allow: /

# Pinterest bot
User-agent: Pinterest
Disallow: /

# SEO tool bots - allow but with slower crawling
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: MozBot
Allow: /
Crawl-delay: 10

# Block genuinely aggressive or low-value bots
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: YandexBot
Crawl-delay: 5

# === SITEMAP DECLARATIONS ===
Sitemap: https://fbargainsgalore.co.uk/sitemap.xml
Sitemap: https://fbargainsgalore.co.uk/sitemap_index.xml
Sitemap: https://fbargainsgalore.co.uk/product-sitemap.xml
Sitemap: https://fbargainsgalore.co.uk/category-sitemap.xml
Sitemap: https://fbargainsgalore.co.uk/post-sitemap.xml
Sitemap: https://fbargainsgalore.co.uk/page-sitemap.xml