# ========================================
# robots.txt for https://www.topregal.com
# Purpose: Efficient crawling control & SEO optimization
# ========================================

# ----------------------------------------
# General rules for all crawlers
# ----------------------------------------
User-agent: *
# Block internal directories
Disallow: /admin/
Disallow: /Core/
Disallow: /tmp/
Disallow: /views/
Disallow: /Setup/
Disallow: /log/

# Error logs
Disallow: /EXCEPTION_LOG.txt

# Sitemap location
Sitemap: https://www.topregal.com/sitemap.php

# ----------------------------------------
# Specific crawler rules
# ----------------------------------------

# Allow full access for internal audit bot
User-agent: sp_auditbot
Allow: /

# ----------------------------------------
# Parameter & wildcard blocking
# ----------------------------------------

# Block cl= parameters anywhere (e.g. ?cl=newsletter or &cl=basket)
Disallow: /*[?&]cl=newsletter
Disallow: /*[?&]cl=basket
Disallow: /*[?&]cl=moredetails
Disallow: /*[?&]cl=review
Disallow: /*[?&]cl=search

# Block fnc= function parameters (e.g. add to basket)
Disallow: /*[?&]fnc=tobasket
Disallow: /*[?&]fnc=tocomparelist
Disallow: /*[?&]addcompare=

# Block tracking or session parameters
Disallow: /*/sid/
Disallow: /*[?&]sid=
Disallow: /*/ldtype/
Disallow: /*[?&]ldtype=
Disallow: /*/sort/
Disallow: /*[?&]sort=
Disallow: /*/cnid/
Disallow: /*[?&]cnid=
Disallow: /*[?&]cdefnid=
Disallow: /*[?&]anid=
Disallow: /*[?&]srsltid=
Disallow: /*[?&]tag=

# ----------------------------------------
# Google Shopping support
# ----------------------------------------
User-agent: Googlebot-Product
Allow: /*lang=
Allow: /*gmc=
Allow: /*cur=

# ----------------------------------------
# Bing Shopping support
# ----------------------------------------
User-agent: msnbot-media
Allow: /*lang=
Allow: /*gmc=
Allow: /*cur=