User-agent: *
Disallow: */are/cart
Disallow: */are/cart/*
Disallow: *_=*
Disallow: */*/c/brands
Disallow: *//c/
Disallow: */_s/saveFulfillmentJson
Disallow: */addBulkProductItemsToCart
Disallow: */addBulkProductItemsToCartReplacementModel
Disallow: */addObseleteReplacementProductItemsToCart
Disallow: */addProductsToCartFormConfigurator
Disallow: */addToCartFromUrl
Disallow: */addtoJoblist/*
Disallow: */c/.seo-collapse
Disallow: */**/c/browseCategoryAndBrandForMobile
Disallow: */checkout
Disallow: */checkout/*
Disallow: */closeNotificationIcon
Disallow: *collapseTwo
Disallow: */combined.css*
Disallow: */combined.js*
Disallow: */compare/
Disallow: */cookieCosent
Disallow: */cookieCosentToggler
Disallow: */enableCookieConsent
Disallow: */erp/*
Disallow: */findDefaultStore
Disallow: */findNearestStoreForAnonymous
Disallow: */footer/globalSites
Disallow: */GDPRCookieContainerComponentController/*
Disallow: */getBranchList*
Disallow: */getLimitedDistributionPickupZip
Disallow: */getNotificationMsgvaluesForLD
Disallow: */getPagablePickupBranchesForAnonymous.json
Disallow: */getRegionList
Disallow: */getReplacementForObsoleteproduct
Disallow: */getTieredPricingMap
Disallow: */isQuantitySectionVisible
Disallow: */login
Disallow: */login/*
Disallow: */logout
Disallow: */mergeCart/*
Disallow: */mobilelogin*
Disallow: */mobileprivacy
Disallow: */mobileRegistration*
Disallow: */mockMedia/*
Disallow: */my-account
Disallow: */my-account/*
Disallow: */myaccountpopup
Disallow: */my-company/*
Disallow: */mysettings
Disallow: */notification/*
Disallow: */obseleteProductAddToCart
Disallow: */order-return/*
Disallow: */p/getEmailPDPpopup
Disallow: */p/getTieredPricing
Disallow: */p/slangKeyword
## Disallow: */p/*/
Disallow: *page=-1
Disallow: *pageSize=*&
Disallow: */priceToolTip/*
Disallow: */product360view/*
Disallow: */productBranchStock
Disallow: */productBranchStock/*
Disallow: */quickorder
Disallow: */recommendedCategories.ajax
Disallow: */register
Disallow: */register/*
Disallow: */removeChangeRegionPopup
Disallow: */removeGDPRLegalNotification
Disallow: */removePCIComplianceNotification
Disallow: */replacementProduct
Disallow: */sales/*
Disallow: */search*
Disallow: */search/?*
Disallow: */updatelanguage
Disallow: */updateRegion
Disallow: */validateUpdateRegion
Disallow: */view/TabbedNavigationComponentController/updateTabIndex
Disallow: */zoomImages
Disallow: *{0}
Disallow: *%7B0%7D


# Limit PLP Facet Crawling 

# Disallow facets but those allowed elsewhere
Disallow: *%253Arelevance%253A*%253A*
Disallow: *%3Arelevance%3A*%3A*
Disallow: *%3A*%3A*
Allow: *%3A*%3A*/p/*
## disallow other
Disallow: */c/*__*
Disallow: */c/*/*productbranchstock*
Disallow: *page=-1*
## disallow facet removal
Disallow: */c/*?q=%3Arelevance&text=&filter=&aliasname=&partialSearch=#

### disallow category facet on category pages
Disallow: */category/*/c/*relevance%3Acategory*

#Disallow brand facet on brand pages
Disallow: */brands/*/c/*relevance%3Abrand*

# Allow / Disallow stock variations
Allow: *%253Arelevance%253AstockDisplayName%253AYes*
Allow: *%3Arelevance%253AstockDisplayName%253AYes*
Disallow: *%3Arelevance%3AstockDisplayName%3AYes%3A*
Disallow: *%3Arelevance%3AstockDisplayName%3ANo*

# Allow / Disallow Brand facets for category pages
Allow: */category/*/c/*%253Arelevance%253Abrand%253A*
Allow: */category/*/c/*%3Arelevance%3Abrand%3A*
Disallow: */category/*/c/*q=%3Arelevance%3Abrand%3A*%3A
Disallow: */category/*/c/*q=%3Arelevance%3Abrand%3A*page=-1*

# Allow / Disallow Category Facets for Brand pages
Allow: */brands/*/c/*%253Arelevance%253Acategory%253A*
Allow: */brands/*/c/*%3Arelevance%3Acategory%3A*
Disallow: */brands/*/c/*q=%3Arelevance%3Acategory%3A*%3A
Disallow: */brands/*/c/*q=%3Arelevance%3Acategory%3A*page=-1*


User-agent: Scrapy
Disallow: /

User-agent: CazoodleBot
Disallow: /
 
User-agent: MJ12bot
Disallow: /
 
User-agent: dotbot/1.2
Disallow: /
 
User-agent: Gigabot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MauiBot
Disallow: /

Sitemap: https://www.rexel.com.au/are/sitemap.xml

#v20201001