# For all robots User-agent: * # Block access to specific groups of pages Disallow: /cart* Disallow: /checkout* Disallow: /my-account* Disallow: /my-company* Disallow: /search* Disallow: /account* Disallow: /register* Disallow: /wishlist* Disallow: /cart* Disallow: /payment* Disallow: /billing* Disallow: /shipping* Disallow: /order* Disallow: */Product-Variation Disallow: */Search-Show Disallow: */Search-UpdateGrid # ASP form Disallow: /athletesupport Disallow: */AthleteSupport* #to disallow Product-ShowQuickView Disallow: /Product-ShowQuickView #to disallow dwvar parameter Disallow: /*dwvar* # Allow search crawlers to discover the sitemap # Index page Sitemap: https://www.bellhelmets.com/sitemap_index.xml # Category pages Sitemap: https://www.bellhelmets.com/sitemap_1-category.xml # Product Pages Sitemap: https://www.bellhelmets.com/sitemap_0-product.xml # Block CazoodleBot as it does not present correct accept content headers User-agent: CazoodleBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / # Block dotbot as it cannot parse base urls properly User-agent: dotbot/1.0 Disallow: / # Block Gigabot User-agent: Gigabot Disallow: / #Block nova.collect.igodigital User-agent: * Disallow: /nova.collect.igodigital.com/*.*