# maximum rate is one page every 10 seconds Request-rate: 1/10s # 10 seconds between page requests Crawl-delay: 10 # only visit between 04:00 and 08:45 UTC Visit-time: 04:00-08:45 # Allow search crawlers to discover the sitemap Sitemap: https://www.premio.de/sitemap.xml # For all robots User-agent: * Disallow: /cart Disallow: /checkout Disallow: /my-account Disallow: /store-finder Disallow: /lkw-suche Disallow: /search Disallow: /offline-appointment Disallow: /*page= Disallow: /cart Disallow: /checkout Disallow: /my-account Disallow: /store-finder Disallow: /lkw-suche Disallow: /search Disallow: /showCase/ Disallow: /offline-appointment # Block CazoodleBot as it does not present correct accept content headers User-agent: CazoodleBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / # Block dotbot as it cannot parse base urls properly User-agent: dotbot/1.0 Disallow: / # Block Gigabot User-agent: Gigabot Disallow: / # Block AdsBot-Google (only works when specifically named) User-agent: AdsBot-Google Disallow: /cart Disallow: /checkout Disallow: /my-account Disallow: /store-finder Disallow: /lkw-suche Disallow: /search Disallow: /*page= Disallow: /cart Disallow: /checkout Disallow: /my-account Disallow: /store-finder Disallow: /lkw-suche Disallow: /search Disallow: /showCase/