Disallow:

User-agent: AhrefsBot
Disallow:

User-agent: SemrushBot
Disallow: /

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block dotbot
User-agent: dotbot
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /

User-agent: *
Disallow: /index.php/
Disallow: /*?
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
Disallow: /catalogsearch/
Disallow: /search/*
Disallow: /result/*
Disallow: /catalogsearch/result/?
Disallow: /review/
Disallow: /*SID=
Disallow: /wishlist/
Crawl-delay: 10
Sitemap: https://www.robot-electronics.co.uk/sitemaps/sitemap.xml


Sitemap: https://www.robot-electronics.co.uk/pub/sitemaps/sitemap.xml