# Robots.txt for Business Directory # Updated for SEO and AI Optimization (AIO) # Default rules for all crawlers User-agent: * Disallow: /backoffice Disallow: /backoffice/* Disallow: /profile Disallow: /profile/* Disallow: /validate-g-auth Disallow: /report-review Disallow: /checkout/* Disallow: /signup Disallow: /signin Disallow: /update-password Disallow: /ajax/* Disallow: /pages/get-a-quote Disallow: /*?page=* Disallow: /*?q=* # Filter and sort combinations multiply every listing URL and carry nothing new Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= # Allow important paths for SEO Allow: /offices/* Allow: /business/* Allow: /articles/* Allow: /services/* # Crawl delay to prevent server overload Crawl-delay: 1 # Google specific rules User-agent: Googlebot Allow: / Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= # Bing specific rules User-agent: Bingbot Allow: / Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 2 # AI crawlers: listings, profiles and articles are welcome; paginated, searched and # filtered listing URLs are not. A named section replaces the default rules above for # that crawler, so each one repeats them - on 19 Nov 2026 a crawler walked # /{service}/{town}?page=N across the network and took every PHP worker. User-agent: GPTBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: ChatGPT-User Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: OAI-SearchBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: Claude-Web Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: ClaudeBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: Claude-User Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: Google-Extended Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: CCBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: PerplexityBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: Perplexity-User Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: Amazonbot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: Bytespider Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: meta-externalagent Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: meta-webindexer Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: KimiBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: MoonshotBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: GrokBot Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 User-agent: Hunyuan Allow: /offices/ Allow: /business/ Allow: /articles/ Allow: /services/ Disallow: /backoffice Disallow: /profile Disallow: /ajax/* Disallow: /checkout/* Disallow: /*?*page= Disallow: /*?*q= Disallow: /*?*sort= Disallow: /*?*rating= Disallow: /*?*verified= Disallow: /*?*premium= Crawl-delay: 5 # Block known bad bots User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / # Sitemap location (per-domain, absolute) Sitemap: https://www.eye-opticians.co.uk/sitemap_eye-opticians.xml