# robots.txt for www.sinergy-store.com # IMPORTANT: Specific bot rules MUST come BEFORE User-agent: * # ======================================== # BLOCK AI CRAWLERS (FIRST - HIGH PRIORITY) # ======================================== # Block OpenAI (causing 16k+ requests/day on main site) User-agent: GPTBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: OpenAI-Collector Disallow: / # Block Anthropic AI User-agent: ClaudeBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Claude-Web Disallow: / # Block Google AI (keep Google Search) User-agent: Google-Extended Disallow: / User-agent: GoogleOther Disallow: / User-agent: GoogleOther-Image Disallow: / # Block Meta/Facebook AI User-agent: meta-externalagent Disallow: / User-agent: FacebookBot Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: Meta-External-Text Disallow: / User-agent: Meta-External-Image Disallow: / # Block Perplexity User-agent: PerplexityBot Disallow: / # Block Amazon AI User-agent: Amazonbot Disallow: / User-agent: Amazon-Ext Disallow: / # Block ByteDance User-agent: Bytespider Disallow: / User-agent: TikTokBot Disallow: / # Block CCBot (Common Crawl) User-agent: CCBot Disallow: / # Block other AI crawlers User-agent: cohere-ai Disallow: / User-agent: Omgilibot Disallow: / User-agent: Omgili Disallow: / # Block aggressive SEO crawlers User-agent: SemrushBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / # ======================================== # LEGITIMATE SEARCH ENGINES # ======================================== # Google Search (allowed) User-agent: Googlebot Crawl-delay: 2 Disallow: /*? Allow: /*?p= Disallow: /*?*=*&*=* Disallow: /index.php/ Disallow: /wishlist/ Disallow: /admin/ Disallow: /review/product/ Disallow: /sendfriend/ Disallow: /enable-cookies/ Disallow: /LICENSE.txt Disallow: /LICENSE.html Disallow: /directory/ Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /app/ Disallow: /bin/ Disallow: /dev/ Disallow: /lib/ Disallow: /phpserver/ Disallow: /pub/ Disallow: /tag/ Disallow: /review/ Disallow: *?SID= # Bing (allowed) User-agent: Bingbot Crawl-delay: 2 Disallow: /*? Allow: /*?p= Disallow: /*?*=*&*=* Disallow: /index.php/ Disallow: /admin/ Disallow: /checkout/ Disallow: /customer/ # Yahoo (allowed) User-agent: Slurp Crawl-delay: 2 Disallow: /admin/ Disallow: /checkout/ Disallow: /customer/ # DuckDuckGo (allowed) User-agent: DuckDuckBot Crawl-delay: 2 Disallow: /admin/ Disallow: /checkout/ Disallow: /customer/ # Baidu (allowed) User-agent: Baiduspider Crawl-delay: 5 Disallow: /admin/ Disallow: /checkout/ Disallow: /customer/ # Yandex (allowed) User-agent: YandexBot Crawl-delay: 3 Disallow: /admin/ Disallow: /checkout/ Disallow: /customer/ # ======================================== # DEFAULT RULES FOR ALL OTHER BOTS # ======================================== User-agent: * Crawl-delay: 2 # Stop crawling URL with parameters: Disallow: /*? # Allow pagination without parameters: Allow: /*?p= Disallow: /*?*=*&*=* # Stop crawling Magento pages: Disallow: /index.php/ Disallow: /wishlist/ Disallow: /admin/ Disallow: /review/product/ Disallow: /sendfriend/ Disallow: /enable-cookies/ Disallow: /LICENSE.txt Disallow: /LICENSE.html Disallow: /directory/ # Stop crawling user account and checkout pages by search engine robot: Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /customer/account/ Disallow: /customer/account/login/ # Blocking native catalog and search pages: Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ # Blocking CMS directories: Disallow: /app/ Disallow: /bin/ Disallow: /dev/ Disallow: /lib/ Disallow: /phpserver/ Disallow: /pub/ # Blocking duplicate content: Disallow: /tag/ Disallow: /review/ Disallow: *?SID= # ======================================== # SITEMAPS # ======================================== Sitemap: https://www.sinergy-store.com/sitemap_index.xml Sitemap: https://www.sinergy-store.com/sitemap_sinergy_cms.xml Sitemap: https://www.sinergy-store.com/sitemap_sinergy_product.xml Sitemap: https://www.sinergy-store.com/sitemap_sinergy_category.xml Sitemap: https://www.sinergy-store.com/sitemap_sinergy_stores.xml