crawl-delay: 60 User-agent: * Disallow: /wp-* Disallow: /log/ Disallow: /lib/ Disallow: /upload/ Disallow: /search/ Disallow: /*/amp/ Disallow: /amp/ Disallow: /*.asp* Disallow: /asset/ Disallow: /sku/ Disallow: /cci* Disallow: /?s= Disallow: /cartoon/ Disallow: /topic/ Allow: /$ User-agent: IRLBot crawl-delay: 3600 User-agent: BLEXBot Disallow: / User-agent: Cliqzbot Disallow: / User-agent: Baiduspider Disallow: / User-agent: dotbot Disallow: / User-agent: BUbiNG Disallow: / User-agent: voltron Disallow: / User-agent: Yandex Disallow: / User-agent: SemrushBot Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: SemrushBot-BA Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: SemrushBot-SWA Disallow: / User-agent: SemrushBot-CT Disallow: / User-agent: SplitSignalBot Disallow: / User-agent: SemrushBot-COUB Disallow: / User-agent: PetalBot Disallow: /