User-agent: *
Disallow: /cache/


User-agent: *
Disallow: /public/

User-agent: Bytespidert
Disallow: /


User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Applebot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: RU_Bot
Disallow: /

User-agent: oBot
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: coccobot
Disallow: /

User-agent: LinkisBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: FaceBot
Disallow: /

User-agent: bitlybot
Disallow: /

User-agent: SMTBot
Disallow: /

User-agent: online-webceo-bot
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Twitterbot
Disallow: /

User-agent: Altimetribot
Disallow: /

User-agent: SerendeputyBot
Disallow: /