User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: PetalBot
User-agent: Owlin
User-agent: trendictionbot
User-agent: dotbot
User-agent: scooperbot
User-agent: Spinn3r
User-agent: twiceler
User-agent: sistrix
User-agent: wotbox
User-agent: Exabot
User-agent: WBSearchBot
User-agent: 008
User-agent: BLEXBot
User-agent: Yandex
User-agent: SeekportBot
User-agent: CCBot
User-Agent: ImagesiftBot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: AmazonAdBot
User-agent: SirdataBot
User-agent: MojeekBot
User-agent: LamarkBot
User-agent: GeedoBot
User-agent: SurdotlyBot
User-agent: DataForSeoBot
User-agent: PhxBot
User-agent: ZoominfoBot
User-agent: AwarioBot
User-agent: HoaxyBot
User-agent: LivelapBot
User-agent: TurnitinBot
Disallow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: *
Disallow: /sections.php
Disallow: /headliner.php
Disallow: /headliner2.php
Disallow: /follow.php
Disallow: /follow2.php
Disallow: /top.php
Disallow: /top2.php
Disallow: /file.php
Disallow: /redirect.php
Disallow: /redirect2.php