User-agent: *

# Allow access to CSS and JS assets
Allow: /application/files/cache/css
Allow: /application/files/cache/js
Allow: /concrete/css
Allow: /concrete/js

Disallow: /application/attributes
Disallow: /application/authentication
Disallow: /application/bootstrap
Disallow: /application/config
Disallow: /application/controllers
Disallow: /application/elements
Disallow: /application/helpers
Disallow: /application/jobs
Disallow: /application/languages
Disallow: /application/mail
Disallow: /application/models
Disallow: /application/page_types
Disallow: /application/single_pages
Disallow: /application/tools
Disallow: /application/views
Disallow: /ccm/system/captcha/picture

Disallow: /application/files
Disallow: /application/files/*
Disallow: /download_file
Disallow: /download_file/*

Disallow: /concrete
Disallow: /concrete/*

Disallow: /demo
Disallow: /demo/*

# This list is compiled by Techie Zone part of Qlogix Network.
 
User-agent: Teleport
Disallow: /
 
User-agent: TeleportPro
Disallow: /
 
User-agent: EmailCollector
Disallow: /
 
User-agent: EmailSiphon
Disallow: /
 
User-agent: WebBandit
Disallow: /
 
User-agent: WebZIP
Disallow: /
 
User-agent: WebReaper
Disallow: /
 
User-agent: WebStripper
Disallow: /
 
User-agent: Web Downloader
Disallow: /
 
User-agent: WebCopier
Disallow: /
 
User-agent: Offline Explorer Pro
Disallow: /
 
User-agent: HTTrack Website Copier
Disallow: /
 
User-agent: Offline Commander
Disallow: /
 
User-agent: Leech
Disallow: /
 
User-agent: WebSnake
Disallow: /
 
User-agent: BlackWidow
Disallow: /
 
User-agent: HTTP Weazel
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: YandexBot
Disallow: /

User-Agent: DotBot
Disallow: /

User-Agent: SemrushBot
Disallow: /

User-Agent: LinkedInBot
Disallow: /

User-Agent: MauiBot (crawler.feedback+wc@gmail.com)
Disallow: /

User-Agent: SafeDNSBot (https://www.safedns.com/searchbot)
Disallow: /

User-Agent: Exabot
Disallow: /

User-Agent: Pingdom Bot
Disallow: /

#User-Agent: Bingbot
#Disallow: /

User-Agent: AdIdxBot
Disallow: /

#User-Agent: BingPreview
#Disallow: /

User-Agent: Adsbot
Disallow: /

User-Agent: AppleWebKit
Disallow: /

User-Agent: Adsbot-Google
Disallow: /

User-Agent: Googlebot-News
Disallow: /

User-Agent: Mediapartners-Google
Disallow: /

User-Agent: Baidu Spider 2.0
Disallow: /

User-Agent: SEMRush Crawler 2.0
Disallow: /

# Allow Twitter bot access to files
User-agent: Twitterbot

Allow: /application/files/thumbnails
Allow: /application/files/thumbnails/*