# Provided courtesy of http://browsers.garykeith.com.
# Created on April 30, 2008 at 1:27:09 PM GMT.
# http://browsers.garykeith.com/stream.asp?RobotsTXT
#
# Place this file in the root public folder of your website.
# It will stop the following bots from indexing your website.
#
User-agent: abot
User-agent: ALeadSoftbot
User-agent: BeijingCrawler
User-agent: BilgiBot
User-agent: bot
User-agent: botlist
User-agent: BOTW Spider
User-agent: bumblebee
User-agent: Bumblebee
User-agent: BuzzRankingBot
User-agent: Charlotte
User-agent: Clushbot
User-agent: Crawler
User-agent: CydralSpider
User-agent: DataFountains
User-agent: DiamondBot
User-agent: Dulance bot
User-agent: DYNAMIC
User-agent: EARTHCOM.info
User-agent: EDI
User-agent: envolk
User-agent: Exabot
User-agent: Exabot-Images
User-agent: Exabot-Test
User-agent: exactseek-pagereaper
User-agent: Exalead NG
User-agent: FANGCrawl
User-agent: Feed::Find
User-agent: flatlandbot
User-agent: Gigabot
User-agent: GigabotSiteSearch
User-agent: GurujiBot
User-agent: Hatena Antenna
User-agent: Hatena Bookmark
User-agent: Hatena RSS
User-agent: HatenaScreenshot
User-agent: Helix
User-agent: HiddenMarket
User-agent: HyperEstraier
User-agent: iaskspider
User-agent: IIITBOT
User-agent: InfociousBot
User-agent: iVia
User-agent: iVia Page Fetcher
User-agent: Jetbot
User-agent: Kolinka Forum Search
User-agent: KRetrieve
User-agent: LetsCrawl.com
User-agent: Lincoln State Web Browser
User-agent: Links4US-Crawler
User-agent: LOOQ
User-agent: Lsearch/sondeur
User-agent: MapoftheInternet.com
User-agent: NationalDirectory
User-agent: NetCarta_WebMapper
User-agent: NewsGator
User-agent: NextGenSearchBot
User-agent: ng
User-agent: nicebot
User-agent: NP
User-agent: NPBot
User-agent: Nudelsalat
User-agent: Nutch
User-agent: OmniExplorer_Bot
User-agent: OpenIntelligenceData
User-agent: Oracle Enterprise Search
User-agent: Pajaczek
User-agent: panscient.com
User-agent: PeerFactor 404 crawler
User-agent: PeerFactor Crawler
User-agent: PlantyNet
User-agent: PlantyNet_WebRobot
User-agent: plinki
User-agent: PMAFind
User-agent: Pogodak!
User-agent: QuickFinder Crawler
User-agent: Radiation Retriever
User-agent: Reaper
User-agent: RedCarpet
User-agent: ScorpionBot
User-agent: Scrubby
User-agent: Scumbot
User-agent: searchbot
User-agent: Seeker.lookseek.com
User-agent: SeznamBot
User-agent: ShowXML
User-agent: snap.com
User-agent: snap.com beta crawler
User-agent: Snapbot
User-agent: SnapPreviewBot
User-agent: sohu
User-agent: SpankBot
User-agent: Speedy Spider
User-agent: Speedy_Spider
User-agent: SpeedySpider
User-agent: spider
User-agent: SquigglebotBot
User-agent: SurveyBot
User-agent: SynapticSearch
User-agent: T-H-U-N-D-E-R-S-T-O-N-E
User-agent: Talkro Web-Shot
User-agent: Tarantula
User-agent: TerrawizBot
User-agent: TheInformant
User-agent: TMCrawler
User-agent: TridentSpider
User-agent: Tutorial Crawler
User-agent: Twiceler
User-agent: unwrapbot
User-agent: URI::Fetch
User-agent: VengaBot
User-agent: Vonna.com b o t
User-agent: Vortex
User-agent: Votay bot
User-agent: WebAlta Crawler
User-agent: Webbot
User-agent: Webclipping.com
User-agent: WebCorp
User-agent: Webinator
User-agent: WIRE
User-agent: WISEbot
User-agent: Xerka WebBot
User-agent: XSpider
User-agent: YodaoBot
User-agent: Yoono
User-agent: yoono

Disallow: /

# Now protect parts of the site from legit spiders

User-agent: *
SITEMAP: http://mckinneynews.net/site_map.xml.gz

Allow: /

Disallow: /admin/
Disallow: /Backups/
Disallow: /Connections/
Disallow: /cron/
Disallow: /include/
Disallow: /js/
Disallow: /lib/
Disallow: /mailtemplates/
Disallow: /skins/
Disallow: /SpryAssets/
Disallow: /textEditor/
Disallow: /test/
Disallow: /Uploads/

Disallow: /bug_report.php
Disallow: /cart.php
Disallow: /cart_2conf.php
Disallow: /cart_3conf.php
Disallow: /cart_4success.php
Disallow: /cart_process.php
Disallow: /cvv2.php
Disallow: /email_confirm.php
Disallow: /e-mail_confirm.php
Disallow: /faq.php
Disallow: /js_enable.php
Disallow: /logout.php
Disallow: /my_article_edit.php
Disallow: /my_article_new.php
Disallow: /my_articles.php
Disallow: /myaccount.php
Disallow: /myaccount_changePw.php
Disallow: /myaccount_resend_info.php
Disallow: /news_tou.php
Disallow: /obit_edit.php
Disallow: /obit_new.php
Disallow: /post_edit.php
Disallow: /post_new.php
Disallow: /posts.php
Disallow: /pp.php
Disallow: /register_confirm.php
Disallow: /search_old.php
Disallow: /signin.php
Disallow: /template.php
Disallow: /template_protected.php
Disallow: /test*.*
Disallow: /tou.php
Disallow: /worklog.txt

# Now protect against specific stories being crawled
Disallow: http://mckinneynews.net/news.php?nid=5570
Disallow: http://mckinneynews.net/news.php?nid=5570&cat=2