How to Block Bots on PBNs

I was reading about how we can use htaccess to block bots, Previously i was using robots.txt.

I just made a code, can someone from you check and tell if something is wrong with this code. I think I'm suppose to add it to the beginning of any sites htaccess file.

The Code
RewriteCond %{HTTP_USER_AGENT} ^.*spbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BLEXBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Wget/1.6.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Wget/1.5.3.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*FairAd Client.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*URL_Spider_Pro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mozilla.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*turingos.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Python-urllib.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Teleport.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Stanford Comp Sci.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*grub-client.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit/3.50.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ia_archiver/1.6.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ProPowerBot/2.14.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*naver.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*RepoMonkey.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000).*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SpankBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebZip.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*moget/2.1.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*archive.org_bot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gigabot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Xenu's Link Sleuth 1.1c.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spanner.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NetMechanic.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CheeseBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Zeus 32297 Webster Pro V2.9 Win32.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Jetbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mozilla/4.0 (compatible; MSIE 4.0; Windows 98).*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sootle.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebCopier.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LNSpiderguy.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mozilla/4.0 (compatible; BullsEye; Windows 95).*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CopyRightCheck.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mozilla.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailWolf.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Keyword Density/0.9.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorld Extractor.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CherryPickerSE/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*moget.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*es.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Webster Pro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ExtractorPro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Microsoft URL Control - 5.01.4511.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Openfind .*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*exabot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Hatena Antenna.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*psbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*TheNomad.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*AhrefsBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*http://www.SearchEngineWorld.com bot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rogerbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*suzuran.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WWW-Collector-E.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailCollector.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*PerMan.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebAuto.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*GetRight/4.2.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mozilla/4.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*VCI WebViewer VCI WebViewer Win32.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Xenu's.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ia_archiver.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BackDoorBot/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Jetbot/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*humanlinks.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebEnhancer.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*DittoSpyder.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Harvest/1.5.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*TeleportPro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*RMA.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BecomeBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mozilla/3.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*scooter.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*True_Robot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Aqua_Products.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*b2w/0.1.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BotALot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorldForumBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LinkScan/8.1a Unix.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Enterprise_Search/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Copernic.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Flaming AttackBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cosmos.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NICErsPRO.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Openbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Microsoft URL Control.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EmailSiphon.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*grub.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Stanford CompClub.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Teoma.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Kenjin Spider.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Foobot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*True_Robot/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LexiBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Stanford.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CherryPickerElite/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MSIECrawler.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Oracle Ultra Search.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mozilla/5.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LinkWalker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Zeus Link Scout.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Stanford Spiderboys.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Alexibot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Crescent.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BlekkoBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mister PiX.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Website Quester.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*searchpreview.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*http://www.WebmasterWorld.com bot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Wget.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Crescent Internet ToolPak HTTP OLE Control v.1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Web Image Collector.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*lwp-trivial/1.34.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*CherryPicker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*QueryN Metasearch.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Nutch.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mata Hari.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dotbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Bookmark search tool.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*The Intraformant.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*toCrawl/UrlDispatcher.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Gaisbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*RepoMonkey Bait & Tackle/v1.01.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc/4.2.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Bullseye/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BuiltBotTough.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Enterprise_Search.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BlowFish/1.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*asterias.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*lwp-trivial.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dumbot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Iron33/1.0.2.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Zeus.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*JennyBot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*VCI.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*URL Control.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hloader.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mozilla/4.0 (compatible; MSIE 4.0; Windows 95).*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*EroCrawler.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BunnySlippers.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*libWeb/clsHTTP.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Radiation Retriever 1.1.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Microsoft URL Control - 6.00.8169.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Offline Explorer.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MJ12bot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Stanford CompSciClub.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*LinkextractorPro.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mozilla/4.0 (compatible; MSIE 4.0; Windows XP).*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Szukacz/1.4 .*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SiteSnagger.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ProWebWalker.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*NetAnts.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*InfoNaviRobot.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Telesoft.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebZip/4.0.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebSauger.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*httplib.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebStripper.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebVac.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*URLy Warning.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mozilla/4.0 (compatible; MSIE 4.0; Windows NT).*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Openfind data gathere.*$ [NC,OR]
RewriteCond %{HTTP_USER_AGENT} ^.*looksmart
RewriteRule ^.*.* https://www.google.com/ [R=301,L]

Why don't you just allow google, yahoo, bing and block everything else

BTW, anyone can spoof the useragent and use whatever useragent he wants too.. I use google bot user agent name all the time.

Hacking 101 : White List is always better Black List.

Oh wow do you really think all these rogue bots obey the robots.txt?

KHer0 said:

Why don't you just allow google, yahoo, bing and block everything else

BTW, anyone can spoof the useragent and use whatever useragent he wants too.. I use google bot user agent name all the time.

Hacking 101 : White List is always better Black List.

Click to expand...

Google uses all sorts of different IP's and agents. You should see the how complex the cloaker database I use is.

You're shooting yourself in the foot if you block everything except Google.
As soon as you accidentally block them your sites get flagged and will get deindexed.

This is a hell of a long htaccess file.