[
{
"name": "Ahrefs Bot",
"re": "AhrefsBot",
"desc": "https://ahrefs.com/robot/",
"state": true
},
{
"name": "MJ12 bot",
"re": "MJ12bot",
"desc": "http://www.majestic12.co.uk/projects/dsearch/mj12bot.php",
"state": true
},
{
"name": "Roger Bot",
"re": "Rogerbot",
"desc": "http://moz.com/help/pro/rogerbot-crawler",
"state": true
},
{
"name": "Semrush Bot",
"re": "SemrushBot",
"desc": "http://www.semrush.com/bot.html",
"state": true
},
{
"name": "ia_archiver",
"re": "ia_archiver",
"desc": "http://archive.org/about/exclude.php",
"state": true
},
{
"name": "ScoutJet",
"re": "ScoutJet",
"desc": "http://scoutjet.com",
"state": true
},
{
"name": "sistrix",
"re": "sistrix",
"desc": "http://crawler.sistrix.net",
"state": true
},
{
"name": "SearchmetricsBot",
"re": "SearchmetricsBot",
"desc": "http://www.searchmetrics.com/en/searchmetrics-bot/",
"state": true
},
{
"name": "SEOkicks-Robot",
"re": "SEOkicks-Robot",
"desc": "http://www.seokicks.de/robot.html",
"state": true
},
{
"name": "Lipperhey Spider",
"re": "Lipperhey Spider",
"desc": "http://www.lipperhey.com/en/website-spider/",
"state": true
},
{
"name": "Exabot",
"re": "Exabot",
"desc": "http://www.exalead.com/search/webmasterguide",
"state": true
},
{
"name": "NC Bot",
"re": "NCBot",
"desc": "https://twitter.com/NetComber/status/334476871691550721",
"state": true
},
{
"name": "BacklinkCrawler",
"re": "BacklinkCrawler",
"desc": "http://www.backlinktest.com/crawler.html",
"state": true
},
{
"name": "archive.org Bot",
"re": "archive.org_bot",
"desc": "http://archive.org/details/archive.org_bot",
"state": true
},
{
"name": "MeanPath Bot",
"re": "meanpathbot",
"desc": "https://meanpath.com/meanpathbot.html",
"state": true
},
{
"name": "PagesInventory Bot",
"re": "PagesInventory",
"desc": "http://www.botsvsbrowsers.com/details/1002332/index.html",
"state": true
},
{
"name": "Aboundex Bot",
"re": "Aboundexbot",
"desc": "http://www.aboundex.com/crawler/",
"state": true
},
{
"name": "SeoProfiler Bot",
"re": "spbot",
"desc": "http://www.seoprofiler.com/bot/",
"state": true
},
{
"name": "Linkdex Bot",
"re": "linkdexbot",
"desc": "http://www.linkdex.com/about/bots/",
"state": true
},
{
"name": "Gigabot",
"re": "Gigabot",
"desc": "http://www.useragentstring.com/pages/Gigabot/",
"state": true
},
{
"name": "DotBot",
"re": "dotbot",
"desc": "http://en.wikipedia.org/wiki/DotBot",
"state": true
},
{
"name": "Nutch",
"re": "Nutch",
"desc": "http://nutch.apache.org/bot.html",
"state": true
},
{
"name": "BLEX Bot",
"re": "BLEXBot",
"desc": "http://webmeup-crawler.com/",
"state": true
},
{
"name": "Ezooms",
"re": "Ezooms",
"desc": "http://graphicline.co.za/blogs/what-is-ezooms-bot",
"state": true
},
{
"name": "Majestic 12",
"re": "Majestic-12",
"desc": "http://www.majestic12.co.uk/projects/dsearch/mj12bot.php",
"state": true
},
{
"name": "Majestic SEO",
"re": "Majestic-SEO",
"desc": "http://www.majestic12.co.uk/projects/dsearch/mj12bot.php",
"state": true
},
{
"name": "DSearch",
"re": "DSearch",
"desc": "http://www.majestic12.co.uk/projects/dsearch/mj12bot.php",
"state": true
},
{
"name": "MJ12",
"re": "MJ12",
"desc": "http://www.majestic12.co.uk/projects/dsearch/mj12bot.php",
"state": true
},
{
"name": "Blekko Bot",
"re": "BlekkoBot",
"desc": "http://blekko.com/about/blekkobot",
"state": true
},
{
"name": "Yandex",
"re": "Yandex",
"desc": "http://help.yandex.com/search/?id=1112030",
"state": false
},
{
"name": "Google Bot",
"re": "googlebot",
"desc": "https://support.google.com/webmasters/answer/182072?hl=en",
"state": false
},
{
"name": "Feedfetcher Google",
"re": "Feedfetcher-Google",
"desc": "https://support.google.com/webmasters/answer/178852",
"state": false
},
{
"name": "Bing Bot",
"re": "BingBot",
"desc": "http://en.wikipedia.org/wiki/Bingbot",
"state": false
},
{
"name": "Nerdy Bot",
"re": "NerdyBot",
"desc": "http://nerdybot.com/",
"state": true
},
{
"name": "James BOT",
"re": "JamesBOT",
"desc": "http://cognitiveseo.com/bot.html",
"state": true
},
{
"name": "Tin Eye",
"re": "TinEye",
"desc": "http://www.tineye.com/crawler.html",
"state": true
},
{
"state": true,
"re": "Baiduspider",
"name": "Baidu",
"desc": "http://www.baidu.com/search/robots_english.html"
},
{
"state": true,
"re": "serpstat",
"name": "Serpstat",
"desc": "https://serpstat.com/"
},
{
"state": true,
"desc": "https://www.spyfu.com/",
"re": "spyfu",
"name": "SpyFu"
}
]
