1. This site uses cookies. By continuing to use this site, you are agreeing to our use of cookies. Learn More.

Instagram Hashtag scraper

Discussion in 'Black Hat SEO Tools' started by botrockets, Sep 6, 2017.

  1. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
    This Instagram scraper scrapes the hashtag's top 1000+ images

    I coded this in great F# language in just 20 min.

    https://www.mediafire.com/file/qqxci2i8wh6zdi7/Instagram Hashtag scraper.rar

    https://www.virustotal.com/#/file/8...4a3df112bc9664351a3d66f66ac9d086906/detection

    Source Code:

    Code:
    let AsyncDownloadImage (url:string) (file:string) =
        async{try
                 let random = new Random()
                 let! req = Http.AsyncRequest(url, headers=["User-Agent",userAgent;"Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";"Accept-Encoding","gzip, deflate"], customizeHttpRequest = (fun req->req.AutomaticDecompression<-DecompressionMethods.GZip|||DecompressionMethods.Deflate; req.Timeout<-20000; req))
                 match req.Body with |Binary bytes -> File.WriteAllBytes(file, bytes)|_->()
                 printfn "Downloaded: %s" file
                 do! Async.Sleep (random.Next(1000 ,2000))
    
              with
               |_ -> printfn "Couldn't download: %A" url}
    let DownloadJson url = Http.RequestString(url, headers = [HttpRequestHeaders.UserAgent userAgent;"x-requested-with","XMLHttpRequest"])
    let GetHashtagMedia (node:JsonValue)=
        {
          Id = node?id.AsInteger64()
          IsVideo = node?is_video.AsBoolean()
          Code = node?code.AsString()
          DisplaySrc=node?display_src.AsString()
          Date = node?date.AsInteger()
        }
    let GetAllHashtagMedia(root: HashtagSearch.Root) =
        let nodes = root.Tag.Media.Nodes |> Array.map (fun n-> GetHashtagMedia (n.JsonValue))
        let topNodes = root.Tag.TopPosts.Nodes |> Array.map (fun n-> GetHashtagMedia (n.JsonValue))
        Seq.concat [|nodes; topNodes|]
    let GetRoot url =
         let json = DownloadJson url
         let root = HashtagSearch.Parse(json)
         root
    let BuildQuery (encoded, next) = sprintf "https://www.instagram.com/explore/tags/%s/?__a=1&max_id=%s" encoded next
    let DownloadMore limit (encodedKeyword:string) (nextPage:string) (arr:ResizeArray<HashtagMedia>) =
      let mutable cnt = 2
      let rec loop (next:string) (acc:ResizeArray<HashtagMedia>) =
        match next with
        |""->acc
        |_ when (acc.Count > limit) -> acc
        |next-> let r = (encodedKeyword, next) |>BuildQuery |>GetRoot
                r |>GetAllHashtagMedia |>acc.AddRange
                let np = if(r.Tag.Media.PageInfo.HasNextPage) then r.Tag.Media.PageInfo.EndCursor else ""
                printfn "Downloading Page %d" cnt
                cnt<-cnt+1
                loop np acc
      loop nextPage arr
      |>Seq.toArray
    let DownloadHashtagImages limit =
        try
           File.ReadAllLines("hashtag.txt")|>Array.filter(fun line -> not (String.IsNullOrWhiteSpace(line))) |>Array.truncate 1
           |>Seq.map(fun keyword-> let encoded = WebUtility.UrlEncode(keyword)
                                   keyword, encoded, (sprintf "https://www.instagram.com/explore/tags/%s/?__a=1" encoded))
           |>Seq.iter (fun (k, encoded, url)->
                            ConsoleMessage ("Downloading Hashtag :"+k) ConsoleColor.Red
                            let root = GetRoot url   
                            let arr = GetAllHashtagMedia root
                            let ra = new ResizeArray<HashtagMedia>(arr)
                            printfn "Downloading Page 1"
                            let allMedia = if(root.Tag.Media.PageInfo.HasNextPage) then
                                              DownloadMore limit encoded (root.Tag.Media.PageInfo.EndCursor) ra                       
                                           else
                                              DownloadMore limit encoded "" ra   
                            let kName = ("", k.Split(Path.GetInvalidPathChars()))|>String.Join
                            let dirName = Path.Combine(Environment.CurrentDirectory,"Download",kName)
                            Directory.CreateDirectory(dirName)|>ignore
                            allMedia                       
                            |>Array.iter
                                  (fun media ->let url = media.DisplaySrc
                                               let fn = Path.Combine(dirName, Path.GetFileName url)
                                               AsyncDownloadImage url fn
                                               |>Async.RunSynchronously|>ignore
                                  )
    
                      )
        with
         |ex -> ConsoleMessage (ex.Message) ConsoleColor.Red
    
     
    • Thanks Thanks x 4
  2. chrispliakos

    chrispliakos Regular Member

    Joined:
    Jan 9, 2017
    Messages:
    239
    Likes Received:
    55
    Gender:
    Male
    Location:
    Greece
    Thanks but I cant get it to work on WIndows 7. It needs NetFrame 4.7 which is not supported by windows 7 and 8
     
  3. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
    ok I have compiled down to .NET Framework 4.6, Now it will work on windows 7 and 8
     
    • Thanks Thanks x 1
    Last edited: Sep 7, 2017
  4. HoNeYBiRD

    HoNeYBiRD Jr. VIP Jr. VIP

    Joined:
    May 1, 2009
    Messages:
    7,152
    Likes Received:
    8,146
    Gender:
    Male
    Occupation:
    Geographer, Tourism Manager
    Location:
    Ghosted
  5. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
    Both, top and general nodes of hashtag
     
    • Thanks Thanks x 1
  6. HoNeYBiRD

    HoNeYBiRD Jr. VIP Jr. VIP

    Joined:
    May 1, 2009
    Messages:
    7,152
    Likes Received:
    8,146
    Gender:
    Male
    Occupation:
    Geographer, Tourism Manager
    Location:
    Ghosted
  7. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
  8. madgenius

    madgenius Newbie

    Joined:
    Mar 8, 2010
    Messages:
    30
    Likes Received:
    5
    Very nice!! Rep'd!
     
  9. mcjoey

    mcjoey Newbie

    Joined:
    Sep 8, 2017
    Messages:
    1
    Likes Received:
    0
    Gender:
    Male
    HI. I like this tool even though I have not tried it yet. can I ask for this maker tool's email?
     
  10. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
    I made this tool for free for BHW. Its not a big deal.
     
    • Thanks Thanks x 1
  11. javadth

    javadth Junior Member

    Joined:
    Aug 27, 2012
    Messages:
    142
    Likes Received:
    10
    Gender:
    Male
    Home Page:
    put an stop button for this
     
  12. mechodnak

    mechodnak Newbie

    Joined:
    Aug 23, 2010
    Messages:
    5
    Likes Received:
    0
    Thanks for sharing this. havent tried it yet though but i would like to know is there any option on this that we can set to find TOP hashtags for Specific Language or country ?

    Thanks in advance
     
  13. HoNeYBiRD

    HoNeYBiRD Jr. VIP Jr. VIP

    Joined:
    May 1, 2009
    Messages:
    7,152
    Likes Received:
    8,146
    Gender:
    Male
    Occupation:
    Geographer, Tourism Manager
    Location:
    Ghosted
    You can just quit. It stops the downloading, when you quit and it saves all the pics until you quit.
     
  14. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
    Get Hashtag from this site https://top-hashtags.com/
    I may add that in future.
     
  15. javadth

    javadth Junior Member

    Joined:
    Aug 27, 2012
    Messages:
    142
    Likes Received:
    10
    Gender:
    Male
    Home Page:
    i am running on vps
     
  16. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
    It will finish downloading 1k pics in 30 min. or so.. so what's the problem? if you can use vps you can quit the program running on it...
     
  17. KartikJha

    KartikJha Junior Member

    Joined:
    Jun 1, 2017
    Messages:
    176
    Likes Received:
    40
    Gender:
    Male
    Occupation:
    Attacking Midfielder and Right Winger
    I am currently downloading, it is getting downloaded. Where will it get saved?
     
  18. KartikJha

    KartikJha Junior Member

    Joined:
    Jun 1, 2017
    Messages:
    176
    Likes Received:
    40
    Gender:
    Male
    Occupation:
    Attacking Midfielder and Right Winger
    To stop it, all you have to do is press "CTRL+C" whenever you want to!

    The operation will break. And whatever it downloaded, it will show you in a folder.

    Edit: My review: Pretty sleek and "awesome" tool :)
     
  19. botrockets

    botrockets Regular Member

    Joined:
    Mar 16, 2013
    Messages:
    339
    Likes Received:
    533
    Gender:
    Male
    Occupation:
    Software Developer
    Location:
    Mumbai, INDIA
    actually it's capable of downloading hundred thousands of images but I just put 1k images to prevent abuse and to stop getting ban from Instagram.
    Also I artificially added 1 to 2 seconds delay between download when in fact it can download those images in matter of seconds as I tried it here.
     
  20. KartikJha

    KartikJha Junior Member

    Joined:
    Jun 1, 2017
    Messages:
    176
    Likes Received:
    40
    Gender:
    Male
    Occupation:
    Attacking Midfielder and Right Winger
    Proxies?