botrockets
Regular Member
- Mar 16, 2013
- 389
- 647
This Instagram scraper scrapes the hashtag's top 1000+ images
I coded this in great F# language in just 20 min.
https://www.mediafire.com/file/qqxci2i8wh6zdi7/Instagram Hashtag scraper.rar
https://www.virustotal.com/#/file/8...4a3df112bc9664351a3d66f66ac9d086906/detection
Source Code:
I coded this in great F# language in just 20 min.
https://www.mediafire.com/file/qqxci2i8wh6zdi7/Instagram Hashtag scraper.rar
https://www.virustotal.com/#/file/8...4a3df112bc9664351a3d66f66ac9d086906/detection
Source Code:
Code:
let AsyncDownloadImage (url:string) (file:string) =
async{try
let random = new Random()
let! req = Http.AsyncRequest(url, headers=["User-Agent",userAgent;"Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";"Accept-Encoding","gzip, deflate"], customizeHttpRequest = (fun req->req.AutomaticDecompression<-DecompressionMethods.GZip|||DecompressionMethods.Deflate; req.Timeout<-20000; req))
match req.Body with |Binary bytes -> File.WriteAllBytes(file, bytes)|_->()
printfn "Downloaded: %s" file
do! Async.Sleep (random.Next(1000 ,2000))
with
|_ -> printfn "Couldn't download: %A" url}
let DownloadJson url = Http.RequestString(url, headers = [HttpRequestHeaders.UserAgent userAgent;"x-requested-with","XMLHttpRequest"])
let GetHashtagMedia (node:JsonValue)=
{
Id = node?id.AsInteger64()
IsVideo = node?is_video.AsBoolean()
Code = node?code.AsString()
DisplaySrc=node?display_src.AsString()
Date = node?date.AsInteger()
}
let GetAllHashtagMedia(root: HashtagSearch.Root) =
let nodes = root.Tag.Media.Nodes |> Array.map (fun n-> GetHashtagMedia (n.JsonValue))
let topNodes = root.Tag.TopPosts.Nodes |> Array.map (fun n-> GetHashtagMedia (n.JsonValue))
Seq.concat [|nodes; topNodes|]
let GetRoot url =
let json = DownloadJson url
let root = HashtagSearch.Parse(json)
root
let BuildQuery (encoded, next) = sprintf "https://www.instagram.com/explore/tags/%s/?__a=1&max_id=%s" encoded next
let DownloadMore limit (encodedKeyword:string) (nextPage:string) (arr:ResizeArray<HashtagMedia>) =
let mutable cnt = 2
let rec loop (next:string) (acc:ResizeArray<HashtagMedia>) =
match next with
|""->acc
|_ when (acc.Count > limit) -> acc
|next-> let r = (encodedKeyword, next) |>BuildQuery |>GetRoot
r |>GetAllHashtagMedia |>acc.AddRange
let np = if(r.Tag.Media.PageInfo.HasNextPage) then r.Tag.Media.PageInfo.EndCursor else ""
printfn "Downloading Page %d" cnt
cnt<-cnt+1
loop np acc
loop nextPage arr
|>Seq.toArray
let DownloadHashtagImages limit =
try
File.ReadAllLines("hashtag.txt")|>Array.filter(fun line -> not (String.IsNullOrWhiteSpace(line))) |>Array.truncate 1
|>Seq.map(fun keyword-> let encoded = WebUtility.UrlEncode(keyword)
keyword, encoded, (sprintf "https://www.instagram.com/explore/tags/%s/?__a=1" encoded))
|>Seq.iter (fun (k, encoded, url)->
ConsoleMessage ("Downloading Hashtag :"+k) ConsoleColor.Red
let root = GetRoot url
let arr = GetAllHashtagMedia root
let ra = new ResizeArray<HashtagMedia>(arr)
printfn "Downloading Page 1"
let allMedia = if(root.Tag.Media.PageInfo.HasNextPage) then
DownloadMore limit encoded (root.Tag.Media.PageInfo.EndCursor) ra
else
DownloadMore limit encoded "" ra
let kName = ("", k.Split(Path.GetInvalidPathChars()))|>String.Join
let dirName = Path.Combine(Environment.CurrentDirectory,"Download",kName)
Directory.CreateDirectory(dirName)|>ignore
allMedia
|>Array.iter
(fun media ->let url = media.DisplaySrc
let fn = Path.Combine(dirName, Path.GetFileName url)
AsyncDownloadImage url fn
|>Async.RunSynchronously|>ignore
)
)
with
|ex -> ConsoleMessage (ex.Message) ConsoleColor.Red
