- Jun 30, 2014
- 13,739
- 5,217
I'm having trouble scraping Linkedin, it seems to always deny my requests (http code 999) with using c#. I've matched my headers exactly with chrome so I'm not sure how it knows.
I thought it was an IP thing but it will deny my residential IP through c# but then allow my chrome browser. Also I've tried loads of other IPs.
Something in my code below but not sure what....
I could always use selenium but I'd like it as fast as possible.
Any ideas?
I thought it was an IP thing but it will deny my residential IP through c# but then allow my chrome browser. Also I've tried loads of other IPs.
Something in my code below but not sure what....
I could always use selenium but I'd like it as fast as possible.
Any ideas?
Code:
ServicePointManager.DefaultConnectionLimit = 1000;
ServicePointManager.CheckCertificateRevocationList = false;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12 | SecurityProtocolType.Ssl3;
HttpClientHandler httpClientHandler = new HttpClientHandler()
{
// = new WebProxy(){ Address = new Uri($"http://*****13012"), UseDefaultCredentials = false }
/*Proxy = new WebProxy()
{
Address = new Uri($"http://*****:80"),
UseDefaultCredentials = false,
Credentials = new NetworkCredential(userName: "***", password: "***")
}*/
};
using (var client = new HttpClient(httpClientHandler, true))
{
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
client.DefaultRequestHeaders.Add("Connection", "keep-alive");
client.DefaultRequestHeaders.Add("Cache-Control", "max-age=0");
client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
client.DefaultRequestHeaders.Add("Referer", "https://www.google.com/");
client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
client.DefaultRequestHeaders.Add("Accept-Language", "en-GB,en-US;q=0.9,en;q=0.8");
try
{
Task<string> t = client.GetStringAsync("https://www.linkedin.com/company/money-co-uk");
string s = t.Result;
}
catch (Exception ex)
{
}
}
