Hi, I'm trying to connect using the below code 1, it's working perfect and land me directly to the Ad page.
Unfortunately, it's not working dynamically, I mean I can't set TextBoxes for Login details as I don't see any login parameters within the code 1.
Whereas I see login parameters in Code 2, but Code 2 doesn't connect to the web-site, I mean it's land me in the login page only.
Both codes come from Fiddler, Please see Picture I uploaded.
- Fiddler Request 302 HTTPS is the one that provides me with all the login details but doesn't connect, it's only land me to the Login page.
- Fiddler Request 200 HTTPS is the one that doesn't provide me the login details (parameters) but connect to the web-site properly.
I don't know what is going on !!! Please can you help?
Please see below code 1
Please see below Code 2
Unfortunately, it's not working dynamically, I mean I can't set TextBoxes for Login details as I don't see any login parameters within the code 1.
Whereas I see login parameters in Code 2, but Code 2 doesn't connect to the web-site, I mean it's land me in the login page only.
Both codes come from Fiddler, Please see Picture I uploaded.
- Fiddler Request 302 HTTPS is the one that provides me with all the login details but doesn't connect, it's only land me to the Login page.
- Fiddler Request 200 HTTPS is the one that doesn't provide me the login details (parameters) but connect to the web-site properly.
I don't know what is going on !!! Please can you help?
Please see below code 1
Code:
Imports System.Net
Imports System.Text
Imports System.IO
Imports System.Diagnostics
Imports System.IO.Compression
Public Class Form1
'Calls request functions sequentially.
Private Sub MakeRequests()
Dim response As HttpWebResponse
Dim responseText As String
If Request_www_gumtree_co_za(response) Then
'Success, possibly use response.
responseText = ReadResponse(response)
WebBrowser1.DocumentText = responseText
response.Close()
Else
'Failure, cannot use response.
End If
End Sub
'Returns the text contained in the response. For example, the page HTML. Only handles the most common HTTP encodings.
Private Shared Function ReadResponse(response As HttpWebResponse) As String
Using responseStream = response.GetResponseStream()
Dim streamToRead As Stream = responseStream
If response.ContentEncoding.ToLower().Contains("gzip") Then
streamToRead = New GZipStream(streamToRead, CompressionMode.Decompress)
ElseIf response.ContentEncoding.ToLower().Contains("deflate") Then
streamToRead = New DeflateStream(streamToRead, CompressionMode.Decompress)
End If
Using streamReader = New StreamReader(streamToRead, Encoding.UTF8)
Return streamReader.ReadToEnd()
End Using
End Using
End Function
''' <summary>
''' Tries to request the URL: https://www.gumtree.co.za/my/ads.html
''' </summary>
''' <param name="response">After the function has finished, will possibly contain the response to the request.</param>
''' <returns>True if the request was successful; false otherwise.</returns>
Private Function Request_www_gumtree_co_za(ByRef response As HttpWebResponse) As Boolean
response = Nothing
Try
'Create request to URL.
Dim request As HttpWebRequest = DirectCast(WebRequest.Create("https://www.gumtree.co.za/my/ads.html"), HttpWebRequest)
'Set request headers.
request.KeepAlive = True
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0")
request.Headers.Add("Upgrade-Insecure-Requests", "1")
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
request.Referer = "https://www.gumtree.co.za/login.html"
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, sdch, br")
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8")
request.Headers.Set(HttpRequestHeader.Cookie, "__gads=ID=2d337c3aa450b3a4:T=1481559404:S=ALNI_MbMwbVKLlKKJujBYe7651DWnaaxXg; searchLocId=3100001; searchLocName=""Western Cape""; geoId=-33.2277918ng21.8568586; __utma=29192163.1986713265.1481559508.1481559510.1481559510.1; __utmz=29192163.1481559510.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); anonUsrId=ciwng6o1w31h11datg9abi4qd; __utmt_siteTracker=1; _gat=1; prevRedirect=/; machguid=a578e35f-62c6-48d1-89dc-28572996b545-158f3d24e21; __utma=29192163.1986713265.1481559508.1481559510.1481559510.1; __utmb=29192163.4.10.1481630003; __utmc=29192163; __utmz=29192163.1481559510.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _ga=GA1.3.1986713265.1481559508; _mkto_trk=id:038-AZF-323&token:_mch-gumtree.co.za-1481560048263-83891; _em_t=true; _em_vt=d0e7d5a4656b8d1802a9da8e161d584d7026d66b35-55904096584fe0e2; _em_v=813a6cda2a448b87343e5bd099c3584fe0ce79f670-21019122584fe0e2; testCookie=true; _gali=login-button; rememberMe=ZnJlZXNpeDAwN0BnbWFpbC5jb206MTQ4NDIyMTk0Mzg0NjphOGQ2MWMwMmVmMjBjNWI2YWMyY2YwZGQ5ZGI3NTQ0OQ; bt_auth=rO0ABXNyAD1vcmcuc3ByaW5nZnJhbWV3b3JrLnNlY3VyaXR5LmNvcmUuY29udGV4dC5TZWN1cml0eUNvbnRleHRJbXBsAAAAAAAAAZACAAFMAA5hdXRoZW50aWNhdGlvbnQAMkxvcmcvc3ByaW5nZnJhbWV3b3JrL3NlY3VyaXR5L2NvcmUvQXV0aGVudGljYXRpb247eHBzcgBPb3JnLnNwcmluZ2ZyYW1ld29yay5zZWN1cml0eS5hdXRoZW50aWNhdGlvbi5Vc2VybmFtZVBhc3N3b3JkQXV0aGVudGljYXRpb25Ub2tlbgAAAAAAAAGQAgACTAALY3JlZGVudGlhbHN0ABJMamF2YS9sYW5nL09iamVjdDtMAAlwcmluY2lwYWxxAH4ABHhyAEdvcmcuc3ByaW5nZnJhbWV3b3JrLnNlY3VyaXR5LmF1dGhlbnRpY2F0aW9uLkFic3RyYWN0QXV0aGVudGljYXRpb25Ub2tlbtOqKH5uR2QOAgADWgANYXV0aGVudGljYXRlZEwAC2F1dGhvcml0aWVzdAAWTGphdmEvdXRpbC9Db2xsZWN0aW9uO0wAB2RldGFpbHNxAH4ABHhwAXNyACZqYXZhLnV0aWwuQ29sbGVjdGlvbnMkVW5tb2RpZmlhYmxlTGlzdPwPJTG17I4QAgABTAAEbGlzdHQAEExqYXZhL3V0aWwvTGlzdDt4cgAsamF2YS51dGlsLkNvbGxlY3Rpb25zJFVubW9kaWZpYWJsZUNvbGxlY3Rpb24ZQgCAy173HgIAAUwAAWNxAH4ABnhwc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAB3BAAAAAB4cQB+AA1zcgBIb3JnLnNwcmluZ2ZyYW1ld29yay5zZWN1cml0eS53ZWIuYXV0aGVudGljYXRpb24uV2ViQXV0aGVudGljYXRpb25EZXRhaWxzAAAAAAAAAZACAAJMAA1yZW1vdGVBZGRyZXNzdAASTGphdmEvbGFuZy9TdHJpbmc7TAAJc2Vzc2lvbklkcQB+AA94cHQADTEwLjMyLjI0NC4xMzJwcHNyAEFjb20uZWJheS5lY2cuYm9sdC5wbGF0Zm9ybS5zaGFyZWQuY29tbW9uLnNlY3VyaXR5LkJvbHRVc2VyRGV0YWlscwvyYa0BfSwTAgACTAALZGlzcGxheU5hbWVxAH4AD0wACnVzZXJDb29raWV0AD5MY29tL2ViYXkvZWNnL2JvbHQvcGxhdGZvcm0vc2hhcmVkL2NvbW1vbi9zZWN1cml0eS9Vc2VyQ29va2llO3hyADJvcmcuc3ByaW5nZnJhbWV3b3JrLnNlY3VyaXR5LmNvcmUudXNlcmRldGFpbHMuVXNlcgAAAAAAAAGQAgAHWgARYWNjb3VudE5vbkV4cGlyZWRaABBhY2NvdW50Tm9uTG9ja2VkWgAVY3JlZGVudGlhbHNOb25FeHBpcmVkWgAHZW5hYmxlZEwAC2F1dGhvcml0aWVzdAAPTGphdmEvdXRpbC9TZXQ7TAAIcGFzc3dvcmRxAH4AD0wACHVzZXJuYW1lcQB+AA94cAEBAQFzcgAlamF2YS51dGlsLkNvbGxlY3Rpb25zJFVubW9kaWZpYWJsZVNldIAdktGPm4BVAgAAeHEAfgAKc3IAEWphdmEudXRpbC5UcmVlU2V03ZhQk5Xth1sDAAB4cHNyAEZvcmcuc3ByaW5nZnJhbWV3b3JrLnNlY3VyaXR5LmNvcmUudXNlcmRldGFpbHMuVXNlciRBdXRob3JpdHlDb21wYXJhdG9yAAAAAAAAAZACAAB4cHcEAAAAAHhwdAAUZnJlZXNpeDAwN0BnbWFpbC5jb210AAdGcmVlc2l4c3IAPGNvbS5lYmF5LmVjZy5ib2x0LnBsYXRmb3JtLnNoYXJlZC5jb21tb24uc2VjdXJpdHkuVXNlckNvb2tpZQAAAAAAAAABAgANWgAabG9nZ2VkSW5Vc2luZ1NvY2lhbFNlcnZpY2VaAApyZWdpc3RlcmVkTAAOZG91YmxlUG9zdGFibGV0ABNMamF2YS9sYW5nL0Jvb2xlYW47TAAFZW1haWxxAH4AD0wAE2ZhY2Vib29rQWNjZXNzVG9rZW5xAH4AD0wACmZhY2Vib29rSWRxAH4AD0wACWZpcnN0TmFtZXEAfgAPTAACaWR0ABBMamF2YS9sYW5nL0xvbmc7TAAFc3RhdGV0ADlMY29tL2ViYXkvZWNnL2JvbHQvcGxhdGZvcm0vc2hhcmVkL2VudGl0eS91c2VyL1VzZXJTdGF0ZTtMAAl0aW1lc3RhbXBxAH4AIUwABnVzZXJJZHQANkxjb20vZWJheS9lY2cvYm9sdC9wbGF0Zm9ybS9zaGFyZWQvZW50aXR5L3VzZXIvVXNlcklkO0wAE3VzZXJQcm9maWxlSW1hZ2VVcmxxAH4AD0wACHVzZXJuYW1lcQB+AA94cAABc3IAEWphdmEubGFuZy5Cb29sZWFuzSBygNWc+u4CAAFaAAV2YWx1ZXhwAHEAfgAddAAAcHEAfgAnc3IADmphdmEubGFuZy5Mb25nO4vkkMyPI98CAAFKAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAAGISI0cHNxAH4AKAAAAVj4Bsb+c3IANGNvbS5lYmF5LmVjZy5ib2x0LnBsYXRmb3JtLnNoYXJlZC5lbnRpdHkudXNlci5Vc2VySWQAAAAAAAAAAQIAAHhyADZjb20uZWJheS5lY2cuYm9sdC5wbGF0Zm9ybS5zaGFyZWQuZW50aXR5LkVudGl0eUlkTG9uZzEAAAAAAAAAAQIAAHhyADxjb20uZWJheS5lY2cuYm9sdC5wbGF0Zm9ybS5zaGFyZWQuZW50aXR5LkVudGl0eUlkQ29tcGFyYWJsZTEAAAAAAAAAAQIAAkwAB2VuY29kZWRxAH4AD0wAEnByaW1hcnlBbmRTaGFyZEtleXQAFkxqYXZhL2xhbmcvQ29tcGFyYWJsZTt4cHQADjEwMTEwMjgzNDc0MDA5cQB+ACpxAH4AJ3EAfgAe__v1__D90496F35184EE12AFD7; bt_logged_in=true")
'Get response to request.
response = DirectCast(request.GetResponse(), HttpWebResponse)
Catch e As WebException
'ProtocolError indicates a valid HTTP response, but with a non-200 status code (e.g. 304 Not Modified, 404 Not Found)
If e.Status = WebExceptionStatus.ProtocolError Then
response = DirectCast(e.Response, HttpWebResponse)
Else
Return False
End If
Catch e As Exception
If response IsNot Nothing Then
response.Close()
End If
Return False
End Try
Return True
End Function
Private Sub Button2_Click(sender As Object, e As EventArgs) Handles Button2.Click
MakeRequests()
End Sub
End Class
Please see below Code 2
Code:
Imports System.Net
Imports System.Text
Imports System.IO
Imports System.Diagnostics
Imports System.IO.Compression
Public Class Form1
'Calls request functions sequentially.
Private Sub MakeRequests()
Dim response As HttpWebResponse
Dim responseText As String
If Request_www_gumtree_co_za(response) Then
'Success, possibly use response.
responseText = ReadResponse(response)
WebBrowser1.DocumentText = responseText
response.Close()
Else
'Failure, cannot use response.
End If
End Sub
'Returns the text contained in the response. For example, the page HTML. Only handles the most common HTTP encodings.
Private Shared Function ReadResponse(response As HttpWebResponse) As String
Using responseStream = response.GetResponseStream()
Dim streamToRead As Stream = responseStream
If response.ContentEncoding.ToLower().Contains("gzip") Then
streamToRead = New GZipStream(streamToRead, CompressionMode.Decompress)
ElseIf response.ContentEncoding.ToLower().Contains("deflate") Then
streamToRead = New DeflateStream(streamToRead, CompressionMode.Decompress)
End If
Using streamReader = New StreamReader(streamToRead, Encoding.UTF8)
Return streamReader.ReadToEnd()
End Using
End Using
End Function
''' <summary>
''' Tries to request the URL: https://www.gumtree.co.za/login
''' </summary>
''' <param name="response">After the function has finished, will possibly contain the response to the request.</param>
''' <returns>True if the request was successful; false otherwise.</returns>
Private Function Request_www_gumtree_co_za(ByRef response As HttpWebResponse) As Boolean
response = Nothing
Try
'Create request to URL.
Dim request As HttpWebRequest = DirectCast(WebRequest.Create("https://www.gumtree.co.za/my/ads.html"), HttpWebRequest)
'Set request headers.
request.KeepAlive = True
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0")
'request.Headers.Add("Origin", "https://www.gumtree.co.za")
request.Headers.Add("Upgrade-Insecure-Requests", "1")
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"
request.ContentType = "application/x-www-form-urlencoded"
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
request.Referer = "redirect=https%3A%2F%2Fwww.gumtree.co.za%2Fmy%2Fads.html&email=ariesio994%40gmail.com&password=dfge871&rememberMe=on"
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br")
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8")
request.Headers.Set(HttpRequestHeader.Cookie, "__gads=ID=b4b222877d192129:T=1481884237:S=ALNI_MbSogYVWSsp9TGS9GUmN3Fy1R-cuA; anonUsrId=ciww90h2r3fl63paunzorjkta; __utma=29192163.977768111.1481884340.1482147707.1482162148.7; __utmc=29192163; __utmz=29192163.1482147707.6.4.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); searchLocId=3100001; searchLocName=""Western Cape""; geoId=-33.2277918ng21.8568586; __utmt_siteTracker=1; _gat=1; prevRedirect=/; machguid=a7c56226-dab8-4234-8fbe-273eb2835a17-159072ef322; __utma=29192163.977768111.1481884340.1482147707.1482162148.7; __utmb=29192163.17.10.1482163373734; __utmc=29192163; __utmz=29192163.1482147707.6.4.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); _ga=GA1.3.977768111.1481884340; _mkto_trk=id:038-AZF-323&token:_mch-gumtree.co.za-1482163886680-68119; _em_t=true; _em_vt=c9befb9497d3270b52c44af2864b5853ad9da62b14-3709727658580ebe; _em_v=1b8bcdeede5ab9005b2ca194de4d5857ff85114b64-1046673958580ebe; testCookie=true; _gali=login-button")
'Set request method
request.Method = "POST"
' Disable 'Expect: 100-continue' behavior. More info: http://haacked.com/archive/2004/05/15/http-web-request-expect-100-continue.aspx
request.ServicePoint.Expect100Continue = False
'Set request body.
Dim body As String = "redirect=https%3A%2F%2Fwww.gumtree.co.za%2Fmy%2Fads.html&email=hjghgdfs878%40gmail.com&password=poryty14&rememberMe=on"
Dim postBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(body)
request.ContentLength = postBytes.Length
Dim stream As Stream = request.GetRequestStream()
stream.Write(postBytes, 0, postBytes.Length)
stream.Close()
'Get response to request.
response = DirectCast(request.GetResponse(), HttpWebResponse)
Catch e As WebException
'ProtocolError indicates a valid HTTP response, but with a non-200 status code (e.g. 304 Not Modified, 404 Not Found)
If e.Status = WebExceptionStatus.ProtocolError Then
response = DirectCast(e.Response, HttpWebResponse)
Else
Return False
End If
Catch e As Exception
If response IsNot Nothing Then
response.Close()
End If
Return False
End Try
Return True
End Function
Private Sub Button2_Click(sender As Object, e As EventArgs) Handles Button2.Click
MakeRequests()
End Sub
End Class