I have been dealing with a large amount of site scrapings lately and have tweaked my generic "Curl" function like mad so that it fits many situations. I wanted to share it on here and give back a little; Hopefully it helps some of you out. Please feel free to ask any questions. I have used this method to do the following:
- Login to sites using basic http authentication
- Login to sites using sessions and CSRF authentication
- Login to a site, switch pages and download a csv file which I then parsed
- many other things
so as you can see the function can be applied to many situations, it is just however you want to apply it.
You would simple put that method in your php project and then you can call it super simple like:
If you are looking for something a little more complex and would like to login to a site, pretend to be firefox on a mac, see debugging information and save the cookies:
** Note: this is just a Curl helper method that I have been tweaking to allow me to login to sites that I tell it to. I know that I login to site 'X' and always get content 'Y' so I use a tool TamperHeaders in firefox to tell me what fields are being submitted with the form and then put those as the post params. I have other functions that perform the processing, I just wanted to share this as it was a PITA learning it and figuring out how everything went together.
- Login to sites using basic http authentication
- Login to sites using sessions and CSRF authentication
- Login to a site, switch pages and download a csv file which I then parsed
- many other things
so as you can see the function can be applied to many situations, it is just however you want to apply it.
Code:
/**
*
* @param type $url
* @param array $params -> array('cookie_file'=>'path_to_cookie_file',
* 'start_new_cookie'=>false,
* 'cookie_jar'=>'cookie_file_name',
* 'user_agent'=>'the_user_agent',
* 'post_params'=>'var=1&var2=2',
* 'follow_location'=>false,
* 'http_referer'=>'specify a page that referred you',
* 'header'=>'header_array',
* 'debug'=>false,
* 'redirect_call_back'=>'function_to_call');
* @return array('html'=>'the html of the page that was retrived',
* 'info'=>'the header information of the page that was retrieved');
*/
private function getWebPage($url, $params=array()){
$return = array();
$ch = curl_init($url);
if(isset($params['cookie_file']) && $params['cookie_file'] != '')
{
// Forces a new Session
if(isset($params['start_new_cookie']) && $params['start_new_cookie'])
curl_setopt($ch, CURLOPT_COOKIESESSION, 1);
curl_setopt($ch, CURLOPT_COOKIEJAR, $params['cookie_file']);
curl_setopt($ch, CURLOPT_COOKIEFILE, $params['cookie_jar']);
}
if(isset($params['user_agent']))
curl_setopt($ch, CURLOPT_USERAGENT, $params['user_agent']);
if(isset($params['post_params']) && $params['post_params'] != '')
{
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $params['post_params']);
}
if(isset($params['header']))
{
curl_setopt($ch, CURLOPT_HTTPHEADER, $params['header']);
}
if(isset($params['debug']) && $params['debug'] == true)
{
curl_setopt($ch, CURLINFO_HEADER_OUT, true);
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
}
if(isset($params['http_referer']))
curl_setopt($ch, CURLOPT_REFERER, $params['http_referer']);
else
curl_setopt($ch, CURLOPT_AUTOREFERER, 1);
curl_setopt($ch, CURLOPT_TIMEOUT, 600);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
if(isset($params['follow_location']))
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, $params['follow_location']);
else
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false ) ;
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false ) ;
$html = curl_exec($ch);
$info = curl_getinfo($ch);
$http_code = $info['http_code'];
$return['html'] = $html;
$return['info'] = $info;
if(isset($params['debug']) && $params['debug'] == true)
{
echo "http_code: $http_code<br />";
echo 'Debug: <br /><pre>'.print_r($return['info'], true).'</pre><br />';
echo 'output: <br /><pre>'.htmlentities($return['html']).'</pre><br />';
echo '----------------------------------------<br />';
}
if(isset($info['redirect_url']) && strlen($info['redirect_url']) > 4 ) {
if(isset($params['redirect_call_back'])){ // callback
$redirectcallback = $params['redirect_call_back'];
$this->$redirectcallback($info['redirect_url'], $url);
}
$return = $this->getWebPage($info['redirect_url'], $params);
} else if(strpos($html, 'CONTENT="0;'))
{
// look for a meta refresh, if one exists, then redirect to it
$meta = substr($html, strpos($html, 'CONTENT="0;')+11);
$redirect_url = substr($meta, 0, strpos($meta, '"'));
$redirectcallback = $params['redirect_call_back'];
$this->$redirectcallback($redirect_url, $url);
echo "Meta Refresh found!<br />";
$return = $this->getWebPage($info['redirect_url'], $params);
}
return $return;
}You would simple put that method in your php project and then you can call it super simple like:
Code:
$results = $this->getWebPage('http://www.google.com');If you are looking for something a little more complex and would like to login to a site, pretend to be firefox on a mac, see debugging information and save the cookies:
Code:
$params = array('debug'=>true,
'post_params'=>'username=username&password=password',
'cookie_jar'=>'path/to/cookie/cookie.txt',
'cookie_file'=>'path/to/cookie/cookie.txt',
'user_agent'=>'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:17.0) Gecko/20100101 Firefox/17.0');
$results = $this->getWebPage('http://www.domain.com/login', $params);** Note: this is just a Curl helper method that I have been tweaking to allow me to login to sites that I tell it to. I know that I login to site 'X' and always get content 'Y' so I use a tool TamperHeaders in firefox to tell me what fields are being submitted with the form and then put those as the post params. I have other functions that perform the processing, I just wanted to share this as it was a PITA learning it and figuring out how everything went together.
