Google scraper problem ..need help

rabies69

Registered Member
Joined
Mar 20, 2009
Messages
91
Reaction score
26
hello , look at this script to scrape google result


ini_set("max_execution_time", 0); set_time_limit(0); // no time-outs!if(isset($_GET['q']))
$query=$_GET['q'];
else
$query=$key;
if(isset($_GET['pages']))
$npages=$_GET['pages'];
else
$npages=10;
if(isset($_GET['start']))
$start=$_GET['start'];
else
$start=0;
if($npages>=100)
$npages=100;

$gg_url = 'http://www.google.com/search?hl=en&q=' . urlencode($query) . '&start=';
$i=1;
$size=0;
$options = array(
CURLOPT_PROXY => "193.138.185.51", //Proxy host if needed
CURLOPT_PROXYPORT => "3128", //Proxy port if needed
CURLOPT_RETURNTRANSFER => true, // return web page
CURLOPT_HEADER => false, // don't return headers
CURLOPT_FOLLOWLOCATION => true, // follow redirects
CURLOPT_ENCODING => "", // handle all encodings
CURLOPT_AUTOREFERER => true, // set referer on redirect
CURLOPT_CONNECTTIMEOUT => 120, // timeout on connect
CURLOPT_TIMEOUT => 120, // timeout on response
CURLOPT_MAXREDIRS => 10, // stop after 10 redirects
CURLOPT_COOKIEFILE => "cookie.txt",
CURLOPT_COOKIEJAR => "cookie.txt",
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3",
CURLOPT_REFERER => "http://www.google.com/",
);
for ($page = $start; $page < $npages; $page++)
{
$ch = curl_init($gg_url.$page.'0');
curl_setopt_array($ch,$options);
$scraped="";
$scraped.=curl_exec($ch);
curl_close( $ch );
$results = array();
preg_match_all('/a href="([^"]+)" class=l.+?>.+?<\/a>/',$scraped,$results);
foreach ($results[1] as $url)
{
echo "$url,";
$i++;
}
$size+=strlen($scraped);
}
it's not working with Proxy , when i removing bellow two lines it's working for few time , after that result showing blank for few hours ??

PHP:
 CURLOPT_PROXY   => "193.138.185.51", //Proxy host if needed
        CURLOPT_PROXYPORT  => "3128",   //Proxy port if needed

what is the problem ?? any other scraper available
 
Are you sure, that your proxy is working?

And google shows captcha when there is too many requests from one ip.
This could be why you are getting blank reslut.
 
using that you are allowed to see only 1000 results per ip, and will probably show captcha.
 
Here are a couple of CURL functions I made up a long time back that still work fine with cookies & proxies:
PHP:
<?php
// CURL Function Defaults
$curl_defaults = array(
    CURLOPT_HEADER => 0,
	CURLOPT_FOLLOWLOCATION => 1,
	CURLOPT_AUTOREFERER => 1,
	CURLOPT_RETURNTRANSFER => 1,
	CURLOPT_CONNECTTIMEOUT => $max_connect_timeout,
	CURLOPT_TIMEOUT => $max_timeout,
	CURLOPT_VERBOSE => 0,
	CURLOPT_SSL_VERIFYHOST => 0,
	CURLOPT_SSL_VERIFYPEER => 0
	);
function Return_Content_From_URL($url,$accountid){
	global $curl_defaults;
	$ch = curl_init();
	curl_setopt_array($ch, $curl_defaults);
	curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; ru; rv:1.9.2) Gecko/20100115 Firefox/3.6",);
	curl_setopt($ch, CURLOPT_URL,$url);
	if($referrer!=0){curl_setopt($ch, CURLOPT_REFERER, $referrer);}
    curl_setopt($ch, CURLOPT_PROXYPORT, $port);
	if ($proxytype=="SOCKS5"){curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);}else{curl_setopt($ch, CURLOPT_PROXYTYPE, "HTTP");}
	curl_setopt($ch, CURLOPT_PROXY, $proxy);
	if ($loginpassw!="0:0"){
		curl_setopt($ch, CURLOPT_PROXYUSERPWD, $loginpassw);
		}
	curl_setopt($ch, CURLOPT_COOKIEJAR, str_replace('\\','/',dirname(__FILE__)).'/cookies/'.$accountid.'.txt');
	curl_setopt($ch, CURLOPT_COOKIEFILE, str_replace('\\','/',dirname(__FILE__)).'/cookies/'.$accountid.'.txt');
	$html= curl_exec($ch);
	$err = 0;
	$err = curl_errno($ch);
	curl_close($ch);
	if ($err!=0){
		$curl_error = Echo_Curl_Error($err);
		if($silent==0){echo "<b>Error Connecting To Proxy With Account ID: $accountid & Proxy: $proxy. CURL Error: $err: ".$curl_error."</b><br />";}
		return(false);
		}
	return $html;
	}
function Echo_Curl_Error($err){
	$error[1] = "The URL you passed to libcurl used a protocol that this libcurl does not support.";
	$error[2] = "Very early initialization code failed. This is likely to be an internal error or problem.";
	$error[3] = "The URL was not properly formatted.";
	$error[5] = "Couldn't resolve proxy. The given proxy host could not be resolved. ";
	$error[6] = "Couldn't resolve host. The given remote host was not resolved.";
	$error[7] = "Failed to connect() to host or proxy.";
	$error[15] = "An internal failure to lookup the host used for the new connection.";
	$error[22] = "Connection Timeout.";
	$error[26] = "Failed creating formpost data.";
	$error[28] = "Operation timeout. The specified time-out period was reached according to the conditions.";
	$error[52] = "Nothing was returned from the server, and under the circumstances, getting nothing is considered an error.";
	$error[56] = "Failure with receiving network data.";
	$error[60] = "Peer certificate cannot be authenticated with known CA certificates.";
	return $error[$err];
	}
?>
 
Which type of proxy is it?

Try to change the GET url to some "what is my ip"- service and see which ip is returned.
 
Back
Top