php - curl multi, very slow when try get content - php

trying with curl_multi_init get info for page.
But when i try get info with curl_multi_getcontent() - page after 30s. down.
How i should correctly use curl_multi_getcontent()? Thanks
class Grab
{
public function getData()
{
$sessions = array('111', '222', '333', '444', '555');
$handle = curl_init();
foreach($sessions as $sId) {
$sessionId = $sId;
echo $sessionId.'<br/>';
$url = 'https://www.mypage.com?id='.$sessionId.'&test=1';
curl_setopt($handle, CURLOPT_URL, $url);
curl_setopt($handle, CURLOPT_RETURNTRANSFER, true);
curl_setopt($handle, CURLOPT_FRESH_CONNECT, false);
curl_setopt($handle, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($handle, CURLOPT_FAILONERROR, true);
$sResponse = $this->curlExecWithMulti($handle);
}
}
function curlExecWithMulti($handle) {
// In real life this is a class variable.
static $multi = NULL;
// Create a multi if necessary.
if (empty($multi)) {
$multi = curl_multi_init();
}
// Add the handle to be processed.
curl_multi_add_handle($multi, $handle);
// Do all the processing.
$active = NULL;
do {
$ret = curl_multi_exec($multi, $active);
} while ($ret == CURLM_CALL_MULTI_PERFORM);
while ($active && $ret == CURLM_OK) {
if (curl_multi_select($multi) != -1) {
do {
$mrc = curl_multi_exec($multi, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
**$res = curl_multi_getcontent($handle); // - very slow**
$this->printData($res);
// Remove the handle from the multi processor.
curl_multi_remove_handle($multi, $handle);
return TRUE;
}
public function printData($res)
{
$oPayment = json_decode($res);
var_dump($oPayment);
var_dump($errorno);
echo '<br/>---------------------<br/>';
}
}
$grab = new Grab;
$grab->getData();

You shouldn't call curlExecWithMulti in foreach loop for every $handle. You should create array of handles, add them by curl_multi_add_handle and only after that do all the processing (curl_multi_exec loop). When processing finish you can read all results in loop using curl_multi_getcontent.
It'll look like:
$handles = array();
foreach($sessions as $sId) {
$handle = curl_init();
$sessionId = $sId;
echo $sessionId.'<br/>';
$url = 'https://www.mypage.com?id='.$sessionId.'&test=1';
curl_setopt($handle, CURLOPT_URL, $url);
curl_setopt($handle, CURLOPT_RETURNTRANSFER, true);
curl_setopt($handle, CURLOPT_FRESH_CONNECT, false);
curl_setopt($handle, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($handle, CURLOPT_FAILONERROR, true);
$handles[] = $handle;
}
// calling curlExecWithMulti once, passing array of handles
// and got array of results
$sResponse = $this->curlExecWithMulti($handles);

Related

How to open multiple URLs with cURL without delay

I was wondering if it's possible to open multiple URLs with cURL or maybe something else.
I tried this until now.
$urls = array(
"http://google.com",
"http://youtube.com",
);
foreach($urls as $url) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT,0);
curl_setopt($ch, CURLOPT_TIMEOUT_MS, 200);
curl_exec($ch);
curl_close($ch);
}
The 200ms are there to let the site open fully.
Maybe you know any alternatives.
Is it possible to open multiple URLs in PHP at the same time? Not client sided, server side.
Your solution would be simultaneous cURL HTTP requests.
For faster implementation, you can use this function (thanks to phpied):
function multiRequest($data, $options = array()) {
// array of curl handles
$curly = array();
// data to be returned
$result = array();
// multi handle
$mh = curl_multi_init();
// loop through $data and create curl handles
// then add them to the multi-handle
foreach ($data as $id => $d) {
$curly[$id] = curl_init();
$url = (is_array($d) && !empty($d['url'])) ? $d['url'] : $d;
curl_setopt($curly[$id], CURLOPT_URL, $url);
curl_setopt($curly[$id], CURLOPT_HEADER, 0);
curl_setopt($curly[$id], CURLOPT_RETURNTRANSFER, 1);
// post?
if (is_array($d)) {
if (!empty($d['post'])) {
curl_setopt($curly[$id], CURLOPT_POST, 1);
curl_setopt($curly[$id], CURLOPT_POSTFIELDS, $d['post']);
}
}
// extra options?
if (!empty($options)) {
curl_setopt_array($curly[$id], $options);
}
curl_multi_add_handle($mh, $curly[$id]);
}
// execute the handles
$running = null;
do {
curl_multi_exec($mh, $running);
} while($running > 0);
// get content and remove handles
foreach($curly as $id => $c) {
$result[$id] = curl_multi_getcontent($c);
curl_multi_remove_handle($mh, $c);
}
// all done
curl_multi_close($mh);
return $result;
}
And use it like this:
$data = array(
'http://search.yahooapis.com/VideoSearchService/V1/videoSearch?appid=YahooDemo&query=Pearl+Jam&output=json',
'http://search.yahooapis.com/ImageSearchService/V1/imageSearch?appid=YahooDemo&query=Pearl+Jam&output=json',
'http://search.yahooapis.com/AudioSearchService/V1/artistSearch?appid=YahooDemo&artist=Pearl+Jam&output=json'
);
$r = multiRequest($data);
echo '<pre>';
print_r($r);
Hope it helps.
Also read this.

Codeigniter recursive function not returning value

I have issues with codeigniter recursive function which returns blank value. below is my function.
function recursive_data($url){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$output = curl_exec($ch);
curl_close($ch);
$instaArr = (array) json_decode($output);
// print_r($instaArr); // print value here
if( $instaArr['pagination'] == "" ) {
return $instaArr['data'];
} else {
return $this->recursive_data($instaArr['pagination']->next_url);
}
}
I am calling this above function in another function like this
$return_data = $this->recursive_data($url);
It is returning blank value. while it is printing the value in commented code print_r($instaArr)
please use it as bellow.
function recursive_data($url){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$output = curl_exec($ch);
curl_close($ch);
$instaArr = json_decode($output,true);
// print_r($instaArr); // print value here
if( $instaArr['pagination'] == "" ) {
return $instaArr['data'];
} else {
return $this->recursive_data($instaArr['pagination']->next_url);
}
}
What about this ?
function recursive_data($url)
{
$objCurlData = new stdClass();
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$output = curl_exec($ch);
curl_close($ch);
$instaArr = (array) json_decode($output);
// print_r($instaArr); // print value here
if( $instaArr['pagination'] == "" ) {
$objCurlData->data = $instaArr['data'];
} else {
$objCurlData->objChild = $this->recursive_data($instaArr['pagination']->next_url);
}
return $objCurlData;
}

How to combine multiple cURL requests in one?

This is the code I am currently using
function curl_get_contents($url) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_URL, $url);
$data = curl_exec($ch);
return $data;
}
function meta_scrap($filename, $other, $programming) {
$link = 'https://graph.facebook.com/?id=' . $filename . '&scrape=true&method=post';
$output = curl_get_contents($link);
$output = json_decode($output);
$ogtitle = $output->title;
}
I call meta_scrap($filename); 8 times on a single webpage. This makes the page load really slow. Is there something that I can do about it? I read about curl_multi_init() I tried to use it like this
function curl_get_contents($pages) {
$ch = curl_init();
$ch = array();
$mh = curl_multi_init();
for ($i = 0; $i < count($pages); $i++) {
$page = $pages[$i];
$ch[$i] = curl_init();
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_URL, $page);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_multi_add_handle($mh, $ch[$i]);
}
$running = 0;
do {
curl_multi_exec($mh, $running);
} while ($running > 0);
$data = curl_exec($ch);
$results = reset(json_decode(curl_multi_getcontent($ch[$i]), true));
$resultCount = count($results);
curl_close($ch);
return $data;
}
With this I get no output? Could anyone help me modify my code so that it gives correct output?
You seem to have started working with multiple requests then cut and pasted a single curl request handler here:
$data = curl_exec($ch);
$results = reset(json_decode(curl_multi_getcontent($ch[$i]), true));
$resultCount = count($results);
....but I can't imagine where you would have found piece of code which uses reset() like this.
You try to put the response into $results, yet you then throw this array away and return something completely different.
Try this....
...
$running = count($pages);
do {
curl_multi_exec($mh, $running);
usleep(5000);
} while ($running > 0);
$responses=array();
for ($i = 0; $i < count($pages); $i++) {
$responses[$i]=json_decode(curl_multi_getcontent($ch[$i]), true);
}
return $responses;
If you will be reusing the function then you should also remove and close each curl handle then close the multi-handle before returning.
See also my recent blog post about curl_multi_exec().

PHP curl questions

<?php
ini_set('display_errors',1);
$url = 'www.google.com.my';
$header = true;
$returntransfer = true;
$connecttimeout = 3;
$timeout = 60;
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, $header);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, $returntransfer);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
$execute = curl_exec($ch);
$info = curl_getinfo($ch);
header('Content-Type: text/plain');
echo $execute;
echo "\n\ncurl_getinfo() said:\n", str_repeat('-', 31 + strlen($url)), "\n";
foreach ($info as $label => $value)
{
printf("%-30s %s\n", $label, $value);
}
echo str_repeat('-', 31 + strlen($url));
?>
Here is my questions:
(1) I want to test the web services using PHP curl. Am I missing something based from the above code?
(2) If I have 2 URLs, should I use curl_setopt or curl_multi_init?
I really hope that someone will answer my questions.
It's a simultaneous requests:
$ch_1 = curl_init('http://url.one.com/');
$ch_2 = curl_init('http://url.two.com/');
curl_setopt($ch_1, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch_2, CURLOPT_RETURNTRANSFER, true);
// build the multi-curl handle, adding both $ch
$mh = curl_multi_init();
curl_multi_add_handle($mh, $ch_1);
curl_multi_add_handle($mh, $ch_2);
// execute all queries simultaneously, and continue when all are complete
$running = null;
do {
curl_multi_exec($mh, $running);
} while ($running);
// all of our requests are done, we can now access the results
$response_1 = curl_multi_getcontent($ch_1);
$response_2 = curl_multi_getcontent($ch_2);
echo "$response_1 $response_2"; // same output as first example

Curl Multi and Return Transfer

I need to make a number of curl requests to the same domain one after the other, but cannot make them in parallel.
I found the following code sample at http://technosophos.com/
which does work well in speeding up the repeated curl calls.
function get2($url) {
// Create a handle.
$handle = curl_init($url);
// Set options...
// Do the request.
$ret = curlExecWithMulti($handle);
// Do stuff with the results...
// Destroy the handle.
curl_close($handle);
}
function curlExecWithMulti($handle) {
// In real life this is a class variable.
static $multi = NULL;
// Create a multi if necessary.
if (empty($multi)) {
$multi = curl_multi_init();
}
// Add the handle to be processed.
curl_multi_add_handle($multi, $handle);
// Do all the processing.
$active = NULL;
do {
$ret = curl_multi_exec($multi, $active);
} while ($ret == CURLM_CALL_MULTI_PERFORM);
while ($active && $ret == CURLM_OK) {
if (curl_multi_select($multi) != -1) {
do {
$mrc = curl_multi_exec($multi, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
// Remove the handle from the multi processor.
curl_multi_remove_handle($multi, $handle);
return TRUE;
}
I have tried multiple times by setting the curl options to get function curlExecWithMulti($handle) to return the results of the curl as a variable, but with no success so far.
Can this be done?
Perhaps this will be of interest, very easy to understand. It will do your curl multi requests and then return an array of results, it also does curl POST.
<?php
//demo receiver
if($_SERVER['REQUEST_METHOD']=='POST'){
echo $_POST['post_var'];
die;
}
/**
* CURL GET|POST Multi
*/
function curl_multi($data, $options = array()) {
$curly = array();
$result = array();
$mh = curl_multi_init();
foreach ($data as $id=>$d) {
$curly[$id] = curl_init();
$url = (is_array($d) && !empty($d['url'])) ? $d['url'] : $d;
$header[0]="Accept: text/xml,application/xml,application/xhtml+xml,application/json";
$header[0].="text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
$header[]="Cache-Control: max-age=0";
$header[]="Connection: keep-alive";
$header[]="Keep-Alive: 2";
$header[]="Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7";
$header[]="Accept-Language: en-us,en;q=0.5";
$header[]="Pragma: ";
curl_setopt($curly[$id], CURLOPT_URL, $url);
curl_setopt($curly[$id], CURLOPT_HEADER, 0);
curl_setopt($curly[$id], CURLOPT_RETURNTRANSFER, true);
curl_setopt($curly[$id], CURLOPT_TIMEOUT, 30);
curl_setopt($curly[$id], CURLOPT_USERAGENT, "cURL (http://".$_SERVER['SERVER_NAME'].")");
curl_setopt($curly[$id], CURLOPT_HTTPHEADER, $header);
curl_setopt($curly[$id], CURLOPT_REFERER, $url);
curl_setopt($curly[$id], CURLOPT_ENCODING, 'gzip,deflate');
curl_setopt($curly[$id], CURLOPT_AUTOREFERER, true);
curl_setopt($curly[$id], CURLOPT_RETURNTRANSFER, true);
// post?
if (is_array($d)) {
if (!empty($d['post'])) {
curl_setopt($curly[$id], CURLOPT_POST, 1);
curl_setopt($curly[$id], CURLOPT_POSTFIELDS, $d['post']);
}
}
// extra options?
if (!empty($options)) {
curl_setopt_array($curly[$id], $options);
}
curl_multi_add_handle($mh, $curly[$id]);
}
$running = null;
do {
curl_multi_exec($mh, $running);
} while($running > 0);
foreach($curly as $id => $c) {
$result[$id] = curl_multi_getcontent($c);
curl_multi_remove_handle($mh, $c);
}
curl_multi_close($mh);
return $result;
}
$request = array(
array('url'=>'http://localhost:8080/testing.php','post'=>array('post_var'=>'a')),
array('url'=>'http://localhost:8080/testing.php','post'=>array('post_var'=>'b')),
array('url'=>'http://localhost:8080/testing.php','post'=>array('post_var'=>'c')),
);
$curl_result = curl_multi($request);
/*
Array
(
[0] => a
[1] => b
[2] => c
)
*/
echo '<pre>'.print_r($curl_result, true).'</pre>';
?>

Categories