I'm having trouble creating multiple xml requests using php's curl_multi_exec.
The problem is that the do...while loop containing the curl_multi_exec command runs only once and then quits.
Resources Used:
http://www.phpied.com/simultaneuos-http-requests-in-php-with-curl/
http://php.net/manual/en/function.curl-multi-exec.php/
http://www.rustyrazorblade.com/2008/02/curl_multi_exec/
Take a look at my code:
//Multi handle curl initialization
$mh = curl_multi_init();
//set url
$url = 'my_url';
foreach($latLng as $id => $l) {
$ch[$id] = curl_init();
//$request previously set
//Initialize and set options
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $request);
//add to multi_handle
curl_multi_add_handle($mh, $ch[$id]);
}
//Execute the handles
$running = null;
do {
$mrc = curl_multi_exec($mh, $running);
$ready=curl_multi_select($mh);
echo "Ran once\n";
} while ($mrc == CURLM_CALL_MULTI_PERFORM && $ready > 0);
while ($active && $mrc == CURLM_OK) {
if ($curl_multi_select($mh) != -1) {
do {
$mrc = curl_multi_exec($mh, $running);
echo "Ran again\n";
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
foreach ($mh as $c) {
// HTTP response code
$code = curl_getinfo($c, CURLINFO_HTTP_CODE);
// cURL error number
$curl_errno = curl_errno($c);
// cURL error message
$curl_error = curl_error($c);
// output if there was an error
if ($curl_error) {
echo("*** cURL error: ($curl_errno) $curl_error\n");
}
}
//get content and remove handles
foreach ($ch as $c) {
$result[] = curl_multi_getcontent($c);
curl_multi_remove_handle($mh, $c);
}
print_r($result);
//Close curl
curl_multi_close($mh);
}
I know the request is valid because I receive the correct return data when I perform a single curl execution. The problem lies with the curl_multi_exec().
The output I am receiving is "Ran once" followed by the empty arrays of the curl_multi_getcontent() calls. See below:
Ran once
Array
(
[0] =>
[1] =>
[2] =>
[3] =>
[4] =>
[5] =>
[6] =>
[7] =>
[8] =>
)
Any help is greatly appreciated.
You're not setting up the curl options correctly:
Currently, you're setting options on $ch which is your array, you need to be setting the options specifically on the current curl handler, which in your loop is $ch[$id]:
//Initialize and set options
curl_setopt($ch[$id], CURLOPT_URL, $url);
curl_setopt($ch[$id], CURLOPT_HEADER, 0);
curl_setopt($ch[$id], CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch[$id], CURLOPT_POST, 1);
curl_setopt($ch[$id], CURLOPT_POSTFIELDS, $request);
change this:
foreach ($mh as $c) {
$code = curl_getinfo($c, CURLINFO_HTTP_CODE);
to:
for($i=1;$i<=count($array);$i++){
$code = curl_multi_getcontent($ch[$i]);
assuming $array is the array for your multiple $url.
Related
I am fetching the data using the curl in php but as many data are present it returning the 0 output. I am providing my code below.
$result = array();
// multi handle
$mh = curl_multi_init();
$idArr=[2,147,92];
foreach ($idArr as $key => $value) {
$fetchURL = "http://example.com/index.php/rest/V1/categories/".$value."/products/";
//echo $fetchURL.'<br>';
$multiCurl[$key] = curl_init();
curl_setopt($multiCurl[$key], CURLOPT_URL,$fetch_url);
curl_setopt($multiCurl[$key], CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($multiCurl[$key], CURLOPT_HEADER,array("Content-Type: application/json", "Authorization: Bearer " . json_decode($token)));
curl_setopt($multiCurl[$key], CURLOPT_RETURNTRANSFER,true);
curl_multi_add_handle($mh, $multiCurl[$key]);
}
$index=null;
do {
curl_multi_exec($mh,$index);
} while($index > 0);
// get content and remove handles
foreach($multiCurl as $k => $ch) {
$result[$k] = curl_multi_getcontent($ch);
curl_multi_remove_handle($mh, $ch);
}
// close
curl_multi_close($mh);
print_r($result);
Here I have to pass the multiple request and get the result but in this case no result is coming. While I am using simple curl the result is coming. Here my requirement is to reduce the response time.
You are missing the implementation of "curl_multi_select". I have done it for you but not tested. Give it a go
$result = array();
// multi handle
$mh = curl_multi_init();
$multiCurl = array();
$idArr=[2,147,92];
foreach ($idArr as $key => $value) {
$fetchURL = "http://example.com/index.php/rest/V1/categories/".$value."/products/";
//echo $fetchURL.'<br>';
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL,$fetch_url);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_HEADER,array("Content-Type: application/json", "Authorization: Bearer " . json_decode($token)));
curl_setopt($ch, CURLOPT_RETURNTRANSFER,true);
$multiCurl[$key] = $ch;
curl_multi_add_handle($mh, $ch);
}
$active =null;
do {
$mrc = curl_multi_exec($mh,$active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active && $mrc == CURLM_OK) {
if (curl_multi_select($mh) != -1) {
do {
$mrc = curl_multi_exec($mh, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
foreach($multiCurl as $k => $ch) {
$result[$k] = curl_multi_getcontent($ch);
curl_multi_remove_handle($mh, $ch);
}
// close
curl_multi_close($mh);
print_r($result);
I think you can't get response due to set wrong variable in curl option set.Your third line in foreach loop should be like that
curl_setopt($multiCurl[$key], CURLOPT_URL,$fetchURL);
also you write wrong syntex for headers option replace CURLOPT_HEADER by CURLOPT_HTTPHEADER
here is sample code that works perfectly
$idArr=[20,18,21];
$mh = curl_multi_init();
$requests = array();
$curl_objs_arr = [];
foreach ($idArr as $key => $cat) {
$fetchURL = "http:example.com/v2/products?category=".$cat;
$requests[$key] = curl_init($fetchURL);
curl_setopt($requests[$key], CURLOPT_URL,$fetchURL);
curl_setopt($requests[$key], CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($requests[$key], CURLOPT_HTTPHEADER,array("Content-Type: application/json","Authorization: Bearer " . json_decode($token)));
curl_setopt($requests[$key], CURLOPT_RETURNTRANSFER,true);
curl_multi_add_handle($mh, $requests[$key]);
}
$running = null;
do {
curl_multi_exec($mh, $running);
} while($running > 0);
foreach ($requests as $key => $request) {
$result[$key] = curl_multi_getcontent($request);
curl_multi_remove_handle($mh, $request);
}
curl_multi_close($mh);
echo "<pre>";
print_r($result);exit;
set your code by this way, that's solve your problem.
I have the following function, running an array of URLs with cURL:
function multiRequest($urls){
$res = array();
// Create get requests for each URL
$mh = curl_multi_init();
foreach($urls as $i => $url)
{
$ch[$i] = curl_init($url);
curl_setopt($ch[$i], CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch[$i], CURLOPT_HEADER, "Accept: application/json");
curl_setopt($ch[$i], CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
curl_setopt ($ch[$i], CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt ($ch[$i], CURLOPT_SSL_VERIFYPEER, 0);
curl_multi_add_handle($mh, $ch[$i]);
}
// Start performing the request
do {
$execReturnValue = curl_multi_exec($mh, $runningHandles);
} while ($execReturnValue == CURLM_CALL_MULTI_PERFORM);
// Loop and continue processing the request
while ($runningHandles && $execReturnValue == CURLM_OK) {
// Wait forever for network
$numberReady = curl_multi_select($mh);
if ($numberReady != -1) {
// Pull in any new data, or at least handle timeouts
do {
$execReturnValue = curl_multi_exec($mh, $runningHandles);
} while ($execReturnValue == CURLM_CALL_MULTI_PERFORM);
}
}
// Check for any errors
if ($execReturnValue != CURLM_OK) {
trigger_error("Curl multi read error $execReturnValue\n", E_USER_WARNING);
}
// Extract the content
foreach($urls as $i => $url)
{
// Check for errors
$curlError = curl_error($ch[$i]);
if($curlError == "") {
$res[$i] = curl_multi_getcontent($ch[$i]);
} else {
print "Curl error on handle $i: $curlError\n";
}
// Remove and close the handle
curl_multi_remove_handle($mh, $ch[$i]);
curl_close($ch[$i]);
}
// Clean up the curl_multi handle
curl_multi_close($mh);
// Print the response data
return $res;
}
I get this error: Maximum execution time of 30 seconds exceeded, the line it points to is $numberReady = curl_multi_select($mh);
The code used to work on PHP version 5.3, after updating to 5.5 the code is no longer working.
I tried setting the limit to more than 30 seconds, it didn't help, also tried ini_set('max_execution_time', 0);
Is there anything wrong with my code? Is there another way to debug it?
This alternative function solved it for me:
function multiRequest($nodes){
//function multiple_threads_request($nodes){
$mh = curl_multi_init();
$curl_array = array();
foreach($nodes as $i => $url) {
$curl_array[$i] = curl_init($url);
curl_setopt($curl_array[$i], CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl_array[$i], CURLOPT_HEADER, "Accept: application/json");
curl_setopt($curl_array[$i], CURLOPT_USERPWD, xxxx);
curl_multi_add_handle($mh, $curl_array[$i]);
}
$running = NULL;
do {
usleep(10000);
curl_multi_exec($mh,$running);
} while($running > 0);
$res = array();
foreach($nodes as $i => $url) {
$res[$url] = curl_multi_getcontent($curl_array[$i]);
}
foreach($nodes as $i => $url) {
curl_multi_remove_handle($mh, $curl_array[$i]);
}
curl_multi_close($mh);
return $res;
}
For some reason curl_errno($value) always returns 0 instead of 6 when I try url like stkovrflow.com. This is a non existing domain. So curl supposed to return 6. But i'm getting 0.
Can someone tell me what's wrong with my code?
This is how I check curl error
if (curl_errno($value) !== 0)
{
$handles[$key]['error_code'] = curl_errno($value);
}
Here is my full code
<?php
protected function curl($url)
{
$mh = curl_multi_init();
$handles = array();
foreach ($url as $link)
{
$handles[$link] = curl_init($link);
curl_setopt($handles[$link], CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($handles[$link], CURLOPT_HEADER, true);
curl_setopt($handles[$link], CURLOPT_RETURNTRANSFER, true);
curl_setopt($handles[$link],CURLOPT_FAILONERROR,true);
curl_setopt($handles[$link], CURLOPT_FOLLOWLOCATION, $this->curlFollowLocation);
curl_setopt($handles[$link], CURLOPT_MAXREDIRS, $this->curlMaxRedirects);
curl_setopt($handles[$link], CURLOPT_TIMEOUT, $this->curlTimeout);
curl_setopt($handles[$link], CURLOPT_USERAGENT, $this->curlUserAgent);
curl_setopt($handles[$link], CURLOPT_AUTOREFERER, true);
curl_multi_add_handle($mh, $handles[$link]);
}
$running = null;
do {
curl_multi_exec($mh, $running);
usleep(200000);
} while ($running > 0);
foreach ($handles as $key => $value)
{
$handles[$key] = false;
$handles[$key]['error_code'] = false;
if (curl_errno($value) !== 0)
{
$handles[$key]['error_code'] = curl_errno($value);
} else {
$response = curl_multi_getcontent($value);
$httpCode = curl_getinfo($value, CURLINFO_HTTP_CODE);
if ( $httpCode != 200 ) {
$handles[$key]['error_code'] = $httpCode;
} else {
$handles[$key]['html'] = $response;
}
}
curl_multi_remove_handle($mh, $value);
curl_close($value);
}
curl_multi_close($mh);
return $handles;
}
Update:
Looks like curl_errno does not work (see bug report) in curl multi mode. Instead we should use curl_multi_info_read. When I use curl_multi_info_read like this
$e_code = curl_multi_info_read($mh);
var_dump($e_code);
this is my var_dump output.
array (size=3)
'msg' => int 1
'result' => int 6
'handle' => resource(7, curl)
As you can it returns 6 correctly. But php doc says
The data the returned resource points to will not survive calling curl_multi_remove_handle().
Unfortunately my script depends on curl_multi_remove_handle(). Any solution?
=======
i want to get several pages thru curl_exec, first page is come normally, but all others - 302 header, what reason?
$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, ROOT_URL);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
$content = curl_exec($curl); // here good content
curl_close($curl);
preg_match_all('/href="(\/users\/[^"]+)"[^>]+>\s*/i', $content, $p);
for ($j=0; $j<count($p[1]); $j++){
$new_curl = curl_init();
curl_setopt($new_curl, CURLOPT_URL, NEW_URL.$p[1][$j]);
curl_setopt($new_curl, CURLOPT_RETURNTRANSFER, 0);
$content = curl_exec($new_curl); // here 302
curl_close($new_curl);
preg_match('/[^#]+#[^"]+/i', $content, $p2);
}
smth like this
You probably want to provide a sample of your code so we can see if you're omitting something.
302 response code typically indicates that the server is redirecting you to a different location (found in the Location response header). Depending on what flags you use, CURL can either retrieve that automatically or you can watch for the 302 response and retrieve it yourself.
Here is how you would get CURL to follow the redirects (where $ch is the handle to your curl connection):
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);// allow redirects
You can use curl multi which is faster and can get data from all the url's in parallel.
You can use it like this
//Initialize
$curlOptions = array(CURLOPT_RETURNTRANSFER => 1);//Add whatever u additionally want.
$curlHandl1 = curl_init($url1);
curl_setopt_array($curlHandl1, $curlOptions);
$curlHandl2 = curl_init($url2);
curl_setopt_array($curlHandl2, $curlOptions);
$multi = curl_multi_init();
curl_multi_add_handle($multi, $curlHandle1);
curl_multi_add_handle($multi, $curlHandle2);
//Run Handles
$running = null;
do {
$status = curl_multi_exec($mh, $running);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($running && $status == CURLM_OK) {
if (curl_multi_select($mh) != -1) {
do {
$status = curl_multi_exec($mh, $running);
} while ($status == CURLM_CALL_MULTI_PERFORM);
}
}
//Retrieve Results
$response1 = curl_multi_getcontent($curlHandle1);
$status1 = curl_getinfo($curlHandle1);
$response1 = curl_multi_getcontent($curlHandle1);
$status1 = curl_getinfo($curlHandle1);
You can find more information here http://www.php.net/manual/en/function.curl-multi-exec.php
Checkout the Example1
I am trying to login to a site and then call numerous URLs to get the source and scrape for images. It works fine using regular curl but when I try to use multi_curl I am getting back the exact same response. So that I only have to login once I am resuing the curl resource (this works fine with regular curl) and I think this may be the reason why it is returning the same response.
Does anyone know how to use multi_curl but authenticate first?
Here is the code I am using:
<?php
// LICENSE: PUBLIC DOMAIN
// The author disclaims copyright to this source code.
// AUTHOR: Shailesh N. Humbad
// SOURCE: http://www.somacon.com/p539.php
// DATE: 6/4/2008
// index.php
// Run the parallel get and print the total time
$s = microtime(true);
// Define the URLs
$urls = array(
"http://localhost/r.php?echo=request1",
"http://localhost/r.php?echo=request2",
"http://localhost/r.php?echo=request3"
);
$pg = new ParallelGet($urls);
print "<br />total time: ".round(microtime(true) - $s, 4)." seconds";
// Class to run parallel GET requests and return the transfer
class ParallelGet
{
function __construct($urls)
{
// Create get requests for each URL
$mh = curl_multi_init();
$count = 0;
$ch = curl_init();
foreach($urls as $i => $url)
{
$count++;
if($count == 1)
{
// SET URL FOR THE POST FORM LOGIN
curl_setopt($ch, CURLOPT_URL, 'https://www.example.com/login.php');
// ENABLE HTTP POST
curl_setopt ($ch, CURLOPT_POST, 1);
// SET POST PARAMETERS : FORM VALUES FOR EACH FIELD
curl_setopt ($ch, CURLOPT_POSTFIELDS, 'user=myuser&password=mypassword');
// IMITATE CLASSIC BROWSER'S BEHAVIOUR : HANDLE COOKIES
curl_setopt ($ch, CURLOPT_COOKIEJAR, realpath($_SERVER['DOCUMENT_ROOT']) . '/cookie.txt');
# Setting CURLOPT_RETURNTRANSFER variable to 1 will force cURL
# not to print out the results of its query.
# Instead, it will return the results as a string return value
# from curl_exec() instead of the usual true/false.
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
// EXECUTE 1st REQUEST (FORM LOGIN)
curl_exec ($ch);
}
$ch = curl_init($url);
curl_setopt ($ch, CURLOPT_COOKIEFILE, realpath($_SERVER['DOCUMENT_ROOT']) . '/cookie.txt');
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
$ch_array[$i] = $ch;
curl_multi_add_handle($mh, $ch_array[$i]);
}
// Start performing the request
do {
$execReturnValue = curl_multi_exec($mh, $runningHandles);
} while ($execReturnValue == CURLM_CALL_MULTI_PERFORM);
// Loop and continue processing the request
while ($runningHandles && $execReturnValue == CURLM_OK) {
// Wait forever for network
$numberReady = curl_multi_select($mh);
if ($numberReady != -1) {
// Pull in any new data, or at least handle timeouts
do {
$execReturnValue = curl_multi_exec($mh, $runningHandles);
} while ($execReturnValue == CURLM_CALL_MULTI_PERFORM);
}
}
// Check for any errors
if ($execReturnValue != CURLM_OK) {
trigger_error("Curl multi read error $execReturnValue\n", E_USER_WARNING);
}
// Extract the content
foreach($urls as $i => $url)
{
// Check for errors
$curlError = curl_error($ch_array[$i]);
if($curlError == "") {
$res[$i] = curl_multi_getcontent($ch_array[$i]);
} else {
print "Curl error on handle $i: $curlError\n";
}
// Remove and close the handle
curl_multi_remove_handle($mh, $ch_array[$i]);
curl_close($ch_array[$i]);
}
// Clean up the curl_multi handle
curl_multi_close($mh);
// Print the response data
print_r($res);
}
}
?>
you need to enable/use cookies with curl as well. look for it on the documentation, don't forget to create the cookies (empty files) with read and write permission for curl.
$cookie = tempnam ("/tmp", "CURLCOOKIE");
$ch = curl_init();
curl_setopt( $ch, CURLOPT_URL, $url );
curl_setopt( $ch, CURLOPT_COOKIEJAR, $cookie );