Asynchronous requests using proxy with Guzzle 7 - php

Im trying create a PHP script that uses proxy and asynchronous requests with Guzzle but Im not reaching the success.
On the Guzzle 7 docs, we can see this example below:
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Pool;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
$client = new Client();
$requests = function ($total) {
$uri = 'http://127.0.0.1:8126/guzzle-server/perf';
for ($i = 0; $i < $total; $i++) {
yield new Request('GET', $uri);
}
};
$pool = new Pool($client, $requests(100), [
'concurrency' => 5,
'fulfilled' => function (Response $response, $index) {
// this is delivered each successful response
},
'rejected' => function (RequestException $reason, $index) {
// this is delivered each failed request
},
]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
And with my low knowledge on Guzzle, I think that we can apply proxies only on Client not on Request class, so what I've to do to make it happens?
I've been created this example how it will works using simple cURL;
$mh = curl_multi_init();
$ch = [];
$headers = array(
'Host: example.com',
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language: en-US,en;q=0.5',
'DNT: 1',
'Connection: keep-alive'
);
$timeout = 7;
$chunked = array_chunk($proxies, 1000);
foreach($chunked as $proxies) {
foreach($proxies as $i => $proxy) {
$ch[$i] = curl_init();
curl_setopt($ch[$i], CURLOPT_AUTOREFERER, true);
curl_setopt($ch[$i], CURLOPT_COOKIESESSION, true);
curl_setopt($ch[$i], CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch[$i], CURLOPT_FRESH_CONNECT, true);
curl_setopt($ch[$i], CURLOPT_HTTPPROXYTUNNEL, true);
curl_setopt($ch[$i], CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch[$i], CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch[$i], CURLOPT_SSL_VERIFYSTATUS, false);
curl_setopt($ch[$i], CURLOPT_PROXY_SSL_VERIFYPEER, false);
curl_setopt($ch[$i], CURLOPT_CONNECTTIMEOUT_MS, $timeout * 1000);
curl_setopt($ch[$i], CURLOPT_MAXREDIRS, -1);
curl_setopt($ch[$i], CURLOPT_SOCKS5_AUTH, CURLAUTH_NONE);
curl_setopt($ch[$i], CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE);
curl_setopt($ch[$i], CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch[$i], CURLOPT_PROXY_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE);
curl_setopt($ch[$i], CURLOPT_PROXY_SSL_VERIFYHOST, 0);
curl_setopt($ch[$i], CURLOPT_TIMEOUT_MS, $timeout * 1000);
curl_setopt($ch[$i], CURLOPT_ENCODING, 'gzip, deflate, br');
curl_setopt($ch[$i], CURLOPT_PROXY, $proxy);
curl_setopt($ch[$i], CURLOPT_URL, 'https://example.com');
curl_setopt($ch[$i], CURLOPT_USERAGENT, 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0');
curl_setopt($ch[$i], CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch[$i], CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch[$i], CURLOPT_SSL_VERIFYHOST, 0);
curl_multi_add_handle($mh, $ch[$i]);
}
do {
curl_multi_exec($mh, $running);
curl_multi_select($mh);
} while ($running > 0);
foreach($ch as $key => $handle) {
$content = curl_multi_getcontent($handle);
if (strstr($content, 'Example')) {
file_put_contents('proxies.txt', $proxies[$key].PHP_EOL , FILE_APPEND | LOCK_EX);
}
curl_multi_remove_handle($mh, $ch[$key]);
}
curl_multi_close($mh);
}

Related

Check proxy before cURL

I need to check whether the selected $proxy is active or not and do a loop until a working one in proxy.txt is found before using it in a cURL. How could I do that? Below is a given proxy function.
<?php
function proxies()
{
$proxylist = file("proxy.txt");
$randomproxy = rand(0, sizeof($proxylist) - 1);
$proxylist = $proxylist[$randomproxy];
$words = explode(':', $proxylist);
return $words;
}
$proxy = proxies();
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'url');
curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
curl_setopt($ch, CURLOPT_PROXY, $proxy[0]);
curl_setopt($ch, CURLOPT_PROXYPORT, $proxy[1]);
curl_setopt($ch, CURLOPT_PROXYUSERPWD, "$username:$password");
....
?>
Thank you.
maybe you can ping, something like this
function proxies()
{
return file("proxy.txt");
}
foreach ($proxies() as $proxy) {
$item = explode(':', $proxy);
$waitTimeoutInSeconds = 1;
if ($fp = fsockopen($item[0], $item[1], $errCode, $errStr, $waitTimeoutInSeconds)){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'url');
curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
curl_setopt($ch, CURLOPT_PROXY, $proxy[0]);
curl_setopt($ch, CURLOPT_PROXYPORT, $proxy[1]);
curl_setopt($ch, CURLOPT_PROXYUSERPWD, "$username:$password");
break;
}
}
fclose($fp);

How to change socks5 if dead cURL

i have script to check something with socks all is ready but when its socks5 dead not change to try other socks from list, look below and please help anyone to fix that :
$ch = curl_init();
if($sock!=''){
curl_setopt($ch, CURLOPT_PROXYTYPE, 7);
curl_setopt($ch, CURLOPT_PROXY, $sock);
}
curl_setopt($ch, CURLOPT_TIMEOUT, 25);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_POSTFIELDS, $dattt);
//curl_setopt($ch, CURLOPT_HEADER, TRUE);
curl_setopt($ch, CURLOPT_HTTPHEADER, array(
Try something like this
<?php
$hosts = array(
'xx.com',
'yy.com'
);
class TimeoutException extends \Exception {
}
class Curl
{
public function connect($sock, $ch, $dattt, $url)
{
curl_setopt($ch, CURLOPT_PROXYTYPE, 7);
curl_setopt($ch, CURLOPT_PROXY, $sock);
curl_setopt($ch, CURLOPT_TIMEOUT, 25);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_POSTFIELDS, $dattt);
// etc
$result = curl_exec($ch);
// 28 is curl timeout
if (curl_errno($ch) === 28) {
throw new TimeoutException;
}
// some other error handling...
return $result;
}
}
$curl = new Curl;
foreach ($hosts as $socks) {
try {
$result = $curl->connect($socks, $curlResouece, $dattt, $url);
break;
} catch (TimeoutException $e) {
continue;
}
}
if (!isset($result)) {
die('Failed all socks');
}
echo $result;
As for an explanation, you're looping through the socks proxies you've specified. If the CURL error is 28, it'll throw a timeout exception which you catch in the try catch block. You can then continue to the next proxy. If is has a result, it'll break from the foreach and return the result straight away. If all of the socks proxies are unavailable, it'll die an error.

PHP Curl Multi post json data

I got stuck for 2 hours on this php curl multiple request
i wantta make post json data start from 1111(This is a start point and as a verificationCode) to 1121(end point 1111 + $process_count)
check this out guys :
<?php
$url = "https://api.mywebsite.com/myapp/customer/verification";
$mh = curl_multi_init();
$handles = array();
$process_count = 10;
for($c=1111;$c <= 1121;$c++){
$data_verification = array(
"phone" => "+6285643103039", // +6285643103039 9025
"verificationCode" => $c
);
$str_verification = json_encode($data_verification);
}
while ($process_count--)
{
$ch = curl_init($url);
$headers= array('Accept: application/json','Content-Type: application/json');
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_HEADER, FALSE);
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_TIMEOUT, 4000);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_POSTFIELDS,$str_verification);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
ob_start();
curl_multi_add_handle($mh, $ch);
$handles[] = $ch;
}
$running=null;
do
{
curl_multi_exec($mh, $running);
}
while ($running > 0);
for($i = 0; $i < count($handles); $i++)
{
$out = curl_multi_getcontent($handles[$i]);
echo "$i. ";
print $out . "\r\n";
echo "<br>";
curl_multi_remove_handle($mh, $handles[$i]);
}
curl_multi_close($mh);
?>
But
curl_setopt($ch, CURLOPT_POSTFIELDS,$str_verification); always given end point value 1121.
And doesn't looping
from 1111 to 1121.
Anyone can figure it out ? i'll glad for any help
You are doing a mistake in your first loop, you are erasing data every time into only one variable and not array
$str_verification = json_encode($data_verification);
Here is what I suggest you to do :
$str_verification = array();
for($c=1111;$c <= 1121;$c++){
$data_verification = array(
"phone" => "+6285643103039", // +6285643103039 9025
"verificationCode" => $c
);
$str_verification[] = json_encode($data_verification);
}
for ($i = 0; $i != 10; $i++)
{
$ch = curl_init($url);
$headers= array('Accept: application/json','Content-Type: application/json');
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_HEADER, FALSE);
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_TIMEOUT, 4000);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_POSTFIELDS,$str_verification[$i]);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
ob_start();
curl_multi_add_handle($mh, $ch);
$handles[] = $ch;
}

Random 500 errors in a multi CURL request

I am trying to get some hal+json data from a web service through a curl_multi to fill in a Bootstrap Typeahead. Every time I run this code, some of my requests in the curl_multi will be 500, and some will return the data I need. The ones that are 500 are completely random; the next time I load the page different queries will be 500 instead (And I get no errors). Why does that keep happening?
<?php
$curlurl = 'https://service.domain.com/'.$_POST['customer-name'].'/contact?callback=?';
$cuurl = singleRequest($curlurl);
global $contacts_typeahead_data;
$contacts_typeahead_data = array();
$clinks = $cuurl['_links']['https://service.domain.com/rel/contacts'];
for ($i=0; $i < count($clinks); $i++) {
$data[] = 'https://service.domain.com'.$clinks[$i]['href'].'?callback=?';
};
print_r($data);
$datar = multiRequest($data);
print_r($datar);
foreach($datar as $c){
$cs = json_decode($c, true);?>
<option value="<?php echo $cs['id']; ?>"><?php echo $cs['id'].' ('.$cs['info'][0]['name'].')'; ?></option>
<?php $contacts_typeahead_data[] = $cs['id'].' ('.$cs['info'][0]['name'].')';
}?>
And here is the code to the singleRequest and multiRequest, which is based on http://www.phpied.com/simultaneuos-http-requests-in-php-with-curl/:
function multiRequest($data, $options = array()) {
// array of curl handles
$curly = array();
// data to be returned
$result = array();
// multi handle
$mh = curl_multi_init();
// loop through $data and create curl handles
// then add them to the multi-handle
foreach ($data as $id => $d) {
$curly[$id] = curl_init();
$url = (is_array($d) && !empty($d['url'])) ? $d['url'] : $d;
$username = base64_encode('username');
$password2 = 'password';
$auth_token = $username . $password2 . 'QQ==';
curl_setopt($curly[$id], CURLOPT_URL, $url);
curl_setopt($curly[$id], CURLOPT_HEADER, 0);
curl_setopt($curly[$id], CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curly[$id], CURLOPT_USERPWD, "$username:$password2");
curl_setopt($curly[$id], CURLOPT_SSLVERSION,3);
curl_setopt($curly[$id], CURLOPT_HTTPHEADER, array(
'Accept: application/hal+json',
'Access-Control-Allow-Origin: *',
'Content-Type: application/hal+json',
'Authorization: Basic ' . $auth_token
));
curl_setopt($curly[$id], CURLOPT_HTTPAUTH, CURLAUTH_ANY);
curl_setopt($curly[$id], CURLOPT_CONNECTTIMEOUT, 0);
curl_setopt($curly[$id], CURLOPT_TIMEOUT, 30);
curl_setopt($curly[$id], CURLOPT_PORT, 443);
curl_setopt($curly[$id], CURLOPT_REFERER, $_SERVER['HTTP_REFERER']);
curl_setopt($curly[$id], CURLOPT_USERAGENT, 'Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)');
curl_setopt($curly[$id], CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($curly[$id], CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($curly[$id], CURLOPT_COOKIE, 'cert_url=https://service.domain.com/cert/yagowyefayygwflagliwygelifyaigwepifgpai');
// post?
if (is_array($d)) {
if (!empty($d['post'])) {
curl_setopt($curly[$id], CURLOPT_POST, 1);
curl_setopt($curly[$id], CURLOPT_POSTFIELDS, $d['post']);
}
}
// extra options?
if (!empty($options)) {
curl_setopt_array($curly[$id], $options);
}
curl_multi_add_handle($mh, $curly[$id]);
}
// execute the handles
$running = null;
do {
curl_multi_exec($mh, $running);
} while($running > 0);
// get content and remove handles
foreach($curly as $id => $c) {
// cURL error number
$curl_errno = curl_errno($c);
// cURL error message
$curl_error = curl_error($c);
// output if there was an error
if ($curl_error) {
echo " *** cURL error: (".$curl_errno.") ".$curl_error;
} else {
$result[$id] = curl_multi_getcontent($c);
}
curl_multi_remove_handle($mh, $c);
}
// all done
curl_multi_close($mh);
return $result;
}
function singleRequest($curlurl){
$username = base64_encode('username');
$password2 = 'password';
$auth_token = $username . $password2 . 'QQ==';
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_USERPWD, "$username:$password2");
curl_setopt($ch, CURLOPT_SSLVERSION,3);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_HTTPHEADER, array(
'Accept: application/hal+json',
'Access-Control-Allow-Origin: *',
'Content-Type: application/hal+json',
'Authorization: Basic ' . $auth_token
));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_COOKIE, 'cert_url=https://service.domain.com/cert/yagowyefayygwflagliwygelifyaigwepifgpai');
$result = curl_exec($ch);
if($result === false){
echo 'Curl error: ' . curl_error($ch). '<br><br>';
print_r(error_get_last());
}
$response = json_decode($result, true);
curl_close($ch);
return $response;
}
Edit Added error check
Why does that keep happening?
The server you send the request to makes this happen. It is just that requests can fail.
Deal with it and design for failure.

PHP function to connect to pingomatic using cURL

I'm creating a PHP function to connect to pingomatic using CURL but the response is always.
Array ( [EXE] => XML-RPC server accepts POST requests only. )
here is my sample code...
function curl_getpage2($url,$data, $referer = null, $agent = null, $header = null, $timeout = 20, $proxy = null, $proxy_username = null, $proxy_password = null) {
//getProxy();
if ($agent == null) {
$agent = getAgent();
}
if ($referer == null) {
$referer = getHost($url);
}
if (!is_array($header)) {
$header = array("Content-Type:text/xml","Host:".getHost($url),"User-Agent:$agent",
"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language:en-us,en;q=0.5",
"Accept-Encoding:gzip,deflate",
"Accept-Charset:ISO-8859-1,utf-8;q=0.7,*;q=0.7",
"Keep-Alive:300",
"Connection:keep-alive",
"Cache-Control:max-age=0",
"Content-length: ".strlen($XML));
}
if($proxy == null){
$proxy = "208.100.27.155:60099";
}
if($proxy_username == null && $proxy_password == null){
$proxyUnPW = "unproxy:pwproxy";
}else{
$proxyUnPW = $proxy_username.":".$proxy_password;
}
$ch = curl_init();
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
curl_setopt($ch, CURLOPT_HTTPHEADER,$header);
curl_setopt($ch, CURLOPT_PROXY, $proxy);
curl_setopt($ch, CURLOPT_PROXYUSERPWD,$proxyUnPW);
curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1);
//curl_setopt($ch, CURLOPT_USERAGENT, $agent);
//curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
//curl_setopt($ch, CURLOPT_REFERER, $referer);
//curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
//curl_setopt($ch, CURLOPT_AUTOREFERER,TRUE);
//curl_setopt($ch, CURLOPT_MAXREDIRS, 10);
//curl_setopt($ch, CURLOPT_COOKIEJAR, '/tmp/cookies.txt');
//curl_setopt($ch, CURLOPT_COOKIEFILE, '/tmp/cookies.txt');
//curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
//curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
$result['EXE'] = curl_exec($ch);
//$result['INF'] = curl_getinfo($ch);
//$result['ERR'] = curl_error($ch);
curl_close($ch);
return $result;
}
I need some help here. I'm just somewhat new to PHP.
You can submit your site once via the web form and then call a GET request for that url (see
question How to ping automatically to pingomatic in PHP?).
So your URL would be something like the following if you want to ping all services (title and url encoded with urlencode()):
$url = 'http://pingomatic.com/ping/?title='.$title.'&blogurl='.$url.'&rssurl=&chk_weblogscom=on&chk_blogs=on&chk_feedburner=on&chk_newsgator=on&chk_myyahoo=on&chk_pubsubcom=on&chk_blogdigger=on&chk_weblogalot=on&chk_newsisfree=on&chk_topicexchange=on&chk_google=on&chk_tailrank=on&chk_skygrid=on&chk_collecta=on&chk_superfeedr=on');
eg:
http://pingomatic.com/ping/?title=Example%20Title&blogurl=http%3A%2F%2Fwww.example.com%2F&rssurl=&chk_weblogscom=on&chk_blogs=on&chk_feedburner=on&chk_newsgator=on&chk_myyahoo=on&chk_pubsubcom=on&chk_blogdigger=on&chk_weblogalot=on&chk_newsisfree=on&chk_topicexchange=on&chk_google=on&chk_tailrank=on&chk_skygrid=on&chk_collecta=on&chk_superfeedr=on

Categories