I have a bunch of curl statements in my app controller file. How can I possibly combine them all into one curl statement. Is it even possible. Thanks
// used for curl and curl_post
curl_setopt($ch, CURLOPT_USERAGENT,'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13');
// used for curl_twitter
curl_setopt($ch, CURLOPT_USERAGENT,'spider');
// used for curl_post, curl_auth
curl_setopt($ch,CURLOPT_POST, count($post_data_count));
curl_setopt($ch,CURLOPT_POSTFIELDS, $post_data_string);
// used for curl_twitter
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
curl_setopt($ch, CURLOPT_TIMEOUT, 20);C
If you use the same curl statement evey time you should create a function and juste pass by parameter the wanted change.
function curlPerform($url, $postfields, $useragent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13'){
$ch = curl_init($url);
// used for curl and curl_post
curl_setopt($ch, CURLOPT_USERAGENT, $useragent);
// used for curl_post, curl_auth
curl_setopt($ch,CURLOPT_POST, count($postfields));
curl_setopt($ch,CURLOPT_POSTFIELDS, $postfields);
// used for curl_twitter
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
curl_setopt($ch, CURLOPT_TIMEOUT, 20);
$reponse = curl_exec($ch);
curl_close($ch);
return $reponse;
}
$reponse = curlPerform($yourUrl, $yourPostFields); // Each time that you want to run a curl request just call this function DRY
Or use curl_setopt_array
$ch = curl_init();
$options = array(CURLOPT_URL => 'http://www.example.com/',
CURLOPT_HEADER => false
);
curl_setopt_array($ch, $options);
curl_exec($ch);
curl_close($ch);
Related
I am trying to find where I'll be redirected at. So I tried to functions for this, but none of those are working properly.
the links is here. when you try to enter, you will be redirected:
https://lions-mansion.jp/MA141070/
so I tried use cURL,
function redirect1($url) {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_USERAGENT,'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT ,0);
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
$data = curl_exec($ch);
$data = curl_getinfo($ch,CURLINFO_EFFECTIVE_URL );
curl_close($ch);
return $data;
}
and also this:
function redirect($url) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$result = curl_exec($ch);
if (preg_match('~Location: (.*)~i', $result, $match)) {
$location = trim($match[1]);
}
return $result;
}
But I couldn't find the redirected url.
this page does not use a redirect-scheme that libcurl understands (it uses a html <meta http-equiv="REFRESH"-redirect, unsupported by libcurl), so libcurl can neither tell you where it is being redirected, nor can libcurl auto-follow the redirect (because libcurl does not understand it)
you need to parse out the redirect url yourself from the HTML, eg
function redirect1($url) {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_USERAGENT,'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT ,0);
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
$data = curl_exec($ch);
$domd=#DOMDocument::loadHTML($data);
$xp=new DOMXPath($domd);
// <META http-equiv="REFRESH" content="0;URL=http://sumai.tokyu-land.co.jp/branz/roppongi4/?iad=daikyo" />
$location=$xp->query("//meta[#http-equiv='REFRESH']")->item(0)->getAttribute("content");
// 0;URL=http://sumai.tokyu-land.co.jp/branz/roppongi4/?iad=daikyo
$location=substr($location,stripos($location,'URL=')+4);
curl_close($ch);
return $location;
}
var_dump(redirect1('https://lions-mansion.jp/MA141070/'));
output:
C:\projects\misc>php re.php
string(57) "http://sumai.tokyu-land.co.jp/branz/roppongi4/?iad=daikyo"
If keep you CURLOPT_RETURNTRANSFER to true, after executing the CURL command you can use this function call to get the redirect of effective URL:
$finalUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
CURLINFO_EFFECTIVE_URL is refusing to work on this URL.
The result is exactly the same as the original URL, but if you follow the URL in your browser, it clearly redirects.
What is going on here?
Here is my code:
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_USERAGENT,'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
curl_exec($ch);
$redirectURL = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
echo $redirectURL;
curl_close($ch);
<?php
$url='https://source.amazon.com/forcelogin?returnToURL=https://www.amazon.com/';
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_USERAGENT,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.7) Gecko/20070914 Firefox/2.0.0.7");
curl_setopt($ch, CURLOPT_COOKIEFILE,'cookies.txt');
curl_setopt($ch, CURLOPT_COOKIEJAR, "cookies.txt");
curl_setopt($ch, CURLOPT_COOKIEFILE, "cookies.txt");
$data = curl_exec($ch);
curl_close($ch);
?>
I don't know why it doesn't load the website.
If I put regular sites like google.com, amazon.com or anything else is working.Anyone can help me out?
You need to add CURLOPT_SSL_VERIFYPEER and CURLOPT_SSL_VERIFYHOST options to access HTTPS URL.
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
I need to create a php script that logs into remote site then downloads a csv file on that site. I've searched all over and haven't found anything that does what I need. Currently I only have the following code which allows me to login to a remote site.
<?php
$username="";
$password="";
$url='';
$cookie="";
$postdata = 'email='.$username.'&pass='.$password;
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6");
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie);
curl_setopt($ch, CURLOPT_REFERER, $url);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata);
curl_setopt($ch, CURLOPT_POST, true);
$result = curl_exec($ch);
echo $result;
curl_close($ch);
?>
How about something like:
<?php
$username="";
$password="";
$url='';
$csvurl='';
$cookie="";
$postdata = 'email='.$username.'&pass='.$password;
/*first log in*/
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6");
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie);
curl_setopt($ch, CURLOPT_REFERER, $url);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata);
curl_setopt($ch, CURLOPT_POST, true);
$result = curl_exec($ch);
echo $result;
/*now download the file*/
curl_setopt($ch, CURLOPT_URL, $csvurl);
curl_setopt($ch, CURLOPT_REFERER, $url);
$result = curl_exec($ch);
echo $result;
curl_close($ch);
?>
When I add an parameter to the CURL init function:
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
It does print the site content I'm sending a request to, the problem is that I want to manually parse the returned content ($response = curl_exec($ch);) but the problem is that, the site is displaying the page content and I want to keep having the site content on my $response variable so I could parse it, but at the same, stop it from displaying it.
Is that feasible?
The curl code:
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $action);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6");
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_COOKIEJAR, '');
curl_setopt($ch, CURLOPT_COOKIEFILE, '');
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
$response = curl_exec($ch);
curl_close($ch);
You should set the CURLOPT_RETURNTRANSFER flag to true. From PHP.NET manual
CURLOPT_RETURNTRANSFER TRUE
To return the transfer as a string of the return value of curl_exec() instead of outputting it out directly.
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);