Related
Good afternoon,
I would like to get the data FROM this website using the API, and especially ids part ( aim ) in PHP but I obtain the following error : HTTP 415 Unsupported Media Type.
The code that I'm currently using is ( Иван Ясинский ) :
function getContentCurl($url)
{
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64;
rv:25.0) Gecko/20100101 Firefox/25.0');
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($ch, CURLOPT_TIMEOUT, 3);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_ENCODING, "gzip");
return $result = curl_exec($ch);
}
$url = "https://cadres.apec.fr/cms/webservices/rechercheOffre/ids";
$content = getContentCurl($url);
var_dump($content);
Does anyone have any idea to overcome this issue ? Thank for your help.
Your missing the post data. I'm not sure of your exact query but an example query is in this code:
<?php
function getContentCurl($url)
{
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0');
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($ch, CURLOPT_TIMEOUT, 3);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_ENCODING, "gzip");
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json'));
curl_setopt($ch, CURLOPT_POSTFIELDS, '{"activeFiltre":true,"motsCles":"","fonctions":[],"lieux":[],"pointGeolocDeReference":{},"secteursActivite":[],"typesContrat":[],"typesConvention":[],"niveauxExperience":[],"sorts":[{"type":"DATE","direction":"DESCENDING"}],"pagination":{"startIndex":0,"range":20},"typeClient":"CADRE"}');
return $result = curl_exec($ch);
}
$url = "https://cadres.apec.fr/cms/webservices/rechercheOffre/ids";
$content = getContentCurl($url);
var_dump($content);
Outputs:
string(4083) "{"totalCount":78073,"offreFilters":[{"offreFiltering":"CONTRACT_TYPE_FILTERING","offreFilterItems":[{"key":101888,"count":69550},{"key":20053,"count":120},{"key":101930,"count":2144},{"key":101887,"count":6259}]},{"offreFiltering":"CONTRACT_DURATION_FILTERING","offreFilterItems":[{"key":143691,"count":4981},{"key":143692,"count":2562},{"key":143693,"count":976}]},{"offreFiltering":"COMPANY_TYPE_FILTERING","offreFilterItems":[{"key":143684,"count":35345},{"key":143685,"count":18112},{"key":143686,"count":13180},{"key":143687,"count":11436}]},{"offreFiltering":"POSITION_STATUS_FILTERING","offreFilterItems":[{"key":143688,"count":65264},{"key":143689,"count":1629},{"key":143690,"count":11180}]},{"offreFiltering":"EXPERIENCE_FILTERING","offreFilterItems":[{"key":101881,"count":36267},{"key":20043,"count":45182},{"key":20044,"count":11797},{"key":20045,"count":11317}]},{"offreFiltering":"GEOLOCALISABLE_FILTERING","offreFilterItems":[{"key":0,"count":6088},{"key":1,"count":71985}]},{"offreFiltering":"JOB_CODE_FILTERING","offreFilterItems":[{"key":101829,"count":1360},{"key":101828,"count":19240},{"key":101831,"count":11451},{"key":101830,"count":1336},{"key":101833,"count":17141},{"key":101832,"count":10470},{"key":101835,"count":4190},{"key":101834,"count":7131},{"key":101837,"count":11231},{"key":101836,"count":1463}]},{"offreFiltering":"LOCATION_FILTERING","offreFilterItems":[{"key":102099,"count":731},{"key":99700,"count":411},{"key":799,"count":77342}]},{"offreFiltering":"NAF_CODE_FILTERING","offreFilterItems":[{"key":101761,"count":1083},{"key":101760,"count":3773},{"key":101763,"count":3607},{"key":101762,"count":11726},{"key":101765,"count":385},{"key":101764,"count":2123},{"key":101767,"count":1134},{"key":101766,"count":1757},{"key":101769,"count":621},{"key":101768,"count":169},{"key":101771,"count":332},{"key":101770,"count":1241},{"key":101773,"count":9665},{"key":101772,"count":9572},{"key":101775,"count":695},{"key":101774,"count":2410},{"key":101777,"count":1597},{"key":101776,"count":1292},{"key":101779,"count":1308},{"key":101778,"count":336},{"key":101780,"count":158},{"key":101753,"count":11418},{"key":101752,"count":1529},{"key":101755,"count":1026},{"key":101754,"count":2418},{"key":101757,"count":4055},{"key":101756,"count":1413},{"key":101759,"count":959},{"key":101758,"count":271}]},{"offreFiltering":"NIVEAU_ETUDE"},{"offreFiltering":"DUREE_STAGE","offreFilterItems":[{"key":599765,"count":381},{"key":599766,"count":233},{"key":599767,"count":710},{"key":599768,"count":441},{"key":599769,"count":280},{"key":599770,"count":2936},{"key":599771,"count":3538}]},{"offreFiltering":"DATE_PRISE_POSTE","offreFilterItems":[{"key":599777,"count":75435},{"key":599778,"count":2209},{"key":599779,"count":429}]},{"offreFiltering":"PERIOD_FILTERING","offreFilterItems":[{"key":101850,"count":11054},{"key":101851,"count":32259},{"key":101852,"count":78073},{"key":101853,"count":78073}]},{"offreFiltering":"WAGE_FILTERING","offreFilterItems":[{"key":597179,"count":7147},{"key":599738,"count":38996},{"key":101840,"count":56141},{"key":101841,"count":25697},{"key":101842,"count":6213},{"key":101843,"count":1408}]}],"resultats":[{"#uriOffre":"offre?numeroOffre=163035045W"},{"#uriOffre":"offre?numeroOffre=163035043W"},{"#uriOffre":"offre?numeroOffre=163035044W"},{"#uriOffre":"offre?numeroOffre=162918884W"},{"#uriOffre":"offre?numeroOffre=162913997W"},{"#uriOffre":"offre?numeroOffre=163035042W"},{"#uriOffre":"offre?numeroOffre=162930465W"},{"#uriOffre":"offre?numeroOffre=162926697W"},{"#uriOffre":"offre?numeroOffre=163035041W"},{"#uriOffre":"offre?numeroOffre=163035040W"},{"#uriOffre":"offre?numeroOffre=163035039W"},{"#uriOffre":"offre?numeroOffre=163035036W"},{"#uriOffre":"offre?numeroOffre=163035032W"},{"#uriOffre":"offre?numeroOffre=163035031W"},{"#uriOffre":"offre?numeroOffre=163035030W"},{"#uriOffre":"offre?numeroOffre=163035029W"},{"#uriOffre":"offre?numeroOffre=163035027W"},{"#uriOffre":"offre?numeroOffre=163035026W"},{"#uriOffre":"offre?numeroOffre=163035022W"},{"#uriOffre":"offre?numeroOffre=163035020W"}]}"
I am running Linux on a VPS.
When I run the this command:
curl https://www.bloomingdales.com/account/signin -H "Cookie: ewqeqweq" -X GET
... I get the source code of the site.
I tried to write the same command in PHP:
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://www.bloomingdales.com/account/signin');
curl_setopt($ch, CURLOPT_ENCODING ,"");
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_COOKIESESSION, 1);
curl_setopt($ch, CURLOPT_HEADER, array("Cookie: ewqeqweq"));
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookie.txt');
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookie.txt');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_TIMEOUT, 10);
curl_setopt($ch, CURLOPT_REFERER, 'https://www.bloomingdales.com/account/signin');
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0');
$get_ = curl_exec($ch);
echo $get_;
curl_close($ch);
However, the result of this code is: NULL.
I am wondering what is wrong in my code?
You can use my source code
function _curl($url,$post="",$usecookie = false,$_sock = false,$timeout = false) {
$ch = curl_init();
if($post) {
curl_setopt($ch, CURLOPT_POST ,1);
curl_setopt ($ch, CURLOPT_POSTFIELDS, $post);
}
if($timeout){
curl_setopt($ch, CURLOPT_TIMEOUT,$timeout);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
}
if($_sock){
curl_setopt($ch, CURLOPT_PROXY, $_sock);
curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
}
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.10) Gecko/20100914 Firefox/3.6.10");
if ($usecookie) {
curl_setopt($ch, CURLOPT_COOKIEJAR, $usecookie);
curl_setopt($ch, CURLOPT_COOKIEFILE, $usecookie);
}
curl_setopt($ch, CURLOPT_HTTPHEADER, array(
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language: en-US,en;q=0.5',
'Accept-Encoding: zip, deflate, sdch'
));
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
$result=curl_exec ($ch);
curl_close ($ch);
return $result;
}
$socks5 = '176.126.196.52:24369';
$cookie = tempnam('cookies','coo'.rand(1000000,9999999));
$url = "https://www.bloomingdales.com/account/signin";
$post = "";
$s = _curl($url,$post,$cookie,$socks5,'');
echo $s;
unlink($cookie);
My result
There is a problem when I try to login to Paypal via Curl, I think I have all post fields correctly setup, but it doesn't give any output. I want to receive the page content after login, but there is no actual output.
Code:
$loginUrl = 'https://www.paypal.com/en/cgi-bin/webscr?cmd=_login-submit&dispatch=5885d80a13c0db1f8e263663d3faee8d66f31424b43e9a70645c907a6cbd8fb4';
//init curl
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $loginUrl);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_HTTPHEADER, array(
'User-agent: Mozilla/5.0 (Windows NT 6.3; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0',
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-language: nl,en-US;q=0.7,en;q=0.3',
'accept-encoding: gzip, deflate'));
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_POSTFIELDS, 'login_email='.urlencode($username).'&login_password='.urlencode($password).'&submit=inloggen&browser_name=Firefox&browser_version=32&browser_version_full=32.0&operating_system=Windows&bp_mid='.urlencode('v=1;a1=na~a2=na~a3=na~a4=Mozilla~a5=Netscape~a6=5.0 (Windows)~a7=20100101~a8=na~a9=true~a10=Windows NT 6.3; WOW64~a11=true~a12=Win32~a13=na~a14=Mozilla/5.0 (Windows NT 6.3; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0~a15=false~a16=nl~a17=na~a18=www.paypal.com~a19=na~a20=na~a21=na~a22=na~a23=1600~a24=900~a25=24~a26=860~a27=na~a28=Wed Oct 29 2014 17:56:23 GMT+0100~a29=1~a30=na~a31=yes~a32=na~a33=na~a34=no~a35=no~a36=yes~a37=no~a38=online~a39=no~a40=Windows NT 6.3; WOW64~a41=no~a42=no~&').'bpks1='. urlencode('v=1;l=8;Di0:141497Ui0:99Di1:182').'&bpks2=&bpks3=');
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookie.txt');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$content = curl_exec($ch);
exit(var_dump($content)); // OUTPUT: string(26) ▒▒";
I have tried this on other websites successfully, so I have no idea why this isn't working? Any advice would be appreciated.
try this
<?php
$ch = curl_init();
//Header
curl_setopt($ch, CURLOPT_URL, "https://www.paypal.com/pl/cgi-bin/webscr?cmd=_login-submit&dispatch=5885d80a13c0db1f8e263663d3faee8de62a88b92df045c56447d40d60b23a7c");
//curl_setopt($ch, CURLOPT_PROXY, $proxy);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); // return server response
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.6 (KHTML, like Gecko) Chrome/16.0.897.0 Safari/535.6');
curl_setopt ($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt ($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt( $ch, CURLOPT_HTTPHEADER, array("REMOTE_ADDR: 127.0.0.1", "HTTP_X_FORWARDED_FOR: 127.0.0.1"));
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookieJar.txt'); // save cookie file
curl_setopt($ch, CURLOPT_COOKIEFILE, 'cookie.txt');
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($ch, CURLOPT_REFERER, 'https://www.paypal.com');
curl_setopt ($ch, CURLOPT_COOKIESESSION, TRUE);
curl_setopt($ch, CURLOPT_POST, 1); // use post data
$post = array(
"login_cmd" => null,
"login_params" => null,
"login_email" => "test",
"login_password" => "test",
"submit.x" => "login",
//"auth" => "AOeCYVv0IxkugC2Pyz2AhTaW2P7hWuy5w9FoeuyB48gjjJZN3mTtuL79Tzs9dY.CF",
"form_charset" => "UTF-8",
"browser_name" => "Chrome",
"browser_version" => "537.36",
//"browser_version_full" => "40.0.2214.115",
//"operating_system" => "Windows",
);
$post = http_build_query($post);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
$data = curl_exec($ch);
if(curl_errno($ch))
{
echo 'error:' . curl_error($ch);
}
curl_close($ch);
print_r($data);
?>
I am trying to log into an .asp site with a https login page from PHP. I can't get logged into the site . There dosen't seem to be a cookie generated, viewstate, etc nor does it work leaving these parameters out. The form fields seem to fill correctly (i can physically see the login name) but im not sure about the password field which is a password type, but i don't think there is a issue there, its correctly spelled etc.
I have tried all the related posts including http://www.mishainthecloud.com/2009/12/screen-scraping-aspnet-application-in.html?showComment=1368565341638#c9104469935977149435
My code is below and returning "not Found" (error code 7 i think..) on the final call. No curl errors are present on the 1st two calls.
Can anyone help with this?
$ckfile = tempnam ("/tmp", "CURLCOOKIE");
// URL to login page
$url = "https://secure2.clubwise.com/glenview/memberlogin.asp";
// Get Login page and its cookies/ viewstate , etc
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
//curl_setopt($ch, CURLOPT_COOKIEJAR, $ckfile); // no cookie stored
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
$output = curl_exec($ch);
//print(esc_html($output));
$viewstate = regexExtract($output,$regexViewstate,$regs,1);
$eventval = regexExtract($output, $regexEventVal,$regs,1);
// rebuild post info -- view state and eventvalidate empty! cant find on page
$fields_string =
'__VIEWSTATE='.rawurlencode($viewstate).
'&__EVENTVALIDATION='.rawurlencode($eventval).
'&login='.urlencode('xxx#xxx.com').
'&password='.urlencode('xxxx').
'&submit='.urlencode('Sign in').
'&redirect=';
echo $fields_string;
// Post login form -- password field ok?
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, 5);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields_string);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
//curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); // Tells cURL to follow redirects
$outputb = curl_exec($ch);
print curl_error;
//var_dump($outputb);
$url = "https://secure2.clubwise.com/glenview/bookclass.asp";
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
//curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$outputc = curl_exec($ch);
print curl_error;
var_dump($outputc);
header information:
//$header= array(
//'HTTP/1.1 200 OK',
//'Date: Mon, 09 Jun 2014 00:55:17',
//'GMT Server: Microsoft-IIS/6.0',
//'X-Powered-By: ASP.NET',
//'Content-Length: 21035',
//'Content-Type: text/html',
//'Set-Cookie: ASPSESSIONIDCCSBSTDC=IHJBDOOBIDMJDDOFLAOOBENL; path=/',
//'Cache-control: private'
//);
SOLUTION: it was curl_init() on each operation that was causing this to break. Viewstate, headers are not needed.
$ourFileName = get_stylesheet_directory()."/cookieFile.txt";
$ckfile = $ourFileName;
// URL to login page
$url = "https://secure2.clubwise.com/glenview/memberlogin.asp";
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $ckfile); // Stores cookies in the temp file
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_HEADER, 1);
$output = curl_exec($ch);
//Cookie obtained, login
$fields_string =
'&redirect='.
'&login='.urlencode('vvv#xxx.com').
'&password='.urlencode('xxx').
'&submit='.urlencode('"Sign in"' )
;
//cookie in the header + header not required
/*
$cookielines =file($ckfile);
foreach($cookielines as $row) {
if($row[0] != '#') {
$cookie=$row;
}
}
$header= array( // not needed at moment
'HTTP/1.1 200 OK',
'Date: Mon, 09 Jun 2014 00:55:17',
'GMT Server: Microsoft-IIS/6.0',
'X-Powered-By: ASP.NET',
'Content-Length: 21035',
'Content-Type: text/html',
'Set-Cookie: $cookie; path=/',
'Cache-control: private'
);
*/
$url = "https://secure2.clubwise.com/glenview/memberlogin.asp";
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, 4);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields_string);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile); //Uses cookies from the temp file
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); // Tells cURL to follow redirects
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
curl_setopt($ch, CURLOPT_ENCODING,"");
curl_setopt($ch, CURLOPT_REFERER,"https://secure2.clubwise.com/glenview/memberlogin.asp");
$outputb = curl_exec($ch);
$err = curl_errno ( $ch ); echo "<br>error=".$err;
$errmsg = curl_error ( $ch ); echo "<br>errmsg=".$errmsg;
$header = curl_getinfo ( $ch ); echo "<br>header="; var_dump($header);
$httpCode = curl_getinfo ( $ch, CURLINFO_HTTP_CODE ); echo "<br>httpcode=".$httpCode;
print curl_error;
//Now you should be able to access any pages within the password-restricted area by just including the cookies for each call:
$url = "https://secure2.clubwise.com/glenview/bookclass.asp?Mode=Area&RecId=67";
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, 0 );
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile); //Uses cookies from the temp file
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$outputc = curl_exec($ch);
print curl_error;
var_dump($outputc);
Please check again that cookie stored on first page. Set CURLOPT_COOKIEFILE and CURLOPT_COOKIEJAR for Session cookies and not reinit or close curl for save a session.
Set CURLOPT_REFERER, some sites check it for login page.
I'm trying to use CURL with PHP to login to Vimeo.com, Vimeo login.
To provide the data for CURL to use (cookie and field data), I'm using a browser extension to read the field data off of the webpage and get the cookies. I'm then passing that data through to my server and am trying to login using curl.
I'm quite positive that the browser extension part works correctly (gets the correct data) because I can verify what it's passing with what it should be passing, and it matches correctly.
Additionally, I've used this on other sites as well, and it has no problem logging in, but on vimeo the exec returns false.
Any thoughts?
function curlpage(){
$ch = curl_init();
$url = $this->input->post('url');
$data = $this->input->post('data');
$cookie = $this->input->post('cookie');
$method = $this->input->post('method');
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POSTFIELDS,$data);
if(strtolower($method)=="put"){
curl_setopt($ch, CURLOPT_PUT, 1);
}
else{
curl_setopt($ch, CURLOPT_PUT, 0);
}
if(strtolower($method)=="get"){
curl_setopt($ch, CURLOPT_HTTPGET, 1);
}
else{
curl_setopt($ch, CURLOPT_HTTPGET, 0);
}
if(strtolower($method)=="post"){
curl_setopt($ch, CURLOPT_POST, 1);
}
else{
curl_setopt($ch, CURLOPT_POST, 0);
}
curl_setopt($ch, CURLOPT_FAILONERROR, true);
curl_setopt($ch, CURLOPT_UNRESTRICTED_AUTH, true);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_CAINFO, getcwd().'/certificates/BuiltinObjectToken-EquifaxSecureCA.crt');
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_COOKIE, $cookie);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:17.0) Gecko/20100101 Firefox/17.0");
curl_setopt($ch, CURLOPT_REFERER, $url);
$error = curl_error($ch);
$url=curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
if(!preg_match('/^http(s)?:\/\//', $url)){
$url = 'http://' . $url;
}
$host = parse_url($url, PHP_URL_HOST);
$page = curl_exec($ch);
curl_close($ch);
return array('page'=>$page, 'url'=>$host, 'error'=>$error);
}
Here's a sample of the data I'm sending to the above function on my server (with a bogus email an password and altered cookies):
data=action%253Dlogin%2526service%253Dvimeo%2526email%253Dhou%2540fah.com%2526password%253Dudwt%2526token%253D6b2fc081bcdf02b1f58a390d6a3f8b83
cookie=__utma%3D18392654.1284111214.1456668252.1456678435.1456181183.3%3B__utmb%3D18302654.2.10.1454681883%3B__utmc%3D18232154%3B__utmz%3D17202654.1456675435.2.2.utmcsr%3Dgoogle%7Cutmccn%3D(organic)%7Cutmcmd%3Dorganic%7Cutmctr%3D(not%2520provided)%3B
method=POST
url=http%3A%2F%2Fvimeo.com%2Flog_in
$ret = customSendDataByCurl("https://vimeo.com/log_in");
preg_match("/xsrft: \'(.*)\',/i",$ret,$token);
$token = $token[1];
echo "$token <hr>";
$cookie = '(copy from your browser using tamper data)... xsrft='.$token;
$headers = array(
"Referer: https://vimeo.com/log_in",
"User-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0",
"Accept: application/json",
"Accept-Language: en-US,en;q=0.5",
"X-Requested-With: XMLHttpRequest",
"X-Request: JSON",
"Content-Type: application/x-www-form-urlencoded; charset=utf-8",
);
$ret = customSendDataByCurl("https://vimeo.com/log_in?action=warm", "POSTDATA=email=(email url encoded)&token=".$token, $headers, $cookie);
$fields = array(
"action" => "login",
"service" => "vimeo",
"email" => "(email)",
"password" => "(pass)",
"token" => $token,
);
$headers = array(
"Referer: https://vimeo.com/log_in",
"User-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0",
"Content-Type: application/x-www-form-urlencoded; charset=utf-8",
);
$ret = customSendDataByCurl("https://vimeo.com/log_in", http_build_query($fields), $headers, $cookie);
$ret = customSendDataByCurl("https://vimeo.com/stats/video/84142281/totals/export:csv", http_build_query($fields), $headers, $cookie);
var_export($ret);
function customSendDataByCurl($agateway, $apostfields=null, $headers=array(), $cookie="") {
if(is_array($apostfields)) $apostfields = http_build_query($apostfields);
$cookiesFile = 'cookies.txt';
$ch = curl_init($agateway);
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt ($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt ($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
curl_setopt( $ch, CURLOPT_COOKIESESSION, true );
if ($headers) curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt( $ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0' );
curl_setopt($ch,CURLOPT_COOKIEJAR,$cookiesFile);
curl_setopt($ch,CURLOPT_COOKIEFILE,$cookiesFile);
curl_setopt($ch, CURLOPT_COOKIE, $cookie);
if(!empty($apostfields)) curl_setopt($ch, CURLOPT_POSTFIELDS, $apostfields);
$response = curl_exec($ch);
if(!$response) $response="CURL #".curl_errno($ch).": ".curl_error($ch);
return $response;
}
Vimeo, I've actually found to be a weird case for websites. They don't set all of their cookies up front, but rather set certain cookies needed to login when the form is submitted. So one of my problems was that I was not submitting all of the correct cookie information.
My second problem was that I was not correctly encoding my data as it was being sent.
All said and done, I now have got it to work!