I am receiving an invalid cookie string when trying capture the cookie using file_get_contents and curl. The cookie received while browsing directly from the browser is valid/active. But, the cookie captured from file_get_contents and curl seems to be invalid.
I am trying to capture from file_get_contents like this
$context = array(
'http' => array(
'method' => 'GET',
'header' => array('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*\/*;q=0.8', 'User-Agent:Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/63.0.3239.84 Chrome/63.0.3239.84 Safari/537.36'),
)
);
$cxContext = stream_context_create($context);
file_get_contents($url, false, $cxContext);
$cookies = array();
foreach ($http_response_header as $hdr) {
if (preg_match('/^Set-Cookie:\s*([^;]+)/', $hdr, $matches)) {
$cookies = $matches[1];
}
}
return $cookies;
I tried playing around with this, by setting headers, but the cookies returned always is either expired or simply invalid.
But, through a browser the cookie I get is always valid.
Anyone faced a similar problem, don't know how to tackle this issue.
There are several unanswered questions from my above comment, but I'll share this bit of code for example purposes. It's what I've used in the past as a base class for browser emulation using cURL:
<?php
if(!function_exists("curl_init")) { throw new Exception("CurlBrowser requires the cURL extension, which is not enabled!"); }
class CurlBrowser
{
public $userAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0";
/*
Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0
Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0
*/
public $cookiesFile = null;
public $proxyURL = null;
public $saveLastOutput = "";
public $caBundle = "cacert.pem";
public $httpHeaders = array();
public function __construct($UseCookies = true)
{
if(is_bool($UseCookies) && $UseCookies)
{
$this->cookiesFile = dirname(__FILE__)."/cookies.txt";
}
elseif(is_string($UseCookies) && ($UseCookies != ""))
{
$this->cookiesFile = $UseCookies;
}
}
public function SetCustomHTTPHeaders($arrHeaders)
{
$this->httpHeaders = $arrHeaders;
}
public function SetProxy($proxy)
{
$this->proxyURL = $proxy;
}
public function Get($url)
{
return $this->_request($url);
}
public function Post($url,$data = array())
{
return $this->_request($url,$data);
}
private function _request($form_url,$data = null)
{
$ch = curl_init($form_url);
// CA bundle
$caBundle = $this->caBundle;
if(file_exists($caBundle))
{
// Detect and convert relative path to absolute path
if(basename($caBundle) == $caBundle)
{
$caBundle = getcwd() . DIRECTORY_SEPARATOR . $caBundle;
}
// Set CA bundle
curl_setopt($ch, CURLOPT_CAINFO, $caBundle);
}
// Cookies
if($this->cookiesFile !== null)
{
curl_setopt($ch, CURLOPT_COOKIEFILE, $this->cookiesFile);
curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookiesFile);
}
// User Agent
curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent);
// Misc
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_ENCODING, "gzip, deflate");
// Optional proxy
if($this->proxyURL !== null)
{
curl_setopt($ch, CURLOPT_PROXY, $this->proxyURL);
}
// Custom HTTP headers
if(count($this->httpHeaders))
{
curl_setopt($ch, CURLOPT_HTTPHEADER, $this->httpHeaders);
}
// POST data
if($data !== null)
{
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
}
// Run operation
$result = curl_exec($ch);
if($result === false)
{
throw new Exception(curl_error($ch));
}
else
{
if(!empty($this->saveLastOutput))
{
file_put_contents($this->saveLastOutput,$result);
}
return $result;
}
}
}
?>
You'd use it like so:
<?php
$browser = new CurlBrowser();
$html = $browser->Get("https://....");
...etc...
My gut guess is that you're simply missing a cookie jar in your original code, but that's mostly based on gut feeling, since we don't have all your problem code at this time.
I got open curl session, that has cookie files set, so it would remember session id (from logging in). But how do I now modify those cookies?
curl_setopt($curl, CURLOPT_HTTPHEADER, array('Cookie:blabla=bleble'));
Does not seem to act correctly, as it adds another Cookie header, so if for example I had between request headers:
...
Cookie:blabla=uhuhu;tralala=ahahaha
...
And I run this:
curl_setopt($curl, CURLOPT_HTTPHEADER, array('Cookie:blabla=bleble'));
It will simply add another Cookie header with same variable but different value:
...
Cookie:blabla=uhuhu;tralala=ahahaha
...
Cookie:blabla=bleble
So how do I modify existing Cookies instead of adding another Cookie header?
For example, when I login to site, keep session cookies in file, and in second request would like to replace it, I get something like this (being send):
When replacing by CURLOPT_HTTPHEADER
curl_setopt($curl, CURLOPT_HTTPHEADER, 'Cookie: 46db1eb1/sessid=blabla; 46db1eb1/zoom-accessibility=small');
POST /sysbus/NeMo/Intf/data:setFirstParameter HTTP/1.1
User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1
Host: 192.168.1.1
Accept: */*
Cookie: 46db1eb1/sessid=YHGDba31faykfyTJraQMewP+
Content-Type:application/x-sah-ws-1-call+json; charset=UTF-8
X-Context:n1psjUEjqs5j7Bb5vvT0J3wXMunQ2e8c3ksb5Uikcdt9Ct1XOJUmAh8azkOqmMqe
X-Prototype-Version:1.7
X-Requested-With:XMLHttpRequest
Referer:http://192.168.1.1/advConfigAccessType.html
Cookie:46db1eb1/sessid=blabla; 46db1eb1/zoom-accessibility=small
Content-Length: 73
As you can see - cookies are doubled.
When replacing with CURLOPT_COOKIE
curl_setopt($curl, CURLOPT_COOKIE, '46db1eb1/sessid=blabla; 46db1eb1/zoom-accessibility=small');
POST /sysbus/NeMo/Intf/data:setFirstParameter HTTP/1.1
User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1
Host: 192.168.1.1
Accept: */*
Cookie: 46db1eb1/sessid=FHjCFhn/VKgkC09y7772fXpp; 46db1eb1/sessid=blabla; 46db1eb1/zoom-accessibility=small
Content-Type:application/x-sah-ws-1-call+json; charset=UTF-8
X-Context:VCtJDc4PrVjybyFcw8jGAc3anO5mXm7GPnfgn8VES54TqwUKr4fP68PhJHtuSn2x
X-Prototype-Version:1.7
X-Requested-With:XMLHttpRequest
Referer:http://192.168.1.1/advConfigAccessType.html
Content-Length: 73
Values are doubled in same cookie header.
curl 7.26.0, Raspbian (Debian 7.8)
You can manipulate the CURLOPT_COOKIEJAR file. Something like this:
<?php
function init($f)
{
$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, "http://localhost/a.php");
curl_setopt($curl, CURLOPT_COOKIEJAR, $f);
curl_setopt($curl, CURLOPT_COOKIEFILE, $f);
return $curl;
}
function readCookies($f)
{
$cookies = [];
if (($handle = fopen($f, "r")) !== FALSE) {
while (($cookie = fgetcsv($handle, 1000, "\t")) !== FALSE) {
$cookies[] = $cookie;
}
fclose($handle);
}
return $cookies;
}
function writeCookies($f, $c)
{
$fp = fopen($f, "w");
foreach ($c as $cookie) {
fputcsv($fp, $cookie, "\t");
}
fclose($fp);
}
$f = './c.tmp';
$curl = init($f);
curl_exec($curl);
// playing with cookies:
$newCookie = ['blabla', '123'];
$cookies = readCookies($f);
if (!empty($cookies)) {
$found = false;
foreach($cookies as $key=>$val) {
// ignore comments and empty lines
if (count($val) == 7) {
if ($val[5] == $newCookie[0]) {
$found = $key;
break;
}
}
}
if ($found) {
$cookies[$found][6] = $newCookie[1];
} else {
// I am using first one as a template,
// but you may need to set all fields explicitly
// as they may differ
$cookie = $cookies[0];
$cookie[5] = $newCookie[0];
$cookie[6] = $newCookie[1];
$cookies[] = $cookie;
}
writeCookies($f, $cookies);
}
$curl = init($f);
if (empty($cookies)) {
curl_setopt($curl, CURLOPT_HTTPHEADER, array('Cookie:blabla=123'));
}
curl_exec($curl);
I've been experimenting over and over but what I've got so far doesn't allow me to login into Pinterest with CURL and I cannot understand why..
function pinLogin()
{
$login_post = array(
'source_url' => '/login/',
'data' => '{
"options":{
"username_or_email":"email",
"password":"password"
},
"context":{}}',
'module_path' => 'App()>LoginPage()>Login()>Button(text=Log In, size=large, class_name=primary, type=submit)',
);
$httpheaders = array(
'Connection: keep-alive',
'Pragma: no-cache',
'Cache-Control: no-cache',
'Content-Type: application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent: Mozilla/5.0 (Windows NT 6.1; rv:35.0) Gecko/20100101 Firefox/35.0',
'Accept: application/json, text/javascript, */*; q=0.01',
'Accept-Language: en-US,en;q=0.5',
'Accept-Encoding: gzip, deflate',
);
$login_header = array(
'X-Pinterest-AppState: active',
'X-NEW-APP: 1',
'X-APP-VERSION: 71854ca',
'X-Requested-With: XMLHttpRequest',
'Accept: application/json, text/javascript, */*; q=0.01'
);
// request home page to establish cookies and a session, set curl options
$ch = curl_init('http://www.pinterest.com/');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 0);
curl_setopt($ch, CURLOPT_AUTOREFERER, 1);
curl_setopt($ch, CURLOPT_COOKIEFILE, 'cookie.txt');
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookie.txt');
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_STDERR, fopen('/tmp/debug.txt', 'w+'));
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_HTTPHEADER, $httpheaders);
$data = curl_exec($ch);
// ----------------------------------------------------------------------------
// parse the csrf token out of the cookies to set later when logging in
list($headers, $body) = explode("\r\n\r\n", $data, 2);
preg_match('/csrftoken=(.*?)[\b;\s]/i', $headers, $csrf_token);
// next request the login page
curl_setopt($ch, CURLOPT_URL, 'http://www.pinterest.com/login/');
$data = curl_exec($ch);
// ----------------------------------------------------------------------------
// perform login post
$login_header[] = 'X-CSRFToken: ' . $csrf_token[1];
curl_setopt($ch, CURLOPT_URL, 'http://www.pinterest.com/resource/UserSessionResource/create/');
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $login_post);
curl_setopt($ch, CURLOPT_HTTPHEADER, array_merge($httpheaders, $login_header));
curl_setopt($ch, CURLOPT_REFERER, 'http://www.pinterest.com/login/');
curl_setopt($ch, CURLOPT_HEADER, 0);
$data = curl_exec($ch);
// ----------------------------------------------------------------------------
if (curl_getinfo($ch, CURLINFO_HTTP_CODE) != 200)
{
echo "Error logging in.<br />";
var_dump(curl_getinfo($ch));
} else {
$response = json_decode($data, true);
if ($response === null)
{
echo "Failed to decode JSON response.<br /><br />";
var_dump($response);
} else if ($response['resource_response']['error'] === null) {
echo "Logged in..";
}
print_r($response);
}
}
I've tried to emulate the same headers that are sent to pinterest but I'm still not able to login for some reason..
https://www.pinterest.com/resource/UserSessionResource/create/
POST /resource/UserSessionResource/create/ HTTP/1.1
Host: www.pinterest.com
User-Agent: Mozilla/5.0 (Windows NT 6.1; rv:35.0) Gecko/20100101 Firefox/35.0
Accept: application/json, text/javascript, */*; q=0.01
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Content-Type: application/x-www-form-urlencoded; charset=UTF-8
X-Pinterest-AppState: active
X-CSRFToken: suv5Dm0MHGc3tWY4GTPHzgBjYSXo94xt
X-NEW-APP: 1
X-APP-VERSION: 71854ca
X-Requested-With: XMLHttpRequest
Referer: https://www.pinterest.com/login/?next=https%3A%2F%2Fwww.pinterest.com%2F%3Fusername%3DUSER&prev=https%3A%2F%2Fwww.pinterest.com%2F%3Fusername%3DUSER
Content-Length: 456
Cookie: __utma=229774877.1495817695.1423754956.1424404967.1424434787.45; __utmz=229774877.1424125793.30.5.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); csrftoken=suv5Dm0MHGc3tWY4GTPHzgBjYSXo94xt; _pinterest_sess=TWc9PSZmWTFLSWM5cGx5aEhiM0ZTdHR2R21xS2JMVlVPejZYV1lMZWZadXBtak9icVlaRjdKZGozMU5vY3k4ZXRVUjZCQS90aFI0NndIeTNWWnR5RkVHY0VtSlM1UHRIZm01UFNGY093OHk0US9GRGY5Qk1FT0JsVEZjdTVSMDA5ODdPZUhhd2tvcWJVc3hqYmlNdG9PLytMQXc9PSZ5RXRjOUdvZFI0L1hoWTVFMnlsb2lNKzRSTW89; _b="AQ1q3LoHG1dIHash9bxk4SiJLwh9Pie2j1AhDB2OYuDFJcwxnUdVLzs9hLcTSKS53mU="; _pinterest_pfob=disabled; c_dpr=1; __utmb=229774877.28.4.1424435987021; __utmc=229774877; __utmt=1; logged_out=True; fba=True; GCSCE_5B243246522C4B23F685F2EB9D5F3C78DF8A0272_S3=C=694505692171-31closf3bcmlt59aeulg2j81ej68j6hk.apps.googleusercontent.com:S=c313ffc1a154b200119a21be80be878b703de85b.BK7j4ooMbUBBATCa.2d62:I=1424435991:X=1424522391
Connection: keep-alive
Pragma: no-cache
Cache-Control: no-cache
source_url=/login/
&data=
{
"options":
{
"username_or_email":"EMAIL#EMAIL.COM",
"password":"PASSWORD1GOES2HERE"
},
"context":{}
}
&module_path=App()>LoginPage()>Login()>Button(text=Log In, size=large, class_name=primary, type=submit)
im not sure why your code doesn't work, but im pretty sure the array_merge will mess up the numeric keys (if any).. and that you're not handling X-CSRFToken header correctly (it changes on several places, and you only check it once).. anyway, doing this without an api isn't as easy as it may look like, but
this works as of 22 february 2015, but be careful with the username/password, as i am probably not escaping it correctly (should probably escape it with json_encode() somehow)
EDIT: updated code so you get the logged in HTML on last request. (this proves beyond all reasonable doubt that you have in fact logged in ;) the way i checked it was to base64_encode() the output, then run this javascript in my browser: document.body.outerHTML=atob("base64"); , then i saw the same "you are logged in" screen)
<?php
error_reporting(E_ALL);
set_error_handler("exception_error_handler");
function exception_error_handler($errno, $errstr, $errfile, $errline ) {
if (!(error_reporting() & $errno)) {
// This error code is not included in error_reporting
return;
}
throw new ErrorException($errstr, 0, $errno, $errfile, $errline);
}
$curlh=hhb_curl_init(array(
CURLOPT_USERAGENT=>"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36"
,CURLOPT_HEADER=>true
)
);
$username="f327410#trbvm.com";
$password="f327410#trbvm.compassword";
$matches=array();
$info=hhb_curl_exec($curlh,'https://www.pinterest.com/login/?next=https%3A%2F%2Fwww.pinterest.com%2F&prev=https%3A%2F%2Fwww.pinterest.com%2F');//get session cookie and stuff (should be handled by curl automatically)
preg_match("/csrftoken\=([^\;]*)/",$info,$matches);
$CSRFToken=$matches[1];
curl_setopt_array($curlh,array(
CURLOPT_URL=>'https://www.pinterest.com/resource/UserSessionResource/create/'
,CURLOPT_POST=>true
,CURLOPT_ENCODING=>"gzip, deflate"
,CURLOPT_HTTPHEADER=>array(
'Accept:application/json, text/javascript, */*; q=0.01',
'Accept-Language:nb-NO,nb;q=0.8,no;q=0.6,nn;q=0.4,en-US;q=0.2,en;q=0.2',
'Connection:keep-alive',
//TODO: Content-Length:414
'Content-Type:application/x-www-form-urlencoded; charset=UTF-8',
//Cookie:csrftoken=wu1TXmJFeCD1q5scixeeK8QFkHSIIXg1; _pinterest_sess=TWc9PSZIbitpRE1Ka2tuRmNXTGNHY3NXQS9reXVvNENxdytpM3BkMCswNldrOUk5WDRucEk5UldYWEIwUERlWG84YXFOT1VrdlRiVHVIMUxTMkthM3hrYTZLTkM0NWJHQzFiQzVvdUQ5Ynp1Q255OUFBOEFVOWFpSzh4NHo2SC9RcTJ5M3NiNEt3YmliTmR2YTRyb0RPMlN3elE9PSZxUWtoVkZ3c0xXYkhMNEtYQVZBWXY5ak1Ec2s9; c_dpr=1; __utmt=1; __utma=229774877.1252202543.1424620619.1424620619.1424620619.1; __utmb=229774877.5.7.1424620619; __utmc=229774877; __utmz=229774877.1424620619.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)
'Host:www.pinterest.com',
'Origin:https://www.pinterest.com',
'Referer:https://www.pinterest.com/',
'X-APP-VERSION:7c24931',
'X-CSRFToken:'.$CSRFToken,
'X-NEW-APP:1',
'X-Pinterest-AppState:active',
'X-Requested-With:XMLHttpRequest',
)
,CURLOPT_POSTFIELDS=>
'source_url='.rawurlencode('/login/?next=https%3A%2F%2Fwww.pinterest.com%2F&prev=https%3A%2F%2Fwww.pinterest.com%2F').
'&data='.rawurlencode('{"options":{"username_or_email":"'.$username.'","password":"'.$password.'"},"context":{}}').
//not sure if username/password is escaped correctly.
'&module_path='.rawurlencode('App()>LoginPage()>Login()>Button(text=Logg inn, size=large, class_name=primary, type=submit)')
));
$info=hhb_curl_exec($curlh,'https://www.pinterest.com/resource/UserSessionResource/create/');;
$matches=array();
preg_match("/csrftoken\=([^\;]*)/",$info,$matches);
$CSRFToken=$matches[1];
//var_dump(__LINE__,$matches,$info);die();
//^this is interesting..
curl_setopt_array($curlh,array(
CURLOPT_URL=>"https://www.pinterest.com/resource/UserRegisterTrackActionResource/update/"
,CURLOPT_POST=>true
,CURLOPT_ENCODING=>"gzip, deflate"
,CURLOPT_HTTPHEADER=>array(
"Origin:https://www.pinterest.com",
"Accept-Language:nb-NO,nb;q=0.8,no;q=0.6,nn;q=0.4,en-US;q=0.2,en;q=0.2",
"Accept:application/json, text/javascript, * /*; q=0.01",
"X-Requested-With:XMLHttpRequest",
"X-NEW-APP:1",
"X-APP-VERSION:7c24931",
"X-Pinterest-AppState:active",
"Referer:https://www.pinterest.com/",
"Connection:keep-alive",
//TODO: Content-Length:358
"Content-Type:application/x-www-form-urlencoded; charset=UTF-8",
"Host:www.pinterest.com",
"X-CSRFToken:".$CSRFToken//TODO: verify that the token has not changed.
)
,CURLOPT_POSTFIELDS=>
'source_url='.rawurlencode('/login/?next=https%3A%2F%2Fwww.pinterest.com%2F&prev=https%3A%2F%2Fwww.pinterest.com%2F').
'&data='.rawurlencode('{"options":{"action":"setting_new_window_location"},"context":{}}').
//not sure if username/password is escaped correctly.
'&module_path='.rawurlencode('App()>LoginPage()>Login()>Button(text=Logg inn, size=large, class_name=primary, type=submit)')
));
$info=hhb_curl_exec($curlh,'https://www.pinterest.com/resource/UserRegisterTrackActionResource/update/');
//var_dump(__LINE__,$info);die();
//now we should be logged in! :D
curl_setopt_array($curlh,array(
CURLOPT_URL=>"https://www.pinterest.com/resource/UserRegisterTrackActionResource/update/"
,CURLOPT_POST=>false
,CURLOPT_ENCODING=>"gzip, deflate"
,CURLOPT_HTTPHEADER=>array(
"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language:nb-NO,nb;q=0.8,no;q=0.6,nn;q=0.4,en-US;q=0.2,en;q=0.2",
"Connection:keep-alive",
"Host:www.pinterest.com",
"Referer:https://www.pinterest.com/"
)
));
/*
//fuckthis Accept-Encoding:gzip, deflate, sdch
//Cookie:c_dpr=1; __utmt=1; __utma=229774877.1252202543.1424620619.1424620619.1424620619.1; __utmb=229774877.5.7.1424620619; __utmc=229774877; __utmz=229774877.1424620619.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _b="AQ3m6m5qQAVDaIkyqRoJYJ9ecazmK4aobP3PczTxb/BtXObCwlC/5kusK9/Ymj2luo8="; csrftoken=EitE4BCiLq3sz0hf5lHtCx6uNvyIaalo; _pinterest_sess="TWc9PSZLclVramZrWGRUMVYzZW1ZbmxXTXFXeWpHU2ZOVFBFNmFUOXU2ZlNJWFJ0TkdzTy9TZ1RQdmxtNmxZa0JNWnliR2VRS2t6UTRZVGtSVnNySlJKRVBEUjh4K3FWR2gxYi8yS3AxTmhqWW9COWZWaGJiK1Q2Y29ydjhLSGRDb2srdTdHaVh6RU12SEZnVmxlM09UNEloQU9JKzQxRDNqOFlISHRHZ0hIVW9kTUttWlhEd1BOaTJnbHZYTDZ5enBRSGtubDJaSnNKSjlzaG9SaWsrMFZaenhLeWpVaElxbTdZOG1sa3ZGeWQ3MWNFQy81YmtHQkxsZDlBQVNEK1FTUUJEYWZqV2tUMzVDVVM4R1VXL0lCOHZ3MHhPcC81YVZjOWRnSkZoTXFVQXRLU21OK05PZmtFczNvY2ZGdVRMS2pWdXR0WG8wakJTeTdYNlRqV3NZVmtHQzBsM0VyVnhVeXIzVkRWdXlqT3Q1eFVqWUJwVkxuR1ZwY3M5YXJBU2xKQ0lua3U1UkRxYWk0c0lVR1lJcHpMOUZNQXo0ZWlRSDRlaGVSa3NUaEFnREl2Q2lvN0xQc05DNjk5emNESDdaM3YxVmFwNU9KVFhLUGJBVStLcVZjVk1pMlREa3JzcW1FWEdSMGF0cXdvTlpGaz0mYmpVenZYQk1UY0xsN0Y1ZXRzTGhLV2FyRThRPQ=="
*/
$info=hhb_curl_exec($curlh,'https://www.pinterest.com');
var_dump(__LINE__,$info);die();
/*
//Cookie:c_dpr=1; __utmt=1; __utma=229774877.1252202543.1424620619.1424620619.1424620619.1; __utmb=229774877.5.7.1424620619; __utmc=229774877; __utmz=229774877.1424620619.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _b="AQ3m6m5qQAVDaIkyqRoJYJ9ecazmK4aobP3PczTxb/BtXObCwlC/5kusK9/Ymj2luo8="; csrftoken=EitE4BCiLq3sz0hf5lHtCx6uNvyIaalo; _pinterest_sess="TWc9PSZLclVramZrWGRUMVYzZW1ZbmxXTXFXeWpHU2ZOVFBFNmFUOXU2ZlNJWFJ0TkdzTy9TZ1RQdmxtNmxZa0JNWnliR2VRS2t6UTRZVGtSVnNySlJKRVBEUjh4K3FWR2gxYi8yS3AxTmhqWW9COWZWaGJiK1Q2Y29ydjhLSGRDb2srdTdHaVh6RU12SEZnVmxlM09UNEloQU9JKzQxRDNqOFlISHRHZ0hIVW9kTUttWlhEd1BOaTJnbHZYTDZ5enBRSGtubDJaSnNKSjlzaG9SaWsrMFZaenhLeWpVaElxbTdZOG1sa3ZGeWQ3MWNFQy81YmtHQkxsZDlBQVNEK1FTUUJEYWZqV2tUMzVDVVM4R1VXL0lCOHZ3MHhPcC81YVZjOWRnSkZoTXFVQXRLU21OK05PZmtFczNvY2ZGdVRMS2pWdXR0WG8wakJTeTdYNlRqV3NZVmtHQzBsM0VyVnhVeXIzVkRWdXlqT3Q1eFVqWUJwVkxuR1ZwY3M5YXJBU2xKQ0lua3U1UkRxYWk0c0lVR1lJcHpMOUZNQXo0ZWlRSDRlaGVSa3NUaEFnREl2Q2lvN0xQc05DNjk5emNESDdaM3YxVmFwNU9KVFhLUGJBVStLcVZjVk1pMlREa3JzcW1FWEdSMGF0cXdvTlpGaz0mYmpVenZYQk1UY0xsN0Y1ZXRzTGhLV2FyRThRPQ=="
Response Headersview source
Accept-Ranges:bytes
Cache-Control:no-cache, no-store, must-revalidate, max-age=0
Connection:keep-alive
Content-Encoding:gzip
Content-Length:348
Content-Type:application/json; charset=utf-8
Date:Sun, 22 Feb 2015 15:57:42 GMT
Expires:Thu, 01 Jan 1970 00:00:00 GMT
Pinterest-Breed:CORGI
Pinterest-Generated-By:ngapp2-1af98e48
Pinterest-Version:7c24931
Pragma:no-cache
Server:nginx
Set-Cookie:_pinterest_pfob=disabled; Domain=.pinterest.com; expires=Wed, 21-Feb-2018 15:57:42 GMT; Max-Age=94607999; Path=/
Vary:User-Agent, Accept-Encoding
*/
function hhb_curl_init($custom_options_array = array()) {
if(empty($custom_options_array)){
$custom_options_array=array();
//i feel kinda bad about this.. argv[1] of curl_init wants a string(url), or NULL
//at least i want to allow NULL aswell :/
}
if (!is_array($custom_options_array)) {
throw new InvalidArgumentException('$custom_options_array must be an array!');
};
$options_array = array(
CURLOPT_AUTOREFERER => true,
CURLOPT_BINARYTRANSFER => true,
CURLOPT_COOKIESESSION => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_FORBID_REUSE => false,
CURLOPT_HTTPGET => true,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_TIMEOUT => 11,
CURLOPT_ENCODING=>"",
CURLOPT_REFERER=>'example.org',
CURLOPT_USERAGENT=>'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36'
);
if (!array_key_exists(CURLOPT_COOKIEFILE, $custom_options_array)) {
//do this only conditionally because tmpfile() call..
static $curl_cookiefiles_arr=array();//workaround for https://bugs.php.net/bug.php?id=66014
$curl_cookiefiles_arr[]=$options_array[CURLOPT_COOKIEFILE] = tmpfile();
$options_array[CURLOPT_COOKIEFILE] =stream_get_meta_data($options_array[CURLOPT_COOKIEFILE]);
$options_array[CURLOPT_COOKIEFILE]=$options_array[CURLOPT_COOKIEFILE]['uri'];
}
//we can't use array_merge() because of how it handles integer-keys, it would/could cause corruption
foreach($custom_options_array as $key => $val) {
$options_array[$key] = $val;
}
unset($key, $val, $custom_options_array);
$curl = curl_init();
curl_setopt_array($curl, $options_array);
return $curl;
}
$hhb_curl_domainCache = "";
function hhb_curl_exec($ch, $url) {
global $hhb_curl_domainCache; //
//$hhb_curl_domainCache=&$this->hhb_curl_domainCache;
//$ch=&$this->curlh;
if(!is_resource($ch) || get_resource_type($ch)!=='curl')
{
throw new InvalidArgumentException('$ch must be a curl handle!');
}
if(!is_string($url))
{
throw new InvalidArgumentException('$url must be a string!');
}
$tmpvar = "";
if (parse_url($url, PHP_URL_HOST) === null) {
if (substr($url, 0, 1) !== '/') {
$url = $hhb_curl_domainCache.'/'.$url;
} else {
$url = $hhb_curl_domainCache.$url;
}
};
curl_setopt($ch, CURLOPT_URL, $url);
$html = curl_exec($ch);
if (curl_errno($ch)) {
throw new Exception('Curl error (curl_errno='.curl_errno($ch).') on url '.var_export($url, true).': '.curl_error($ch));
// echo 'Curl error: ' . curl_error($ch);
}
if ($html === '' && 203 != ($tmpvar = curl_getinfo($ch, CURLINFO_HTTP_CODE)) /*203 is "success, but no output"..*/ ) {
throw new Exception('Curl returned nothing for '.var_export($url, true).' but HTTP_RESPONSE_CODE was '.var_export($tmpvar, true));
};
//remember that curl (usually) auto-follows the "Location: " http redirects..
$hhb_curl_domainCache = parse_url(curl_getinfo($ch, CURLINFO_EFFECTIVE_URL), PHP_URL_HOST);
return $html;
}
you can see the code live in action here: http://codepad.viper-7.com/D8qk6q (for a few days until the server delete the code, anyway. or until some internet dickhead changes the password. it's a throwaway account anyway, obviously)
I'm pretty sure this is not going to work without getting an request_identifier which is required.
To explain, when you load the page you get an unique number for that 'session' which is compared when you are going to log in. This is for a reason to avoid CSRF (Cross-Site Request Forgery).
If you examine the actual POST, you will notice that not only an username or password are posted, but a few items more.
I think you should use https instead of http
$ch = curl_init('https://www.pinterest.com/'); // <-- HERE
and comment this line :
// $login_header[] = 'X-CSRFToken: ' . $csrf_token[1];
While trying to use cUrl with the Post method in Yii 2, I receive a 400 error code.
Bad Request (#400)
Unable to verify your data submission.
The above error occurred while the Web server was processing your request.
Please contact us if you think this is a server error. Thank you.
This is my code, where i instantiate the CurlTool class:
public function actionSend() {
$model = new \app\models\Licitatie;
if ($model->load(Yii::$app->request->post())) {
$curl_tool = new \common\components\CurlTool();
$result = $curl_tool->fetchContent('http://www.william.ro/licitatia_bursa/frontend/web/index.php/organizator/licitatie/evrika', $model->attributes);
print_r($result);
}
}
public function actionEvrika() {
return json_encode(
array(
'a' => 'b',
)
);
}
this is the curltool class code:
<?php
namespace common\components;
class CurlTool {
public static $userAgents = array(
'FireFox3' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; pl; rv:1.9) Gecko/2008052906 Firefox/3.0',
'GoogleBot' => 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
'IE7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
'Netscape' => 'Mozilla/4.8 [en] (Windows NT 6.0; U)',
'Opera' => 'Opera/9.25 (Windows NT 6.0; U; en)'
);
public static $options = array(
CURLOPT_USERAGENT => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
CURLOPT_AUTOREFERER => true,
CURLOPT_FOLLOWLOCATION => false,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FRESH_CONNECT => true,
CURLOPT_COOKIEJAR => "cookies.txt",
CURLOPT_COOKIEFILE => "cookies.txt",
CURLOPT_SSL_VERIFYPEER => false,
//CURLOPT_COOKIESESSION => false,
);
private static $proxyServers = array();
private static $proxyCount = 0;
private static $currentProxyIndex = 0;
public static $getinfo;
public static function addProxyServer($url) {
self::$proxyServers[] = $url;
++self::$proxyCount;
}
public static function fetchContent($url, $fields = null, $verbose = false) {
//print '*'.$fields.'*';
if (($curl = curl_init($url)) == false) {
throw new Exception("curl_init error for url $url.");
}
if (self::$proxyCount > 0) {
$proxy = self::$proxyServers[self::$currentProxyIndex++ % self::$proxyCount];
curl_setopt($curl, CURLOPT_PROXY, $proxy);
if ($verbose === true) {
echo "Reading $url [Proxy: $proxy] ... ";
}
} else if ($verbose === true) {
echo "Reading $url ... ";
}
//$verbose=TRUE;
//print_r($fields);
// debug_print_backtrace();
//url-ify the data for the POST
$fields_string = '';
if (is_array($fields))
foreach ($fields as $key => $value) {
if (empty($key))
continue;
$fields_string .= $key . '=' . urlencode($value) . '&';
if ($verbose === true) {
echo $key . ": " . $value;
}
}
rtrim($fields_string, '&');
if (count($fields) > 0) {
curl_setopt($curl, CURLOPT_POST, count($fields));
curl_setopt($curl, CURLOPT_POSTFIELDS, $fields_string);
}
if ($verbose === true) {
echo "Fields string $fields_string ... ";
}
curl_setopt_array($curl, self::$options);
$content = curl_exec($curl);
self::$getinfo = curl_getinfo($curl);
if ($content === false) {
throw new Exception("curl_exec error for url $url " . curl_error($curl));
}
curl_close($curl);
if ($verbose === true) {
echo "Done.\n";
}
$content = preg_replace('#\n+#', ' ', $content);
$content = preg_replace('#\s+#', ' ', $content);
return $content;
}
}
class Controller extends \yii\base\Controller
{
/**
* #var boolean whether to enable CSRF validation for the actions in this controller.
* CSRF validation is enabled only when both this property and [[Request::enableCsrfValidation]] are true.
*/
public $enableCsrfValidation = false; <- set this to false
...
be careful, i just found out that if this setting is used within the action itself, it might fail;
it might fail within beforeaction;
you can disable csrf in beforeaction
public function beforeAction($action)
{
if($action->id == 'source-in')
{
return true;
}
return parent::beforeAction($action);
}
Is there another way to fix it? Maybe generate a new CSRF token and send it in the POST data?