Php curl returning blank page for some sites where curl command working fine.
for example : curl www.wikipedia.org generating output but php curl giving blank pages with <html> tags
$ch = curl_init(); // initialize curl with given url
//TBD: all setopt commands return true/false. Should be handled
//curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER["HTTP_USER_AGENT"]); // set useragent
$res = curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); // write the response to a variable
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // follow redirects if any
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout); // max. seconds to execute
curl_setopt($ch, CURLOPT_FAILONERROR, 0); // stop when it encounters an error
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // iNetClean is web-crawling, no need to verify certificates
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
curl_setopt($ch, CURLOPT_ENCODING, ""); // accept all encodings - identity, deflate, gzip
// now set the URL
curl_setopt($ch, CURLOPT_URL, $url);
$fp_out = fopen($html_file, 'w');
if (!$fp_out) {
if ($DEBUG) {
error_log("couldn't create " . $html_file);
}
} else {
if ($DEBUG) {
error_log("file created " . $html_file);
}
}
$fp_err = fopen($html_err_file, 'w');
if (!$fp_err) {
if ($DEBUG) {
error_log("couldn't create " . $html_err_file);
}
} else {
if ($DEBUG) {
error_log("error file created " . $html_err_file);
}
}
curl_setopt($ch, CURLOPT_FILE, $fp_out); //rawurlencode($url) . "txt"); // for debugging only
curl_setopt($ch, CURLOPT_STDERR, $fp_err);
$result = curl_exec($ch);
//0 size file is created if no data is downloaded or URL does not exist such as pron00.com. Hence added handler to such errors.
if (#filesize($html_file) > 0) {
//file exists and contain some data
} else {
return false;
}
if ($result == false) {
trigger_error(curl_error($ch));
if ($DEBUG) {
error_log("Curl_exec fail");
}
return false;
}
fclose($fp_out);
fclose($fp_err);
curl_close($ch);
return $result;
You can do this using below code:
<?php
$debug = 1;
$fb_page_url = "http://www.wikipedia.org";
$cookies = 'cookies.txt';
touch($cookies);
$uagent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/36.0.1985.125 Chrome/36.0.1985.125 Safari/537.36';
/**
Get __VIEWSTATE & __EVENTVALIDATION
*/
$ch = curl_init($fb_page_url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookies);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, $uagent);
$html = curl_exec($ch);
curl_close($ch);
preg_match('~<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="(.*?)" />~', $html, $viewstate);
preg_match('~<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="(.*?)" />~', $html, $eventValidation);
$viewstate = $viewstate[1];
$eventValidation = $eventValidation[1];
/**
Start Fetching process
*/
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $fb_page_url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookies);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookies);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 9850);
curl_setopt($ch, CURLOPT_USERAGENT, $uagent);
// Collecting all POST fields
$postfields = array();
//$postfields['__EVENTTARGET'] = ""; //this is for further clicking any link
//$postfields['__EVENTARGUMENT'] = ""; //this is for further clicking any link
$postfields['__LASTFOCUS'] = "";
$postfields['__VIEWSTATE'] = $viewstate;
$postfields['__EVENTVALIDATION'] = $eventValidation;
$postfields['hidStates'] = "";
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postfields);
$ret = curl_exec($ch); // Get result as fetched web page.
if ($debug) {
echo $ret;
}
curl_close($ch);
?>
Related
The form can be fount here: https://services.smartree.com/mcdonalds/applicationform/WebForms/ApplyToJob.aspx
After I get __VIEWSTATE & __EVENTVALIDATION I start the submit process.
// Start Submit process
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEJAR, $ckfile);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt($ch, CURLOPT_HEADER, FALSE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_REFERER, $url);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_STDERR, $f);
curl_setopt($ch, CURLOPT_HTTPHEADER, $request);
curl_setopt($ch, CURLOPT_USERAGENT, $useragent);
// Populate all POST fields
$postfields = array();
$postfields['__EVENTVALIDATION'] = urlencode($eventValidation);
$postfields['__VIEWSTATEENCRYPTED'] = urlencode('');
$postfields['__VIEWSTATEGENERATOR'] = urlencode($eventGenerator);
$postfields['ucApplyToJob$cmbSelectedJob']=urlencode($values['job_type']);
$postfields['ucApplyToJob$txtLastName'] = $values['lname'];
$postfields['ucApplyToJob$txtFirstName'] = $values['name'];
$p = "";
foreach ($postfields as $k => $v) {
$p .= $k . '=' . $v . '&';
}
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postfields);
$ret = curl_exec($ch); // Get result after login page.
$err = curl_errno($ch);
$errmsg = curl_error($ch);
$header = curl_getinfo($ch);
curl_close($ch);
$header['errno'] = $err;
$header['errmsg'] = $errmsg;
$header['content'] = $ret;
print_r($header);
I have populated all required fields but nothing. Please help.
I'm trying to keep the session along different requests with curl when login into twitter because at some point it logs out from Twitter
$sTarget = "https://twitter.com";
curl_setopt($ch, CURLOPT_URL, $sTarget);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_COOKIEFILE, "/tmp/cookie". $usuario ."_tweet.txt");
curl_setopt($ch, CURLOPT_COOKIEJAR, $usuario ."_tweet.txt");
curl_setopt($ch, CURLOPT_COOKIESESSION, true);
curl_setopt($ch, CURLOPT_REFERER, $sTarget);
curl_setopt($ch, CURLOPT_HEADER, TRUE);
$html = curl_exec($ch);
if(curl_errno($ch))
{
echo 'error:' . curl_error($c);
}
preg_match('<input name="authenticity_token" type="hidden" value="([a-zA-Z0-9]*)">', $html, $match);
$authenticity_token = $match[1];
if ($authenticity_token == "");
{
preg_match('<input type="hidden" value="([a-zA-Z0-9]*)" name="authenticity_token">', $html, $matchprima);
$authenticity_token = $matchprima[1];
}
$username = $usuario;
$password = "*******";
$sPost = "session[username_or_email]=$username&session[password]=$password&return_to_ssl=true&scribe_log=&redirect_after_login=%2F&authenticity_token=$authenticity_token";
$sTarget = "https://twitter.com/sessions";
curl_setopt($ch, CURLOPT_URL, $sTarget);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $sPost);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, array("Content-type: application/x-www-form-urlencoded"));
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_HEADER, TRUE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
# display server response
$htmldos = curl_exec($ch);
preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $htmldos, $matches);
$cookies = array();
foreach($matches[1] as $item) {
parse_str($item, $cookie);
$cookies = array_merge($cookies, $cookie);
}
var_dump($cookies);
if(curl_errno($ch))
{
echo 'error:' . curl_error($ch);
}
But when doing a var_dump in cookies to see if the cookies values are the same, under ["_twitter_sess"] I get different strings in the 2 requests. Shouldn't I get the same string if the session is kept between requests? If so, how do I do that?
This code works for logging in Twitter, and keeping session as for doing other things such after it as sending tweets. I've been using it for 6 months without problem. You need simple_html_dom.php from PHP Simple HTML DOM Parser for it to work
<?php
include_once('simple_html_dom.php');
global $usuario;
$usuario = "universohumor2"; //your twitter account username
$ch = curl_init();
/************************************** COMIENZAN LAS LLAMADAS A FUNCIONES *************************************/
/***************************************************************************************************************/
$vector_retorno = logueo_twitter($ch);
/******************************************************************************************************************/
function logueo_twitter($ch)
{
//$ch = curl_init();
global $usuario;
$sTarget = "https://twitter.com";
curl_setopt($ch, CURLOPT_URL, $sTarget);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_COOKIEFILE, "/tmp/cookie". $usuario ."_tweet.txt");
curl_setopt($ch, CURLOPT_COOKIESESSION, true);
curl_setopt($ch, CURLOPT_REFERER, $sTarget);
curl_setopt($ch, CURLOPT_HEADER, TRUE);
$html = curl_exec($ch);
if(curl_errno($ch))
{
echo 'error:' . curl_error($c);
}
preg_match('<input name="authenticity_token" type="hidden" value="([a-zA-Z0-9]*)">', $html, $match);
$authenticity_token = $match[1];
if ($authenticity_token == "");
{
preg_match('<input type="hidden" value="([a-zA-Z0-9]*)" name="authenticity_token">', $html, $matchprima);
$authenticity_token = $matchprima[1];
}
$username = $usuario;
$password = "password"; // your twitter account password
$sPost = "session[username_or_email]=$username&session[password]=$password&return_to_ssl=true&scribe_log=&redirect_after_login=%2F&authenticity_token=$authenticity_token";
$sTarget = "https://twitter.com/sessions";
curl_setopt($ch, CURLOPT_URL, $sTarget);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $sPost);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, array("Content-type: application/x-www-form-urlencoded"));
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_HEADER, TRUE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
# display server response
$htmldos = curl_exec($ch);
//echo $htmldos;
if(curl_errno($ch))
{
echo 'error:' . curl_error($ch);
}
$vector["sPost"] = $sPost;
$vector["authenticity_token"] = $authenticity_token;
return $vector;
}
?>
I'm trying to log in to site and then go to one of the pages and retrieve the data. I have a problem with authorization.
Sample code:
$loginUrl = 'https://strona.pl/authorization'; //action from the login form
$loginFields = array('username'=>'mail#mojmail.pl', 'password'=>'haslo'); //login form field names and values
$remotePageUrl = 'https://strona.pl/pdstrona'; //url of the page you want to save
$login = getUrl($loginUrl, 'post', $loginFields); //login to the site
$remotePage = getUrl($remotePageUrl); //get the remote page
function getUrl($url, $method='', $vars='') {
$ch = curl_init();
if ($method == 'post') {
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $vars);
}
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookies/cookies.txt');
curl_setopt($ch, CURLOPT_COOKIEFILE, 'cookies/cookies.txt');
$buffer = curl_exec($ch);
//jeżeli pojawił się błąd to go wyświetlimy
if(curl_exec($ch) === false)
{
echo 'Curl error: ' . curl_error($ch);
}
curl_close($ch);
return $buffer;
print $status;
}
// wyłącza pokazywanie błędów dla funkcji loadHTML
libxml_use_internal_errors(true);
$dom = new DOMDocument;
$dom ->strictErrorChecking = FALSE;
$dom->loadHTML($remotePage);
$xpath = new DOMXpath($dom);
As a result of this query I am redirected to the main page (not the sub). What is important, I enter email address into a script in encoded form. Example: email% 40mojmail.pl
edit
Ok - I will add some information . I am a publisher and working with afilo.pl . Daily checking of rates is very tiring, so I wanted to prepare a script that will collect data once a day and informed me of the changes.
Unfortunately I can not retrieve the data.
Sample cookies from my browser :
Set- Cookie: PHPSESSID = jat102p33s0pmfairri1qiih24 ; expires = Thu, 25 -Dec- 2013 9:16:40 p.m. GMT ; path = / ; domain = . Afilo.pl
I modified the code but it still does not work.
loginUrl = 'https://opentrack.afilo.pl/logowanie'; //action from the login form
$loginFields = array('loginemail'=>'.......','loginhaslo'=>'........');
//login form field names and values
$remotePageUrl = 'https://opentrack.afilo.pl/partner/programy-lista'; //url of the page you want to save
$login = getUrl($loginUrl, 'post', $loginFields); //login to the site
$remotePage = getUrl($remotePageUrl); //get the remote page
function getUrl($url, $method='', $vars='') {
$ch = curl_init();
if ($method == 'post') {
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $vars);
}
session_start();
$strCookie = session_name() . '=' . $_COOKIE[ session_name() ] . '; path=/; domain=.afilo.pl';
session_write_close();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
//curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookies/cookies.txt');
//curl_setopt($ch, CURLOPT_COOKIEFILE, 'cookies/cookies.txt');
curl_setopt( $ch, CURLOPT_COOKIE, $strCookie );
$buffer = curl_exec($ch);
//je¿eli pojawi³ siê b³¹d to go wyœwietlimy
if(curl_exec($ch) === false)
{
echo 'Curl error: ' . curl_error($ch);
}
curl_close($ch);
return $buffer;
print $status;
}
This is the answer to my question. This code works well.
// options
$EMAIL = 'login';
$PASSWORD = 'haslo';
$cookie_file_path = "/cookies/cookies.txt";
$LOGINURL = "https://domainname.com/auth";
$agent = "Nokia-Communicator-WWW-Browser/2.0 (Geos 3.0 Nokia-9000i)";
$ch = curl_init();
$headers[] = "Accept: */*";
$headers[] = "Connection: Keep-Alive";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_file_path);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file_path);
curl_setopt($ch, CURLOPT_URL, $LOGINURL);
$fields = array();
$fields['loginemail'] = $EMAIL;
$fields['loginhaslo'] = $PASSWORD;
$POSTFIELDS = http_build_query($fields);
curl_setopt($ch, CURLOPT_URL, $LOGINURL);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $POSTFIELDS);
$result = curl_exec($ch);
$remotePageUrl = 'https://domainname.com/partner/programy-lista';
curl_setopt($ch, CURLOPT_URL, $remotePageUrl);
$result = curl_exec($ch);
function query($url, $pfields = 0, $cookie = 0)
{
curl_setopt($ch, CURLOPT_HEADER, 1);
if (!empty($pfields))
{
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $pfields);
}
if (!empty($cookie))
{
curl_setopt($ch, CURLOPT_COOKIE, $cookie);
}
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_ENCODING,'gzip');
if (!$login)
{
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
}
$content = curl_exec($ch);
return $content;
}
$cookie = 'sessionID=3864cab58412ec567b634db3c317898;OAGEO=RU%7C%7C%7C%7C%7C%7C%7C%7C%7C%7C;';
$p = '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++';
$post = 'clientid=23&campaignid=52&bannerid=111&appendsave=1&appendtype=0&append=' . urlencode($p) . '&submitbutton=';
echo query('http://example.com/in.php', $post, $cookie);
This code is returned 417 error(
BUT $p is not usage urlencode but IS OK but +(plus) change for " "(space)
Sooooorry for my very bad english
Try adding this:
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Expect:'));
I'm creating a PHP function to connect to pingomatic using CURL but the response is always.
Array ( [EXE] => XML-RPC server accepts POST requests only. )
here is my sample code...
function curl_getpage2($url,$data, $referer = null, $agent = null, $header = null, $timeout = 20, $proxy = null, $proxy_username = null, $proxy_password = null) {
//getProxy();
if ($agent == null) {
$agent = getAgent();
}
if ($referer == null) {
$referer = getHost($url);
}
if (!is_array($header)) {
$header = array("Content-Type:text/xml","Host:".getHost($url),"User-Agent:$agent",
"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language:en-us,en;q=0.5",
"Accept-Encoding:gzip,deflate",
"Accept-Charset:ISO-8859-1,utf-8;q=0.7,*;q=0.7",
"Keep-Alive:300",
"Connection:keep-alive",
"Cache-Control:max-age=0",
"Content-length: ".strlen($XML));
}
if($proxy == null){
$proxy = "208.100.27.155:60099";
}
if($proxy_username == null && $proxy_password == null){
$proxyUnPW = "unproxy:pwproxy";
}else{
$proxyUnPW = $proxy_username.":".$proxy_password;
}
$ch = curl_init();
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
curl_setopt($ch, CURLOPT_HTTPHEADER,$header);
curl_setopt($ch, CURLOPT_PROXY, $proxy);
curl_setopt($ch, CURLOPT_PROXYUSERPWD,$proxyUnPW);
curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1);
//curl_setopt($ch, CURLOPT_USERAGENT, $agent);
//curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
//curl_setopt($ch, CURLOPT_REFERER, $referer);
//curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
//curl_setopt($ch, CURLOPT_AUTOREFERER,TRUE);
//curl_setopt($ch, CURLOPT_MAXREDIRS, 10);
//curl_setopt($ch, CURLOPT_COOKIEJAR, '/tmp/cookies.txt');
//curl_setopt($ch, CURLOPT_COOKIEFILE, '/tmp/cookies.txt');
//curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
//curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
$result['EXE'] = curl_exec($ch);
//$result['INF'] = curl_getinfo($ch);
//$result['ERR'] = curl_error($ch);
curl_close($ch);
return $result;
}
I need some help here. I'm just somewhat new to PHP.
You can submit your site once via the web form and then call a GET request for that url (see
question How to ping automatically to pingomatic in PHP?).
So your URL would be something like the following if you want to ping all services (title and url encoded with urlencode()):
$url = 'http://pingomatic.com/ping/?title='.$title.'&blogurl='.$url.'&rssurl=&chk_weblogscom=on&chk_blogs=on&chk_feedburner=on&chk_newsgator=on&chk_myyahoo=on&chk_pubsubcom=on&chk_blogdigger=on&chk_weblogalot=on&chk_newsisfree=on&chk_topicexchange=on&chk_google=on&chk_tailrank=on&chk_skygrid=on&chk_collecta=on&chk_superfeedr=on');
eg:
http://pingomatic.com/ping/?title=Example%20Title&blogurl=http%3A%2F%2Fwww.example.com%2F&rssurl=&chk_weblogscom=on&chk_blogs=on&chk_feedburner=on&chk_newsgator=on&chk_myyahoo=on&chk_pubsubcom=on&chk_blogdigger=on&chk_weblogalot=on&chk_newsisfree=on&chk_topicexchange=on&chk_google=on&chk_tailrank=on&chk_skygrid=on&chk_collecta=on&chk_superfeedr=on