$cookiefile = 'cookie.txt';
$params = 'page=http%3A%2F%2Fstackoverflow.com%2F&x=50&y=40';
$url='https://kproxy.com/doproxy.jsp';
$ch = curl_init();
curl_setopt($ch,CURLOPT_URL,$url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch,CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_POST, count($params));
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0');
curl_setopt($ch, CURLOPT_POSTFIELDS, $params);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookiefile);
echo curl_exec($ch);
curl_close($ch);
/// at the end of the execution redirect me to this url
///
https://server4.kproxy.com/servlet/redirect.srv/sruj/sfywreskebutnn/p1/
what I want is to know how to keep the url in a data base or a txt file or save in variable sessions
You can get the redirect url as follows:
$redirectUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
Related
I am trying to login into a site using php curl but in page have multi csrf_token from form.
i was try with DOM and have 4 csrf_token value, this website for login https://www.useetv.com/login
an here is my code
$url = 'https://www.useetv.com/login';
$username = "email";
$password = "pass";
$cookie_file_path = "cookie.txt";
$agent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4098.0 Safari/537.36";
$POSTDATA = "email=".$username."&password=".$password."&csrf_token=value from hidden input";
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file_path);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_close ($ch);
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_POST, 1 );
curl_setopt($ch, CURLOPT_POSTFIELDS, $POSTDATA);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_file_path);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file_path);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
$content = curl_exec ($ch);
print_r($content);
curl_close ($ch);
how to get csrf_token from true form ?
please write with right code, thanks advance
to scrape data from other site or API I am using below code but problem is that when URL has other languages font than its getting error. it's not a problem when URL having the English language
$url='https://techblogs.site/अगर-हम-इन-7-फीचर्स-को-2019-में-पा/';
function ftch($url,$post="", $ck = 'cookie.dat')
{
$ver = rand(4, 6);
$agent = 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36';
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, '' . $url . '');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
curl_setopt($ch, CURLOPT_REFERER, $url);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_COOKIESESSION, 'cookie.s');
//curl_setopt($ch, CURLOPT_PROXY, $proxyIP);
curl_setopt($ch, CURLOPT_COOKIEJAR, $ck);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ck);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_TIMEOUT,2);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT,2);
curl_setopt($ch, CURLOPT_HTTPHEADER, array(
'Accept-Language: en-us,en;q=0.7,de-de;q=0.3',
'Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5'
));
$content = curl_exec($ch);
curl_close($ch);
return $content;
}
$content=ftch($url);
echo $content;
You should be able to encode the URL (the parts that are not UTF-8) and that will allow the connection to get through.
<?php
$url = 'https://techblogs.site/'.urlencode("अगर-हम-इन-7-फीचर्स-को-2019-में-पा");
Im trying to make an application that do a login via CURL, but i cant make this application to work, with the cookie saved after the login. Aparently i can save the cookie, but can use it. I dont know if im doing wrong in the login code or the way to use the cookie saved. This application already work if i get the cookie manually via Chrome after login. I think it generate a primary cookie after the login at this page: https://www.comprasnet.gov.br/seguro/loginPortalFornecedor.asp, that redirects to this one https://www.comprasnet.gov.br/intro.htm, generating the first cookie. After i hit the menu to this page: https://www.comprasnet.gov.br/pregao/fornec/Acompanhar.asp, it generates the two cookies needed to work.
Code i made to try to do the login:
<?php
$cookieold = __DIR__.DIRECTORY_SEPARATOR.'cookieJar.txt';
$cookieactive = __DIR__.DIRECTORY_SEPARATOR.'cookieactive.txt';
$ch = curl_init();
$url = "https://www.comprasnet.gov.br/seguro/loginPortalFornecedor.asp";
curl_setopt($ch, CURLOPT_URL,$url);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0');
curl_setopt($ch, CURLOPT_SSLVERSION, 3);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIESESSION, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, "perfil=Fornecedor&txtLogin=seulogin&txtSenha=suasenha&ambiente=Produção");
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookieold);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
$output = curl_exec($ch);
// Define uma nova URL para ser chamada (após o login)
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0');
curl_setopt($ch, CURLOPT_SSLVERSION, 3);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookieold);
curl_setopt($ch, CURLOPT_URL, 'https://www.comprasnet.gov.br/intro.htm');
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
// Executa a segunda requisição
$content = curl_exec ($ch);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0');
curl_setopt($ch, CURLOPT_SSLVERSION, 3);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookieold);
curl_setopt($ch, CURLOPT_URL, 'https://www.comprasnet.gov.br/pregao/fornec/Acompanhar.asp');
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookieactive);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
// Executa a terceira requisição
$content2 = curl_exec ($ch);
if ($output === FALSE) { // verifica erros no curl
echo "cURL Error: " . curl_error($ch);
}
curl_close($ch);
//echo $content2;
?>
Code that i already use with the cookie a get manually:
// iniciar sessao
$ch = curl_init();
// opcoes
curl_setopt($ch, CURLOPT_URL, "https://www.comprasnet.gov.br/pregao/fornec/Acompanhar.asp");
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0');
curl_setopt($ch, CURLOPT_SSLVERSION, 3);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_COOKIESESSION, true);
curl_setopt($ch, CURLOPT_COOKIE, 'ASPSESSIONIDQGAADDTC=DDLGMJABKGCABIKENDGOBLJJ;ASPSESSIONIDQGCADASC=LFFGLJABPEAKKIOLNLLCOFIC');
//executar sessao
$output = curl_exec($ch);
if ($output === FALSE) { // verifica erros no curl
echo "cURL Error: " . curl_error($ch);
}
$output = strtr($output, $conversao); // função de conversão acentuado/não acentuado
//fechar sessao
curl_close($ch);
Any help will be aprecciate. Im not a programmer, just a curious guy trying to make my job easier. This application im making id for personnal use only.
my suggestion is to curl_close / curl_init after each output line. You are carrying over settings from the previous curl session like curl_setopt($ch, CURLOPT_COOKIESESSION, true) and CURLOPT_POSTFIELDS
so
$output = curl_exec($ch);
curl_close($ch);
$ch = curl_init();
$content = curl_exec($ch);
curl_close($ch);
$ch = curl_init();
I am trying to log into an .asp site with a https login page from PHP. I can't get logged into the site . There dosen't seem to be a cookie generated, viewstate, etc nor does it work leaving these parameters out. The form fields seem to fill correctly (i can physically see the login name) but im not sure about the password field which is a password type, but i don't think there is a issue there, its correctly spelled etc.
I have tried all the related posts including http://www.mishainthecloud.com/2009/12/screen-scraping-aspnet-application-in.html?showComment=1368565341638#c9104469935977149435
My code is below and returning "not Found" (error code 7 i think..) on the final call. No curl errors are present on the 1st two calls.
Can anyone help with this?
$ckfile = tempnam ("/tmp", "CURLCOOKIE");
// URL to login page
$url = "https://secure2.clubwise.com/glenview/memberlogin.asp";
// Get Login page and its cookies/ viewstate , etc
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
//curl_setopt($ch, CURLOPT_COOKIEJAR, $ckfile); // no cookie stored
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
$output = curl_exec($ch);
//print(esc_html($output));
$viewstate = regexExtract($output,$regexViewstate,$regs,1);
$eventval = regexExtract($output, $regexEventVal,$regs,1);
// rebuild post info -- view state and eventvalidate empty! cant find on page
$fields_string =
'__VIEWSTATE='.rawurlencode($viewstate).
'&__EVENTVALIDATION='.rawurlencode($eventval).
'&login='.urlencode('xxx#xxx.com').
'&password='.urlencode('xxxx').
'&submit='.urlencode('Sign in').
'&redirect=';
echo $fields_string;
// Post login form -- password field ok?
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, 5);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields_string);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
//curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); // Tells cURL to follow redirects
$outputb = curl_exec($ch);
print curl_error;
//var_dump($outputb);
$url = "https://secure2.clubwise.com/glenview/bookclass.asp";
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
//curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$outputc = curl_exec($ch);
print curl_error;
var_dump($outputc);
header information:
//$header= array(
//'HTTP/1.1 200 OK',
//'Date: Mon, 09 Jun 2014 00:55:17',
//'GMT Server: Microsoft-IIS/6.0',
//'X-Powered-By: ASP.NET',
//'Content-Length: 21035',
//'Content-Type: text/html',
//'Set-Cookie: ASPSESSIONIDCCSBSTDC=IHJBDOOBIDMJDDOFLAOOBENL; path=/',
//'Cache-control: private'
//);
SOLUTION: it was curl_init() on each operation that was causing this to break. Viewstate, headers are not needed.
$ourFileName = get_stylesheet_directory()."/cookieFile.txt";
$ckfile = $ourFileName;
// URL to login page
$url = "https://secure2.clubwise.com/glenview/memberlogin.asp";
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $ckfile); // Stores cookies in the temp file
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_HEADER, 1);
$output = curl_exec($ch);
//Cookie obtained, login
$fields_string =
'&redirect='.
'&login='.urlencode('vvv#xxx.com').
'&password='.urlencode('xxx').
'&submit='.urlencode('"Sign in"' )
;
//cookie in the header + header not required
/*
$cookielines =file($ckfile);
foreach($cookielines as $row) {
if($row[0] != '#') {
$cookie=$row;
}
}
$header= array( // not needed at moment
'HTTP/1.1 200 OK',
'Date: Mon, 09 Jun 2014 00:55:17',
'GMT Server: Microsoft-IIS/6.0',
'X-Powered-By: ASP.NET',
'Content-Length: 21035',
'Content-Type: text/html',
'Set-Cookie: $cookie; path=/',
'Cache-control: private'
);
*/
$url = "https://secure2.clubwise.com/glenview/memberlogin.asp";
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, 4);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields_string);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile); //Uses cookies from the temp file
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); // Tells cURL to follow redirects
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
curl_setopt($ch, CURLOPT_ENCODING,"");
curl_setopt($ch, CURLOPT_REFERER,"https://secure2.clubwise.com/glenview/memberlogin.asp");
$outputb = curl_exec($ch);
$err = curl_errno ( $ch ); echo "<br>error=".$err;
$errmsg = curl_error ( $ch ); echo "<br>errmsg=".$errmsg;
$header = curl_getinfo ( $ch ); echo "<br>header="; var_dump($header);
$httpCode = curl_getinfo ( $ch, CURLINFO_HTTP_CODE ); echo "<br>httpcode=".$httpCode;
print curl_error;
//Now you should be able to access any pages within the password-restricted area by just including the cookies for each call:
$url = "https://secure2.clubwise.com/glenview/bookclass.asp?Mode=Area&RecId=67";
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, 0 );
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile); //Uses cookies from the temp file
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$outputc = curl_exec($ch);
print curl_error;
var_dump($outputc);
Please check again that cookie stored on first page. Set CURLOPT_COOKIEFILE and CURLOPT_COOKIEJAR for Session cookies and not reinit or close curl for save a session.
Set CURLOPT_REFERER, some sites check it for login page.
I am trying to get a website source with a curl call but some data is missing and I don't know what I'm doing wrong.
If i make a call to the url in browser in the page source will apear <!-- request captchas: blabla -->..
but after I call in php it will bring in the page source: "<!-- request captchas: null -->"
My code:
$url='http://www.anaf.ro/anaf/internet/ANAF/informatii_publice/informatii_agenti_economici/registrul_inactivi_reactivati/';
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0");
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt( $ch, CURLOPT_AUTOREFERER, true );
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_COOKIE, "JSESSIONID=0001R8jflCwCXeYrmbF45vBxtS-:3KMDRVUOUV");
curl_setopt($ch, CURLOPT_URL, $url);
$html = curl_exec($ch);
curl_close ($ch);
echo $html;
You forgot to add curl_init().
Do like this..
<?php
$url='http://www.anaf.ro/anaf/internet/ANAF/informatii_publice/informatii_agenti_economici/registrul_inactivi_reactivati/';
$ch = curl_init(); //<-------------- Add this here
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0");
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt( $ch, CURLOPT_AUTOREFERER, true );
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_COOKIE, "JSESSIONID=0001R8jflCwCXeYrmbF45vBxtS-:3KMDRVUOUV");
curl_setopt($ch, CURLOPT_URL, $url);
$html = curl_exec($ch);
curl_close ($ch);
echo $html;
Try removing this line:
curl_setopt($ch, CURLOPT_COOKIE, "JSESSIONID=0001R8jflCwCXeYrmbF45vBxtS-:3KMDRVUOUV");