I'm trying to send build my API, but I have to login first.
When I hit login, there's response header and request header. They set cookies, as shown in pictures.
How could I get these cookies into a variable to use it in other file or something like that?
reqested header
responed header
my code
$file = 'cookies.txt';
$buffer = "__RequestVerificationToken=$ver;";
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "http://www.xxxx.org/L/DoLogin");
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0');
curl_setopt($ch, CURLOPT_HEADER ,array("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"));
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, "__RequestVerificationToken=$buffer&username=xxxx&pw=xxxx&rememberMe=true&loginAcc=");
curl_setopt($ch, CURLOPT_RETURNTRANSFER ,1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION ,1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
$content = curl_exec($ch);
// get cookies
$cookies = array();
preg_match_all('/Set-Cookie:(?<cookie>\s{0,}.*)$/im', $content, $cookies);
print_r($cookies['cookie']); // show harvested cookies
// basic parsing of cookie strings (just an example)
$cookieParts = array();
preg_match_all('/Set-Cookie:\s{0,}(?P<name>[^=]*)=(?P<value>[^;]*).*?expires=(?P<expires>[^;]*).*?path=(?P<path>[^;]*).*?domain=(?P<domain>[^\s;]*).*?$/im', $content, $cookieParts);
print_r($cookieParts);
Related
Like the title says, this is really weird, this is supposed to fetch the page below but it instead only fetches the "cookie settings page".
My code:
<?php
// Set the URL of the webpage
//visit this in your browswer to see the actual page
$url = "https://arbetsformedlingen.se/platsbanken/annonser/26981350";
// Set the cookie jar file
$cookie_jar = "cookies.txt";
// Set the form data
$form_data = array(
'search-input' => '26981350'
);
// Initialize cURL
$ch = curl_init();
// Set the URL
curl_setopt($ch, CURLOPT_URL, $url);
// Set the user agent
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36");
// Set the cookie jar file
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_jar);
// Set the cookie file
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_jar);
// Set the form data
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $form_data);
// Follow redirects
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
// Return the response as a string
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
// Execute the request
$content = curl_exec($ch);
// Close cURL
curl_close($ch);
// Print the content of the webpage
echo $content;
?>
Any ideas on getting the actual page? Been banging my head on this with no results :(
Following is full code per your comment which did print the content, notice I made two changes to your code, comment out the post and http_build_query rest is same as you posted
// Set the URL of the webpage
//visit this in your browswer to see the actual page
$url = "https://arbetsformedlingen.se/platsbanken/annonser/26981350";
// Set the cookie jar file
$cookie_jar = "cookies.txt";
// Set the form data
$form_data = array(
'search-input' => '26981350'
);
// Initialize cURL
$ch = curl_init();
// Set the URL
curl_setopt($ch, CURLOPT_URL, $url);
// Set the user agent
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36");
// Set the cookie jar file
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_jar);
// Set the cookie file
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_jar);
// Set the form data
//curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS,http_build_query($form_data));
// Follow redirects
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
// Return the response as a string
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
// Execute the request
$content = curl_exec($ch);
// Close cURL
curl_close($ch);
// Print the content of the webpage
echo $content;
I am Trying to Make A PHP Curl Request to a site that apparently seems to be protected agains Cross-Site Forgery
and have done all my best to edit the headers, parameters and cookies to resemble that of a browser but I still keep getting ERROR 400 (Bad Request) Please I would like to know how to make this work with php curl the code below works on site with and without the crsf protection but doesn't work in the one i am currently trying to access
$url="https://candidate.scholastica.ng/candidate/login";
$cookie_file_path = "cookie.txt";
// I first load the page to save the cookie and the request token
// Below are the required curl settings that works in other sites
$ch = curl_init();
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_NOBODY, false);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file_path);
//set the cookie the site has for certain features, this is optional
// curl_setopt($ch, CURLOPT_COOKIE, "cookiename=0");
curl_setopt($ch, CURLOPT_USERAGENT,"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0");
curl_setopt($ch, CURLOPT_ENCODING,'gzip, deflate');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
// curl_setopt($ch, CURLOPT_REFERER, "http://www.google.com/bot.html");
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
$headers = array("User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.8)
Gecko/20061025 Firefox/1.5.0.8","origin:http://www.google.com/bot.html","accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "accept-language:en-US,en;q=0.5","accept-encoding:gzip, deflate","connection:keep-alive","upgrade-insecure-requests:1","keep-alive:300","accept-charset:ISO-8859-1,utf-8;q=0.7,*;q=0.7");
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_POST, 1);
// curl_setopt($ch, CURLOPT_POSTFIELDS, "");
$details = curl_exec($ch);// I load the Page containing the form
// I remove all newline and return statements from the form to make my regex simple for me
$pattern = '/\r\n*/m';
$replace = '';
$details = preg_replace( $pattern, $replace, $details);
// Regular Expression to grab request token from form page
$pattern = '/<input.+?name="__RequestVerificationToken".+?value="(.*?)".+?>/m';
preg_match($pattern, $details, $matches);
$token = $matches[1];
// I set up urlencoded post form values and get content length
$field1 = urlencode("__RequestVerificationToken") ."=". urlencode($token);
$field2 = urlencode("Username") ."=". urlencode("example#gmail.com");
$field3 = urlencode("Pwd") ."=". urlencode("examplepassword");
$fields = "{$field1}&{$field2}&{$field3}";
$content_length = strlen($fields);
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_NOBODY, false);
curl_setopt($ch, CURLINFO_HEADER_OUT, true);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
curl_setopt($ch, CURLOPT_POST, 1);
// $headers = array("Host:candidate.scholastica.ng",
// "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0",
//I Modify header to add content length
$headers = array("Host:candidate.scholastica.ng","User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0","Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8","Accept-Language:en-US,en;q=0.5","Accept-Encoding:gzip, deflate, br","Connection:keep-alive","Upgrade-Insecure-Requests:1","Origin:https://candidate.scholastica.ng","Referer:https://candidate.scholastica.ng/candidate/login","content-type:application/x-www-form-urlencoded","TE:Trailers","content-length:{$content_length}");
// I assign the headers and set the post fields
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
I make request with existing curl thereby sending back the set cookie
$details = curl_exec($ch);
echo $details;
curl_close($ch);
This still returns a Bad Request
Your Response will be Greatly appreciated
i want to fetch a URL from bio with php.
URL: https://www.instagram.com/sukhcha.in/ (It can be anyone's profile)
I tried using simple_html_dom but it always shows https error while fetching html from url.
As advised in my comment, you should use cURL, because it supports HTTPS protocol :
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_TIMEOUT, 0); // Timeout (0 : no timeout)
curl_setopt($ch, CURLOPT_HEADER, false); // Do not download header
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0'); // creates user-agent
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); // do not output content
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // follow redirections
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // do not check HTTPS host (very important, if you set it to true, it probably won't work)
curl_setopt($ch, CURLOPT_URL, 'https://www.instagram.com/sukhcha.in/');
$content = curl_exec($ch);
?>
Then you have to use XPath on your $content variable to extract the part you want.
You can use CURLto get data.
$url = 'https://weather.com/weather/tenday/l/USMO0460:1:US';
$curl = curl_init($url);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl, CURLOPT_HTTPHEADER, array('Content-Type: application/x-www-form-urlencoded'));
$curl_response = curl_exec($curl);
Debug data
echo '<pre>';
print_r($curl_response);
echo '</pre>';
Close curl
curl_close($curl);
I have a problem with a two step authorization with Oauth in PHP.
The first request like:
$AUTHORIZE_URL ='https://oauth.vk.com/authorize?client_id=myID&scope=MyWall&redirect_uri=https://oauth.vk.com/blank.html&response_type=code';
This request is making a redirection to https://oauth.vk.com/blank.html with "#code=Anumber" parameter.
After that I have a second request which need this code.
I use Curl to make those requests and parse the Json result of the second one but how can I get the paramters of the redirect url in curl.
I tried to parse the header of the answer but I found no location.
EDIT:
Example code of my request:
curl_setopt_array($ch = curl_init(), array(
CURLOPT_USERAGENT => '',
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => false,
CURLINFO_REDIRECT_URL => true,
CURLOPT_BINARYTRANSFER => 1,
CURLOPT_URL => $url
));
curl_exec($ch);
$info=curl_getinfo($ch);
prin_r($info);
curl_close($ch);
Ok, the problem is that you are not logged in, if you are logged in the OAuth authentication redirects you to the blank page with your code, but if you are not, there is NO REDIRECT and it shows you a html page with login.
So, in order to get that code you have to login, you can login using your browser and then get the cookies from your browser and use them in your code. This option is good if you only want that code for testing.
The second options is more complex you have to do the login programmatically, that implies more than one curl, saving cookies and sending them to the next request. I made an example for this OAuth authentication in particular that at least works for me. Is quite ugly but is ok as proof of concept.
<?php
$email = "myemailorphone";
$pass = "mypassword";
$id = "myID";
//this url returns a login page
$url= "https://oauth.vk.com/authorize?".http_build_query(["client_id"=>$id,"scope"=>"MyWall","redirect_uri"=>"https://oauth.vk.com/blank.html","response_type"=>"code"]);
$b64url = str_replace("==","--",base64_encode($url)); //different base64 code, just to have all parameters
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
//curl_setopt($ch,CURLOPT_NOBODY,true);
curl_setopt($ch, CURLOPT_USERAGENT,"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$result = curl_exec($ch);
curl_close($ch);
$cookieString = "";
if(strpos($result,"log in")) {
//get all the cookies
preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $result, $matches);
$cookies = array();
foreach($matches[1] as $item) {
parse_str($item, $cookie);
$cookies = array_merge($cookies, $cookie);
}
$cookieString = "";
foreach($cookies as $key=>$val){
$cookieString .= $key."=".$val.";";
}
//CREATE LOGIN POST
$ip_h = explode("name=\"ip_h\" value=\"",$result);
$ip_h = substr($ip_h[1],0,18); // some hidden fields on that page, maybe important
$lg_h = explode("name=\"lg_h\" value=\"",$result);
$lg_h = substr($lg_h[1],0,18); // some hidden fields on that page, maybe important
$fields = [
"origin"=>"https://oauth.vk.com",
"to"=>$b64url, // this is where it redirects after login, not used in the php code but, just for the request
"email"=>$email,//phone or email of user
"expire"=>0,
"pass"=>$pass, //your password
"ip_h"=>$ip_h,
"lg_h"=>$lg_h
];
$post = http_build_query($fields);
$login_url = "https://login.vk.com/?act=login&soft=1";
$ch = curl_init($login_url);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Cookie: '.$cookieString));
curl_setopt($ch, CURLOPT_USERAGENT,"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
//get the new cookies
$result = curl_exec($ch);
preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $result, $matches);
$login_cookies = array();
foreach($matches[1] as $item) {
parse_str($item, $cookie);
$login_cookies = array_merge($login_cookies, $cookie);
}
foreach($login_cookies as $key=>$val){
$cookieString .= $key."=".$val.";";
}
//get next location redirect
preg_match_all('/^Location:\s*(.*)/mi', $result, $matches);
$first_redirect = str_replace("\"","_",$matches[1][0]);
$first_redirect = filter_var($first_redirect,FILTER_SANITIZE_URL); //sanitize url, because it returns unwanted chars
//use the second location redirect
$ch = curl_init($first_redirect);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT,"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Cookie: '.$cookieString,"accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"));
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$result = curl_exec($ch);
curl_close($ch);
//last location forward
preg_match_all('/^Location:\s*(.*)/mi', $result, $matches);
$second_redirect = str_replace("\"","_",$matches[1][0]);
$second_redirect = filter_var($second_redirect,FILTER_SANITIZE_URL);
$ch = curl_init($second_redirect);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT,"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Cookie: '.$cookieString,"accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"));
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$result = curl_exec($ch);
curl_close($ch);
preg_match_all('/^Location:\s*(.*)/mi', $result, $matches);
$blank = str_replace("\"","_",$matches[1][0]);
$blank = filter_var($blank,FILTER_SANITIZE_URL);
echo "Blank url: ".$blank;
}
?>
I am trying to figure out why pasing a custom header is resulting in a 400 BAD REQUEST from the server.
$headers = array(
'API KEY: asdf',
);
$ch = curl_init();
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0" );
curl_setopt($ch, CURLOPT_URL, 'http://url');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POST, 1 );
curl_setopt($ch, CURLOPT_POSTFIELDS, 'stuff');
curl_setopt($ch, CURLOPT_COOKIEFILE, './tmp/cookie.txt');
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_PROXYPORT, '0000');
curl_setopt($ch, CURLOPT_PROXYTYPE, 'HTTP');
curl_setopt($ch, CURLOPT_PROXY, '0.0.0.0');
$result = curl_exec($ch);
curl_close($ch);
I thought that using CURLOPT_HTTPHEADER would add a custom header to the request, but I'm now wondering whether it's simply overriding everything else I set?
There are more reasons a server will give a 400 response than just a header value. Without more information about the endpoint it's difficult to say what's causing the 400 response. With the exception of the extra "," in the headers array in the example the code looks okay. cURL Options