Combining PHP CURL and DOM - php

Question

I have a code that combines CURL and DOM. My code:
<?php
// Create temp file to store cookies
$ckfile = tempnam ("/tmp", "CURLCOOKIE");
// URL to login page
$url = "https://www.investagrams.com/login";
// Get Login page and its cookies and save cookies in the temp file
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
#$output = curl_exec($ch);
$fields = array(
'ctl00$WelcomePageMainContent$ctl00$Username' => '********',
'ctl00$WelcomePageMainContent$ctl00$Password' => '********',
);
$fields_string = '';
foreach($fields as $key=>$value) {
$fields_string .= $key . '=' . $value . '&';
}
rtrim($fields_string, '&');
// Post login form and follow redirects
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, count($fields));
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields_string);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
#$output = curl_exec($ch);
$url = "https://www.investagrams.com/Stock/RealTimeMonitoring";
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$output = curl_exec($ch);
#echo $output;
$dom = new DomDocument;
$dom->loadHtmlFile($output);
$xpath = new DomXPath($dom);
// collect header names
$headerNames = array();
foreach ($xpath->query('//table[#id="StockQuoteTable"]//th') as $node) {
$headerNames[] = $node->nodeValue;
}
// collect data
$data = array();
foreach ($xpath->query('//tbody[#id="StockQuoteTable:tbody_element"]/tr') as $node) {
$rowData = array();
foreach ($xpath->query('td', $node) as $cell) {
$rowData[] = $cell->nodeValue;
}
$data[] = array_combine($headerNames, $rowData);
}
print_r($data);
?>
This loads to just "Arrays():"
Here's the info of table I want to extract:
I don't know which part is wrong. The Curl part is 100% working, the error is in DOM part. Thank you
<div class="dataTables_scrollBody" style="overflow: auto; height: 300px; width: 100%;">
<table id="StockQuoteTable" class="table dataTable no-footer" role="grid" aria-describedby="StockQuoteTable_info" style="width: 1166px;">
<thead></thead>
<tbody>
<tr id="num1" class="odd" role="row"

Answer 1

I was able to find out part of the problem with your code, however it seems that the HTML code supplied from the curl request seems to have some errors in it preventing the function DOMXPath::query from returning a valid match.
The problem I was able to fix in your code was caused by you using DOMDocument::loadHTMLfile instead of DOMDocument::loadHTML to include the HTML retrieved from your curl request. So the valid script should be:
<?php
// Create temp file to store cookies
$ckfile = tempnam ("/tmp", "CURLCOOKIE");
// URL to login page
$url = "https://www.investagrams.com/login";
// Get Login page and its cookies and save cookies in the temp file
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
#$output = curl_exec($ch);
$fields = array(
'ctl00$WelcomePageMainContent$ctl00$Username' => '********',
'ctl00$WelcomePageMainContent$ctl00$Password' => '********',
);
$fields_string = '';
foreach($fields as $key=>$value) {
$fields_string .= $key . '=' . $value . '&';
}
rtrim($fields_string, '&');
// Post login form and follow redirects
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, count($fields));
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields_string);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
#$output = curl_exec($ch);
$url = "https://www.investagrams.com/Stock/RealTimeMonitoring";
$ch = curl_init();
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); // Accepts all CAs
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEFILE, $ckfile);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$output = curl_exec($ch);
#echo $output;
#print_r($output);
$dom = new DomDocument;
#$dom->loadHtml($output);
$xpath = new DomXPath($dom);
// collect header names
$headerNames = array();
foreach ($xpath->query('//table[#id="StockQuoteTable"]//th') as $node) {
$headerNames[] = $node->nodeValue;
}
// collect data
$data = array();
foreach ($xpath->query('//tbody[#id="StockQuoteTable:tbody_element"]/tr') as $node) {
$rowData = array();
foreach ($xpath->query('td', $node) as $cell) {
$rowData[] = $cell->nodeValue;
}
$data[] = array_combine($headerNames, $rowData);
}
print_r($data);
?>
Additionally I added an # symbol before the loadHTML function to suppress errors.