Related
I have a issue, my INSERT ... ON DUPLICATE KEY UPDATE is inserting a new record instead of updating the row, the Table i am using has both an primary key and a unique key. So i am confused to why this is happening.
Table
CREATE TABLE `Product` (
`Product_Id` bigint(255) NOT NULL AUTO_INCREMENT,
`Resturant_ID` bigint(255) NOT NULL,
`Product_Desc` text NOT NULL,
`Product_Name` varchar(100) NOT NULL,
`Product_Price` decimal(8,0) NOT NULL,
`Add_On_ID` int(11) NOT NULL,
PRIMARY KEY (`Product_Id`),
UNIQUE KEY `Product_Name` (`Product_Name`)
) ENGINE=InnoDB AUTO_INCREMENT=30 DEFAULT CHARSET=utf8
QUERY
$add_product_errors = array();
if (isset($_POST['add'])) {
$item_name = $_POST['item_name'];
$desc = $_POST['desc'];
$price = $_POST['price'];
$rest_id = mysqli_real_escape_string($dbc, $_SESSION['Resturant_ID']);
if (empty($_POST['price']) || !filter_var($_POST['price'], FILTER_VALIDATE_FLOAT) || ($_POST['price'] <= 0)) {
$add_product_errors['price'] = "Please enter a product price";
}
if (empty($_POST['item_name'])) {
$add_product_errors['item_name'] = "Please enter a name";
}
if (empty($_POST['desc'])) {
$add_product_errors['desc'] = "Please enter a product description";
}
$query = "INSERT INTO Product(Resturant_ID,Product_Name,Product_Desc,Product_Price) VALUES (?,?,?,?)
ON DUPLICATE KEY
UPDATE
Resturant_ID = VALUES(Resturant_ID)
,Product_Name = VALUES(Product_Name)
,Product_Desc = VALUES(Product_Desc)
,Product_Price = VALUES(Product_Price)";
$run_query = mysqli_prepare($dbc, $query);
if (!$run_query) {
die(mysqli_error($dbc));
}
mysqli_stmt_bind_param($run_query, 'sssd', $rest_id, $item_name, $desc, $price);
$execute = mysqli_stmt_execute($run_query);
$item_name = strip_tags($_POST['item_name']);
$desc = strip_tags($_POST['desc']);
//100 - changes the way the decimal displays in database
$price = strip_tags($_POST['price'] * 100);
if ($execute) {
echo "<script> alert('Addrrss Saved')</script>";
} else {
echo "<b>Oops! we have an issue </b>";
mysqli_stmt_close($run_query);
}
}
?>
The syntax just looks off to me. Maybe try writing the SQL and testing it first in console or MySQL workbench or whatever first? Try this:
$query = "INSERT INTO Product(Resturant_ID,Product_Name,Product_Desc,Product_Price) VALUES (?,?,?,?)
ON DUPLICATE KEY UPDATE
Resturant_ID = ?
,Product_Name = ?
,Product_Desc = ?
,Product_Price = ?";
$run_query = mysqli_prepare($dbc, $query);
if (!$run_query) {
die(mysqli_error($dbc));
}
mysqli_stmt_bind_param($run_query, 'issdissd', $rest_id, $item_name, $desc, $price, $rest_id, $item_name, $desc, $price);
Or maybe ? eight times and binding things twice... not sure off hand if mysqli supports named parameters...
Updated [again] per Martin's feedback.
In the first foreach statement, I have 4 attendeeid's in the attendees table.
In the second foreach, I have 1 attendeeid in the attend_date_temp table.
I'm trying to load a select box with names from the attendees table, less the one in the attend_date_temp table.
I thought that, since the first foreach would loop 4 times, the second foreach would also loop 4 times. But it doesn't. It loops one time, causing the code in the second foreach to not execute and load the select box with names.
How can this be written so that the second foreach loops 4 times like the first foreach so the select box will have the names loaded to it?
// Load Button Clicked
if(isset($_POST['loadnames'])) {
/* Read the history file and get the last record for each attendee for a particular group
and a particular member and write them to the attend_date_temp table if attend_date = CURDATE().*/
$stmt = $db->prepare('SELECT historyid, attend_date, attendeeid, groupid, memberid
FROM history
WHERE groupid = :groupid
AND memberid = :memberid
AND attend_date = CURDATE()
ORDER BY historyid
DESC LIMIT 1');
$stmt->bindValue(':groupid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->bindValue(':memberid', $_SESSION['memberid'], PDO::PARAM_INT);
$stmt->execute();
$result = $stmt->fetchAll();
foreach($result as $row) {
$aid = $row[2]; // set the attendeeid
$stmt = $db->prepare('INSERT INTO attend_date_temp (attendeeid, groupid, memberid)
VALUES(:aid, :gid, :mid)');
$stmt->bindValue(':aid', $aid, PDO::PARAM_INT);
$stmt->bindValue(':gid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->bindValue(':mid', $_SESSION['memberid'], PDO::PARAM_INT);
$stmt->execute();
}
$aaa = 0; // used to set the first select box entry to "Select"
/* Load the Select Box with names, less the ones found in attend_date_temp Table. */
$stmt = $db->prepare('SELECT a.attendeeid, fname, lname, a.groupid, a.memberid, s.attendeeid, suspend
FROM attendees AS a
JOIN suspended AS s ON a.attendeeid = s.attendeeid
WHERE a.memberid = :memberid
AND suspend = "N"
AND a.groupid = :groupid
ORDER BY lname');
$stmt->bindValue(':memberid', $_SESSION["memberid"], PDO::PARAM_INT);
$stmt->bindValue(':groupid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->execute();
$result = $stmt->fetchAll();
foreach($result as $row){
echo '<script type="text/javascript">alert("In the first loop"); </script>';
$aid = $row[0];
$lname = $row[2];
$fname = $row[1];
$stmt = $db->prepare('SELECT attendeeid, memberid
FROM attend_date_temp
WHERE groupid = :groupid
AND attendeeid = :aid');
$stmt->bindValue(':groupid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->bindValue(':aid', $aid, PDO::PARAM_INT);
$stmt->execute();
$result2 = $stmt->fetchAll();
foreach ($result2 as $row2) {
echo '<script type="text/javascript">alert("In the second loop"); </script>';
// evaluate attendees attendeeid against attend_date_temp attendeeid
if($row2['attendeeid'] != $aid){
// Load the flush Table with the IDs from the selected group
if($_SESSION['flush'] == 0) {
$stmt = $db->prepare('INSERT INTO flush (attendeeid, memberid)
VALUES(:attendeeid, :memberid)');
$stmt->bindValue(':attendeeid', $aid, PDO::PARAM_INT);
$stmt->bindValue(':memberid', $_SESSION['memberid'], PDO::PARAM_INT);
$stmt->execute();
}
if($aaa == 0) {
echo "<option value='Select'>Select</option>";
echo "<option value=".$aid.">".$lname.", ". $fname."</option>";
$aaa = 1;
} else {
echo "<option value=".$aid.">".$lname.", ". $fname."</option>";
}
}
}
}
$_SESSION['flush'] = 1;
exit();
} // last brace: loadnames
The attend_date_temp table:
DROP TABLE IF EXISTS `attend_date_temp`;
CREATE TABLE `attend_date_temp` (
`attendeeid` int(10) unsigned NOT NULL,
`groupid` int(10) unsigned NOT NULL,
`memberid` int(10) unsigned NOT NULL,
KEY `attendeeid` (`attendeeid`),
KEY `memberid` (`memberid`),
CONSTRAINT `attend_date_temp_ibfk_1` FOREIGN KEY (`attendeeid`) REFERENCES `attendees` (`attendeeid`) ON DELETE CASCADE,
CONSTRAINT `attend_date_temp_ibfk_2` FOREIGN KEY (`memberid`) REFERENCES `members` (`memberid`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
The history table:
DROP TABLE IF EXISTS `history`;
CREATE TABLE `history` (
`historyid` int(10) unsigned NOT NULL AUTO_INCREMENT,
`amount` float NOT NULL,
`subsidy` char(1) NOT NULL,
`last_payment` date NOT NULL,
`amount_paid` float NOT NULL,
`balance` float NOT NULL,
`attend` char(1) NOT NULL DEFAULT 'N',
`attend_date` date NOT NULL,
`groupid` char(1) NOT NULL,
`attendeeid` int(10) unsigned NOT NULL,
`memberid` int(10) unsigned NOT NULL,
PRIMARY KEY (`historyid`),
KEY `attendeeid` (`attendeeid`),
CONSTRAINT `history_ibfk_15` FOREIGN KEY (`attendeeid`) REFERENCES `attendees` (`attendeeid`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
UPDATE:
This is a small part of a payment posting page. Names are loaded in the select box based on the group selected, then payments are posted by the name selected. This not only posts their payments but also their attendance. Once all the money has been collected, the remaining names not selected are marked as absent.
However, there are group members that attend groups that are not their own. When they make a payment, their money is posted and attendance recorded. BUT, and that's what this is all about, when that same person's group gets selected for payments, I don't want that person's name to get loaded. He's already paid, and his attendance has already been updated. To have him load again and be processed wld corrupt the history table. So I have to keep from double loading the same person. That's why I'm trying to use this attend_date_temp table.
Simple fix, use different variable names for the inner and outer loops:-
/* Load the Select Box with names, less the ones found in attend_date_temp Table. */
$stmt = $db->prepare('SELECT a.attendeeid, fname, lname, a.groupid, a.memberid, s.attendeeid, suspend
FROM attendees AS a
JOIN suspended AS s ON a.attendeeid = s.attendeeid
WHERE a.memberid = :memberid
AND suspend = "N"
AND a.groupid = :groupid
ORDER BY lname');
$stmt->bindValue(':memberid', $_SESSION["memberid"], PDO::PARAM_INT);
$stmt->bindValue(':groupid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->execute();
$result = $stmt->fetchAll();
foreach($result as $row){
echo '<script type="text/javascript">alert("In the first loop"); </script>';
$aid = $row[0];
$lname = $row[2];
$fname = $row[1];
$stmt = $db->prepare('SELECT attendeeid, memberid
FROM attend_date_temp
WHERE groupid = :groupid
AND attendeeid = :aid');
$stmt->bindValue(':groupid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->bindValue(':aid', $aid, PDO::PARAM_INT);
$stmt->execute();
$result2 = $stmt->fetchAll();
foreach ($result2 as $row2) {
echo '<script type="text/javascript">alert("In the second loop"); </script>';
// evaluate attendees attendeeid against attend_date_temp attendeeid
if($row2['attendeeid'] != $aid){
// Load the flush Table with the IDs from the selected group
if($_SESSION['flush'] == 0) {
$stmt = $db->prepare('INSERT INTO flush (attendeeid, memberid)
VALUES(:attendeeid, :memberid)');
$stmt->bindValue(':attendeeid', $aid, PDO::PARAM_INT);
$stmt->bindValue(':memberid', $_SESSION['memberid'], PDO::PARAM_INT);
$stmt->execute();
}
if($aaa == 0) {
echo "<option value='Select'>Select</option>";
echo "<option value=".$aid.">".$lname.", ". $fname."</option>";
$aaa = 1;
} else {
echo "<option value=".$aid.">".$lname.", ". $fname."</option>";
}
}
}
}
To do a join you would do something like this:-
$stmt = $db->prepare('SELECT a.attendeeid, fname, lname, a.groupid, a.memberid, s.attendeeid, suspend, adt.attendeeid AS adt_attendeeid, adt.memberid AS adt_memberid
FROM attendees AS a
INNER JOIN suspended AS s ON a.attendeeid = s.attendeeid
LEFT OUTER JOIN attend_date_temp adt ON adt.groupid = a.groupid AND adt.attendeeid = a.attendeeid
WHERE a.memberid = :memberid
AND suspend = "N"
AND a.groupid = :groupid
AND adt.groupid IS NULL
ORDER BY lname');
$stmt->bindValue(':memberid', $_SESSION["memberid"], PDO::PARAM_INT);
$stmt->bindValue(':groupid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->execute();
EDIT
Think it can be more simply done like this (not tested so please excuse any typos)
<?php
$first = true;
/* Load the Select Box with names, less the ones found in attend_date_temp Table. */
$stmt = $db->prepare('SELECT a.attendeeid, fname, lname
FROM attendees AS a
INNER JOIN suspended AS s ON a.attendeeid = s.attendeeid
LEFT OUTER JOIN attend_date_temp adt ON adt.groupid = a.groupid AND adt.attendeeid = a.attendeeid
WHERE a.memberid = :memberid
AND suspend = "N"
AND a.groupid = :groupid
AND adt.groupid IS NULL
ORDER BY lname');
$stmt->bindValue(':memberid', $_SESSION["memberid"], PDO::PARAM_INT);
$stmt->bindValue(':groupid', $_POST['groupid'], PDO::PARAM_INT);
$stmt->execute();
$result = $stmt->fetchAll();
foreach($result as $row)
{
$aid = $row[0];
$lname = $row[2];
$fname = $row[1];
// Load the flush Table with the IDs from the selected group
if($_SESSION['flush'] == 0)
{
$stmt = $db->prepare('INSERT INTO flush (attendeeid, memberid)
VALUES(:attendeeid, :memberid)');
$stmt->bindValue(':attendeeid', $aid, PDO::PARAM_INT);
$stmt->bindValue(':memberid', $_SESSION['memberid'], PDO::PARAM_INT);
$stmt->execute();
}
if($first)
{
echo "<option value='Select'>Select</option>";
echo "<option value='".$aid."'>".$lname.", ". $fname."</option>";
$first = false;
}
else
{
echo "<option value='".$aid."'>".$lname.", ". $fname."</option>";
}
}
I am trying to figure out why this while statement returns the same account ID each time. Below is the function I am using:
function Hourly(){
$sql = "SELECT * FROM users WHERE users.id IN(SELECT uid FROM point WHERE zoneid=1)";
$arr = array();
$user = $this->database->DBQry($sql, $arr);
$id = $user[0]['ID'];
$sql = "SELECT * FROM point WHERE zoneid > -1";
$arr = array();
$row = $this->database->DBCtr($sql, $arr);
$i=0;
while($i < $row){
$arr = array(":userid" => $id, ":zoneid" => 1, ":cash" => 500);
$this->database->DBIns($arr, 'cashtable');
echo"$id DONE!<br>";
$i++;
}
}
Here are my prepared statements I'm using.
// Query
function DBQry($sql,$arr){
$sth = $this->prepare($sql, array(PDO::ATTR_CURSOR => PDO::CURSOR_FWDONLY));
$sth->execute($arr);
$rs = $sth->fetchAll();
return $rs;
}
// Count
function DBCtr($sql,$arr){
$sth = self::prepare($sql, array(PDO::ATTR_CURSOR => PDO::CURSOR_FWDONLY));
$sth->execute($arr);
return $sth->rowCount();
}
I think the issue I am encountering is with one of the statements.
What I am trying to achieve from the function is a result like this:
64 DONE!
80 DONE!
124 DONE!
648 DONE!
984 DONE!
1341 DONE!
1468 DONE!
1887 DONE!
2017 DONE!
2193 DONE!
2267 DONE!
But this is what comes out of the function:
64 DONE!
64 DONE!
64 DONE!
64 DONE!
64 DONE!
64 DONE!
64 DONE!
64 DONE!
64 DONE!
64 DONE!
64 DONE!
Code I am working from
<?
$DBHost = "localhost";
$DBUser = "user";
$DBPassword = "password";
$DBName = "db";
$Hcash=50;
$Link = MySQL_Connect($DBHost, $DBUser, $DBPassword) or die ("Can't connect to MySQL");
MySQL_Select_Db($DBName, $Link) or die ("Database ".$DBName." does not exist.");
$OnlineAccountQuery = Mysql_Query("SELECT * FROM point WHERE zoneid > -1");
$OnlineAccountNum = Mysql_Num_Rows($OnlineAccountQuery);
$i=0;
WHILE($i < $OnlineAccountNum){
$OnlineAccountArray = Mysql_Fetch_Array($OnlineAccountQuery);
$UID = $OnlineAccountArray['uid'];
MySQL_Query("INSERT INTO cashtable (userid, zoneid, cash) VALUES ($UID, 1, $Hcash)");
echo"$UID DONE!<br>";
$i++;
}
?>
Response to Shivan Raptor's Answer
function Hourly(){
$sql = "SELECT * FROM point WHERE zoneid > -1";
$arr = array();
$row = $this->database->DBCtr($sql, $arr);
$i=0;
while($i < $row){
$sql = "SELECT * FROM users WHERE users.id IN(SELECT uid FROM point WHERE zoneid=1)";
$arr = array();
$user = $this->database->DBQry($sql, $arr);
$id = $user[0]['ID'];
$arr = array(":userid" => $id, ":zoneid" => 1, ":cash" => 500);
$this->database->DBIns($arr, 'cashtable');
echo"$id DONE!<br>";
$i++;
}
}
Table Structures
CREATE TABLE `point` (
`uid` int(11) NOT NULL DEFAULT '0',
`aid` int(11) NOT NULL DEFAULT '0',
`time` int(11) NOT NULL DEFAULT '0',
`zoneid` int(11) DEFAULT '0',
`zonelocalid` int(11) DEFAULT '0',
`accountstart` datetime DEFAULT NULL,
`lastlogin` datetime DEFAULT NULL,
`enddate` datetime DEFAULT NULL,
PRIMARY KEY (`uid`,`aid`),
KEY `IX_point_aidzoneid` (`aid`,`zoneid`) USING BTREE
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
CREATE TABLE `cashtable` (
`userid` int(11) NOT NULL,
`zoneid` int(11) NOT NULL,
`cash` int(11) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
You didn't overwrite $id. It's defined once before the while loop.
Please double check your logic.
UPDATE after question edit:
In your response's Hourly() function, $row should contain the count of rows of the first $sql. However, in the while loop, you're querying the 2nd same $sql again & again. Therefore, $id is always the same.
I think your 2nd query should relate to the result of first query, but you didn't. Consider joining 2 queries if possible.
I have analyzed 3 million images using libpuzzle. 2 million from my main server and 1 million from another. I would like to combine the information into 1 MySQL database.
I need to take records in test_images_pending database and insert them into test_images but I have to do it in a way where there isn't duplicated data.
test_images has 115 million records total across all tables, words having 110 million by itself. Size ~4.4 GB
test_images_pending has 69 million and 65 million respectfully. Size ~2.6 GB
I have 8GB ram on my computer, and I am willing to load everything (or try) in memory if I have to, to speed things up.
I hoping with some optimizations to my code and or techniques to make MySQL faster I can improve the rate from about 2 pictures per second (from test_images_pending.picture table) to something more manageable. The very least would be something like 100 pictures per second.
Here is the table setup for both test_images and test_images_pending:
--
-- Table structure for table `errors`
--
CREATE TABLE IF NOT EXISTS `errors` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`url` varchar(255) NOT NULL,
`num` int(11) NOT NULL,
`pid` bigint(20) unsigned NOT NULL,
`error` varchar(512) NOT NULL,
`datetime` datetime NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=245688 ;
-- --------------------------------------------------------
--
-- Table structure for table `pictures`
--
CREATE TABLE IF NOT EXISTS `pictures` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`digest` char(32) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_digest` (`digest`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1107725 ;
-- --------------------------------------------------------
--
-- Table structure for table `signatures`
--
CREATE TABLE IF NOT EXISTS `signatures` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`compressed_signature` varchar(338) NOT NULL,
`picture_id` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `picture_id` (`picture_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1107725 ;
-- --------------------------------------------------------
--
-- Table structure for table `stored_pictures`
--
CREATE TABLE IF NOT EXISTS `stored_pictures` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`url` varchar(255) NOT NULL,
`pid` bigint(20) unsigned NOT NULL,
`num` int(11) NOT NULL,
`updated_at` datetime DEFAULT NULL,
`created_at` datetime DEFAULT NULL,
`picture_id` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_url` (`url`),
KEY `idx_picture_id` (`picture_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=2773867 ;
-- --------------------------------------------------------
--
-- Table structure for table `words`
--
CREATE TABLE IF NOT EXISTS `words` (
`pos_and_word` char(5) NOT NULL,
`signature_id` int(11) NOT NULL,
KEY `idx_pos_and_word` (`pos_and_word`),
KEY `signature_id` (`signature_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
Here is my php PDO code I am running:
<html>
<head>
<link href="../css/print.css" rel="stylesheet" type="text/css" media="print" /> <!-- siehe screen.css -->
<link href="../css/screen.css" rel="stylesheet" type="text/css" media="screen, projection" />
<!--[if lte IE 6]><link rel="stylesheet" href="../css/ielte6.css" type="text/css" media="screen" /><![endif]-->
</head>
<body>
<?php
ini_set('max_execution_time', 0);
$dbh = new PDO("mysql:host=127.0.0.1;port=3306;dbname=test_images_pending;charset=utf-8", "root", "");
$dbh->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION );
$dbh->setAttribute(PDO::ATTR_AUTOCOMMIT, FALSE);
try {
$query = "select id,digest from test_images_pending.pictures";
$sth = $dbh->prepare($query);
$sth->execute();
while ($pending_pictures_rows = $sth->fetch(PDO::FETCH_ASSOC, PDO::FETCH_ORI_NEXT)) {
// Print out what id it's on.
print $pending_pictures_rows['id']."<br>";
buffer_flush();
try {
$dbh->beginTransaction();
$query = "SELECT COUNT(id) from test_images.pictures WHERE digest = :digest";
$sth1 = $dbh->prepare($query);
$sth1->bindParam(':digest', $pending_pictures_rows['digest']);
$sth1->execute();
$count = $sth1->fetchColumn();
if ($count == 1) {
$query = "SELECT id from test_images.pictures WHERE digest = :digest";
$sth2 = $dbh->prepare($query);
$sth2->bindParam(':digest', $pending_pictures_rows['digest']);
$sth2->execute();
$correct_pic_id = $sth2->fetchColumn();
if(!isset($correct_pic_id) or empty($correct_pic_id)) {
throw new PDOException('correct_pic_id was empty');
}
$query = "select * from test_images_pending.stored_pictures WHERE picture_id = :picture_id";
$sth3 = $dbh->prepare($query);
$sth3->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth3->execute();
while ($row = $sth3->fetch(PDO::FETCH_ASSOC, PDO::FETCH_ORI_NEXT)) {
$query = "INSERT INTO test_images.stored_pictures
(id, url, pid, num, updated_at, created_at, picture_id)
VALUES
(default, :url, :pid, :num, :updated_at, :created_at, :picture_id);";
$sth4 = $dbh->prepare($query);
$sth4->bindParam(':url', $row['url']);
$sth4->bindParam(':pid', $row['pid']);
$sth4->bindParam(':num', $row['num']);
$sth4->bindParam(':updated_at', $row['updated_at']);
$sth4->bindParam(':created_at', $row['created_at']);
$sth4->bindParam(':picture_id', $correct_pic_id);
$sth4->execute();
}
$query = "DELETE FROM test_images_pending.stored_pictures WHERE picture_id = :picture_id;";
$sth5 = $dbh->prepare($query);
$sth5->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth5->execute();
$query = "select id from test_images_pending.signatures WHERE picture_id = :picture_id;";
$sth6 = $dbh->prepare($query);
$sth6->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth6->execute();
$signature_id = $sth6->fetchColumn();
if(!isset($signature_id) or empty($signature_id)) {
throw new PDOException('signature_id was empty');
}
$query = "DELETE FROM test_images_pending.words WHERE signature_id = :signature_id;";
$sth7 = $dbh->prepare($query);
$sth7->bindParam(':signature_id', $signature_id);
$sth7->execute();
$query = "DELETE FROM test_images_pending.signatures WHERE picture_id = :picture_id";
$sth8 = $dbh->prepare($query);
$sth8->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth8->execute();
$query = "DELETE FROM test_images_pending.pictures WHERE digest = :digest";
$sth9 = $dbh->prepare($query);
$sth9->bindParam(':digest', $pending_pictures_rows['digest']);
$sth9->execute();
} else if ($count == 0){
$query = "INSERT INTO test_images.pictures
(id, digest)
VALUES
(default, :digest);";
$sth2 = $dbh->prepare($query);
$sth2->bindParam(':digest', $pending_pictures_rows['digest']);
$sth2->execute();
$new_pic_id = $dbh->lastInsertId();
$query = "select * from test_images_pending.stored_pictures WHERE picture_id = :picture_id";
$sth3 = $dbh->prepare($query);
$sth3->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth3->execute();
while ($row = $sth3->fetch(PDO::FETCH_ASSOC, PDO::FETCH_ORI_NEXT)) {
$query = "INSERT INTO test_images.stored_pictures
(id, url, pid, num, updated_at, created_at, picture_id)
VALUES
(default, :url, :pid, :num, :updated_at, :created_at, :picture_id);";
$sth4 = $dbh->prepare($query);
$sth4->bindParam(':url', $row['url']);
$sth4->bindParam(':pid', $row['pid']);
$sth4->bindParam(':num', $row['num']);
$sth4->bindParam(':updated_at', $row['updated_at']);
$sth4->bindParam(':created_at', $row['created_at']);
$sth4->bindParam(':picture_id', $new_pic_id);
$sth4->execute();
}
$query = "DELETE FROM test_images_pending.stored_pictures WHERE picture_id = :picture_id;";
$sth5 = $dbh->prepare($query);
$sth5->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth5->execute();
$query = "select id,compressed_signature from test_images_pending.signatures WHERE picture_id = :picture_id;";
$sth6 = $dbh->prepare($query);
$sth6->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth6->execute();
$fetched = $sth6->fetch(PDO::FETCH_ASSOC);
$signature_id = $fetched['id'];
if(!isset($signature_id) or empty($signature_id)) {
print_r($sth6->fetch(PDO::FETCH_ASSOC));
throw new PDOException('signature_id was empty');
}
$compressed_signature = $fetched['compressed_signature'];
if(!isset($compressed_signature) or empty($compressed_signature)) {
print_r($sth6->fetch(PDO::FETCH_ASSOC));
throw new PDOException('compressed_signature was empty');
}
$query = "INSERT INTO test_images.signatures
(id, compressed_signature, picture_id)
VALUES
(default, :compressed_signature, :picture_id);";
$sth7 = $dbh->prepare($query);
$sth7->bindParam(':picture_id', $new_pic_id);
$sth7->bindParam(':compressed_signature', $compressed_signature);
$sth7->execute();
$new_sig_id = $dbh->lastInsertId();
$query = "SELECT pos_and_word FROM test_images_pending.words WHERE signature_id = :signature_id";
$sth8 = $dbh->prepare($query);
$sth8->bindParam(':signature_id', $signature_id);
$sth8->execute();
while ($row = $sth8->fetch(PDO::FETCH_ASSOC, PDO::FETCH_ORI_NEXT)) {
$query = "INSERT INTO test_images.words
(pos_and_word, signature_id)
VALUES
(:pos_and_word, :signature_id);";
$sth9 = $dbh->prepare($query);
$sth9->bindParam(':pos_and_word', $row['pos_and_word']);
$sth9->bindParam(':signature_id', $new_sig_id);
$sth9->execute();
}
$query = "DELETE FROM test_images_pending.words WHERE signature_id = :signature_id;";
$sth10 = $dbh->prepare($query);
$sth10->bindParam(':signature_id', $signature_id);
$sth10->execute();
$query = "DELETE FROM test_images_pending.signatures WHERE picture_id = :picture_id";
$sth11 = $dbh->prepare($query);
$sth11->bindParam(':picture_id', $pending_pictures_rows['id']);
$sth11->execute();
$query = "DELETE FROM test_images_pending.pictures WHERE digest = :digest";
$sth12 = $dbh->prepare($query);
$sth12->bindParam(':digest', $pending_pictures_rows['digest']);
$sth12->execute();
} else {
throw new PDOException("Found more than 1 match for the digest '{$pending_pictures_rows['digest']}' in 'test_images.pictures' ", $query);
}
$dbh->commit();
} catch (PDOException $e) {
$dbh->rollback();
print "<pre>"; print_r($e); print "</pre>"; exit;
}
}
try {
$dbh->beginTransaction();
$query = "SELECT * FROM test_images_pending.errors";
$sth13 = $dbh->prepare($query);
$sth13->execute();
while ($row = $sth13->fetch(PDO::FETCH_ASSOC, PDO::FETCH_ORI_NEXT)) {
$query = "INSERT INTO test_images.errors
(id, url, num, pid, error, datetime)
VALUES
(default, :url, :num, :pid, :error, :datetime);";
$sth14 = $dbh->prepare($query);
$sth14->bindParam(':url', $row['url']);
$sth14->bindParam(':num', $row['num']);
$sth14->bindParam(':pid', $row['pid']);
$sth14->bindParam(':error', $row['error']);
$sth14->bindParam(':datetime', $row['datetime']);
$sth14->execute();
}
$query = "DELETE FROM test_images_pending.errors WHERE 1";
$sth15 = $dbh->prepare($query);
$sth15->execute();
$dbh->commit();
} catch (PDOException $e) {
$dbh->rollback();
print "<pre>"; print_r($e); print "</pre>"; exit;
}
} catch (PDOException $e) {
print "<pre>"; print_r($e); print "</pre>"; exit;
}
function buffer_flush(){
echo str_pad('', 512);
echo '<!-- -->';
if(ob_get_length()){
#ob_flush();
#flush();
#ob_end_flush();
}
#ob_start();
}
?>
</body>
</html>
Edit:
Some profiling:
This INSERT gets ran 100 times each non-similar picture (~5 every 6 thus far). It normally takes 0.5 to 0.9 seconds to finish the while loop with an average of 0.007 per INSERT.
$query = "INSERT INTO test_images.words
(pos_and_word, signature_id)
VALUES
(:pos_and_word, :signature_id);";
$sth9 = $dbh->prepare($query);
$sth9->bindParam(':pos_and_word', $row['pos_and_word']);
$sth9->bindParam(':signature_id', $new_sig_id);
$sth9->execute();
DELETE FROM test_images_pending.stored_pictures WHERE picture_id = :picture_id;
select * from test_images_pending.stored_pictures WHERE picture_id = :picture_id
DELETE FROM test_images_pending.stored_pictures WHERE picture_id = :picture_id;
all take an average of 0.15 seconds or so per similar picture (~1 out of 6).
Edit 2:
Going by this benchmarking: http://we-love-php.blogspot.com/2012/08/mass-inserts-updates-sqlite-vs-mysql.html
Replacing the slow while loop previously mentioned in Edit 1 with just simple writing to text file such as:
$inserts = array();
while ($row = $sth8->fetch(PDO::FETCH_ASSOC, PDO::FETCH_ORI_NEXT)) {
$inserts[] = "(".$dbh->quote($row['pos_and_word']).", ".$dbh->quote($new_sig_id).")";
}
$query = "INSERT INTO imvu_images.words (pos_and_word, signature_id) VALUES " . implode(',',$inserts) . ";";
file_put_contents("inserts.sql", $query."\n", FILE_APPEND);
Makes it a l ot faster. Not 100 per second though, more like 10-20. I can then just execute the SQL later on and it runs instantly without delay. (That's why I think there is an issue with my code). The reason why I want 100 per second is because I can analyze images and insert them into 1 database at 30 per second. At this rate, it's faster for me to analyze 2 million images and have it insert each one by one than it is to mass insert the rows. This doesn't seem right, that the server can download 30 images, analyze 30 images, and then do 30 inserts in 1 second yet just doing these various SQL statements cannot even match that.
Edit 3:
Updated my.ini with:
key_buffer_size=4000M
read_buffer_size=32M
read_rnd_buffer_size=200M
bulk_insert_buffer_size=1000M
myisam_max_sort_file_size=10000M
myisam_repair_threads=1
tmp_table_size = 1024M
max_heap_table_size = 1024M
join_buffer_size=8M
sort_buffer_size=8M
max_allowed_packet=32M
max_connect_errors=10
myisam_sort_buffer_size=256M
query_cache_limit=12M
query_cache_size=256M
query_cache_type=1
Which seems to have improved performance 2 fold without using the file_put_contents hack. Still though, 5 records a second isn't cutting it.
The reason this process is so slow is not because the individual queries are slow - in fact, I'm surprised at how fast it's all going - but because you're processing millions of records, one at a time, by looping through each record in your outer resultset. What SQL is good at is processing millions of records all in one go.
There's too much business logic in your code for me to want to re-write the whole thing for you, but I think you want to have a re-write the code along the lines of
INSERT INTO test_images.pictures
(id, digest)
SELECT id, digest
from test_images_pending.pictures
where id not in
(select id from test_images.pictures)
Do the same for the other tables. This should run pretty fast - if you've got a good indexing scheme, you'll almost certainly be I/O bound. You should definitely reach more than 2 records per second!
Why can't you use Mysql stored procedures? They execute in Mysql server directly and Faster than Query execution from php.
http://dev.mysql.com/doc/refman/5.0/en/create-procedure.html
Call the stored procedure from php like this:
$res = mysql_query('call sp_sel_test()');
if ($res === FALSE) {
die(mysql_error());
}
You need to set client flags while connecting for using stored procedures with php. Use this:
mysql_connect($this->h,$this->u,$this->p,false,65536);
See MySQL Client Flags for more details.
Edit: The main issue was indices on the source tables being INSERTing into. It is recommended to drop any non needed indices before doing mass inserts, then rebuild afterwords.
With a combination of tweaking the mysql settings and the following code, I was able to get the duplicate images (The join portion) to do 50,000 in 30 seconds or so, 25 seconds being just the JOIN operation.
The 2nd part I am using NOT IN and this is where most of the time occurs but it inserts at a rate of 800 records per second, so it exceeds my goal.
I am going to leave the question open for a bit longer to see if it can be optimized more, because I have 39 million records to process.
<html>
<head>
<link href="../css/print.css" rel="stylesheet" type="text/css" media="print" /> <!-- siehe screen.css -->
<link href="../css/screen.css" rel="stylesheet" type="text/css" media="screen, projection" />
<!--[if lte IE 6]><link rel="stylesheet" href="../css/ielte6.css" type="text/css" media="screen" /><![endif]-->
</head>
<body>
<?php
ini_set('max_execution_time', 0);
$benchmark = false;
$delete = false;
$dbh = new PDO("mysql:host=127.0.0.1;port=3306;dbname=test_images_pending;charset=utf-8", "root", "");
$dbh->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION );
$dbh->setAttribute(PDO::ATTR_AUTOCOMMIT, FALSE);
$timers = array();
try {
$query = "SELECT * FROM test_images.pictures
INNER JOIN test_images_pending.pictures
USING ( digest )";
$sth = $dbh->prepare($query);
$sth->execute();
while ($join_rows = $sth->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
$digest = $join_rows[0];
$correct_pic_id = $join_rows[1];
$wrong_pic_id = $join_rows[2];
try {
$dbh->beginTransaction();
$query = "INSERT INTO test_images.stored_pictures
(url, pid, num, updated_at, created_at, picture_id)
SELECT
url, pid, num, updated_at, created_at, :correct_pic_id FROM test_images_pending.stored_pictures WHERE picture_id = :wrong_pic_id;";
$sth4 = $dbh->prepare($query);
$sth4->bindParam(':correct_pic_id', $correct_pic_id);
$sth4->bindParam(':wrong_pic_id', $wrong_pic_id);
$sth4->execute();
$dbh->commit();
} catch (PDOException $e) {
$dbh->rollback();
print "<pre>"; print_r($e); print "</pre>"; exit;
}
}
} catch (PDOException $e) {
print "<pre>"; print_r($e); print "</pre>"; exit;
}
try {
$query = "SELECT COUNT(id) FROM `signatures` WHERE (`id` - `picture_id` !=0) ";
$sth = $dbh->prepare($query);
$sth->execute();
$count = $sth->fetchColumn();
if($count > 0) {
die("we got a sig that aint matching its pic_id, we cant assume sig_id = pic_id. Back to drawing board");
}
$sth = null;
$query = " SELECT digest, id
FROM test_images_pending.pictures
WHERE digest NOT IN
(
SELECT digest
FROM test_images.pictures
)";
$sth = $dbh->prepare($query);
$sth->execute();
while ($not_in_rows = $sth->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
$digest = $not_in_rows[0];
$wrong_pic_id = $not_in_rows[1];
try {
$dbh->beginTransaction();
$query = "INSERT INTO test_images.pictures
(id, digest)
VALUES
(default, :digest);";
$sth2 = $dbh->prepare($query);
$sth2->bindParam(':digest', $digest);
$sth2->execute();
$new_pic_id = $dbh->lastInsertId();
$query = "INSERT INTO test_images.stored_pictures
(url, pid, num, updated_at, created_at, picture_id)
SELECT
url, pid, num, updated_at, created_at, :new_pic_id FROM test_images_pending.stored_pictures WHERE picture_id = :wrong_pic_id;";
$sth3 = $dbh->prepare($query);
$sth3->bindParam(':new_pic_id', $new_pic_id);
$sth3->bindParam(':wrong_pic_id', $wrong_pic_id);
$sth3->execute();
$query = "INSERT INTO test_images.signatures
(compressed_signature, picture_id)
SELECT
compressed_signature, :new_pic_id FROM test_images_pending.signatures WHERE picture_id = :wrong_pic_id;";
$sth4 = $dbh->prepare($query);
$sth4->bindParam(':new_pic_id', $new_pic_id);
$sth4->bindParam(':wrong_pic_id', $wrong_pic_id);
$sth4->execute();
$new_sig_id = $dbh->lastInsertId();
$query = "INSERT INTO test_images.words
(pos_and_word, signature_id)
SELECT
pos_and_word, :new_sig_id FROM test_images_pending.words WHERE signature_id = :old_sig_id
";
$sth9 = $dbh->prepare($query);
$sth9->bindParam(':old_sig_id', $wrong_pic_id);
$sth9->bindParam(':new_sig_id', $new_sig_id);
$sth9->execute();
$dbh->commit();
} catch (PDOException $e) {
$dbh->rollback();
print "<pre>"; print_r($e); print "</pre>"; exit;
}
}
} catch (PDOException $e) {
print "<pre>"; print_r($e); print "</pre>"; exit;
}
function buffer_flush(){
echo str_pad('', 512);
echo '<!-- -->';
if(ob_get_length()){
#ob_flush();
#flush();
#ob_end_flush();
}
#ob_start();
}
?>
</body>
</html>
I'm trying to add the articles id to the title, summary and content but I don't know how to do it can some one help me solve this problem.
Here is the code that is giving me the problem.
while($row = mysqli_fetch_assoc($run)) {
$id[] = $row['id'];
$title[] = $row['id']['title'];
$summary[] = $row['id']['summary'];
$content[] = $row['id']['article_content'];
}
And here is my PHP and MySQL code in full below.
$x = 0;
$con = null;
$search = $_REQUEST['search'];
$id = array();
$title = array();
$summary = array();
$content = array();
$search_explode = mysqli_real_escape_string($dbc, $search);
$search_explode = explode(' ', $search_explode);
foreach($search_explode as $search_each) {
$x++;
if($x == 1){
$con .= " article_content LIKE '%$search_each%' OR title LIKE '%$search_each%' OR summary LIKE '%$search_each%'";
} else {
$con .= " OR article_content LIKE '%$search_each%' OR title LIKE '%$search_each%' OR summary LIKE '%$search_each%'";
}
}
$con = "SELECT users.*, users_articles.* FROM users_articles
INNER JOIN users ON users_articles.user_id = users.user_id
WHERE ($con)
AND users.active IS NULL
AND users.deletion = 0";
$run = mysqli_query($dbc, $con);
$search_term = mysqli_num_rows($run);
while($row = mysqli_fetch_assoc($run)) {
$id[] = $row['id'];
$title[] = $row['id']['title'];
$summary[] = $row['id']['summary'];
$content[] = $row['id']['article_content'];
}
while($row = mysqli_fetch_assoc($run)) {
// $id[] = $row['id']; you probably do not need this anymore
$title[$row['id']] = $row['title'];
$summary[$row['id']] = $row['summary'];
$content[$row['id']] = $row['article_content'];
}
// at this point, each array will contain rows
// with keys matching the corresponding id
var_dump($title);
var_dump($summary);
var_dump($content);
$title[] = $row['id']['title'];
$summary[] = $row['id']['summary'];
$content[] = $row['id']['article_content'];
mysqli_fetch_assoc will fetch an associated array. Accessing the data directly like so $row['summary'] should fix your problem.
EDIT
Are you talking about string concatenation?
$title[] = $row['id'] . ' ' . $row['title'];
$summary[] = $row['id'] . ' ' . $row['summary'];
$content[] = $row['id'] . ' ' . $row['article_content'];
The above code will basically append the title/summary/content to your article ID and add it into respective arrays.
I am not sure what you want to achieve but I would suggest the the following solution. Please take a look.
CREATE TABLE IF NOT EXISTS users (
user_id int(11) NOT NULL AUTO_INCREMENT,
name varchar(100) NOT NULL,
deletion int(11) NOT NULL DEFAULT '0',
active int(11) NOT NULL DEFAULT '1',
PRIMARY KEY (user_id)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=3 ;
--
-- Dumping data for table users
INSERT INTO users (user_id, name, deletion, active) VALUES
(1, 'John', 0, 1),
(2, 'George', 0, 1);
--
-- Table structure for table users_articles
CREATE TABLE IF NOT EXISTS users_articles (
id int(11) NOT NULL AUTO_INCREMENT,
user_id int(11) NOT NULL,
article_content varchar(255) NOT NULL,
title varchar(255) NOT NULL,
summary text NOT NULL,
PRIMARY KEY (id)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=5 ;
--
-- Dumping data for table users_articles
INSERT INTO users_articles (id, user_id, article_content, title, summary) VALUES
(1, 1, 'test', 'test', 'test test test test'),
(2, 2, 'test 2', 'test 2', 'test test'),
(3, 1, 'test 3', 'test 3 ', 'test test test test 3'),
(4, 2, 'test 3', 'test 3', 'test test 3');
<?php
error_reporting(E_ALL);
ini_set('display_errors', 1);
$mysqli = new mysqli("localhost", "root", "", "articles");
/* check connection */
if ($mysqli->connect_errno) {
printf("Connect failed: %s\n", $mysqli->connect_error);
exit();
}
/* Select queries return a resultset */
$con = 1;
$sql = "SELECT users.*, users_articles.* FROM users_articles
INNER JOIN users ON users_articles.user_id = users.user_id
WHERE ($con)
AND users.active = 1
AND users.deletion = 0";
$results = array();
if ($result = $mysqli->query($sql)) {
printf("Select returned %d rows.\n", $result->num_rows);
print '<pre>';
while ($row = $result->fetch_assoc()) {
$results[] = $row;
}
print_r($results);
print '</pre>';
/* free result set */
$result->close();
}
$mysqli->close();
?>
and when you will iterate $results you can concatenate id and other fields in that case you can utilize the result array in other areas too.