I need to perform a batch insert in MySQL/MariaDB but since data is dynamic I need to build the proper SQL query. In a few steps:
I should find whether the current row exists or not in table - this is the first SELECT inside the loop
Right now I have 1454 but have to insert around 150k later, is better a batch query than 150k INSERT per item on the loop
If record already exists I should update it if doesn't then I should insert ,I just not care about UPDATE yet and the code you're seeing is only for INSERT
So here is what I am doing:
// Get values from Csv file as an array of values
$data = convertCsvToArray($fileName);
echo "DEBUG count(data): ", count($data), "\n";
$i = 0;
$sqlInsert = "INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt) ";
// Processing on each row of data
foreach ($data as $row) {
$sql = "SELECT id,lastSyncAt FROM reps WHERE veeva_rep_id='{$row['Id']}'";
echo "DEBUG: ", $sql, "\n";
$rs = $conn->query($sql);
if ($rs === false) {
echo 'Wrong SQL: '.$sql.' Error: '.$conn->error, E_USER_ERROR;
} else {
$rows_returned = $rs->num_rows;
$veeva_rep_id = "'".$conn->real_escape_string($row['Id'])."'";
$first = "'".$conn->real_escape_string(ucfirst(strtolower($row['FirstName'])))."'";
$last = "'".$conn->real_escape_string(ucfirst(strtolower($row['LastName'])))."'";
$email = "'".$conn->real_escape_string($row['Email'])."'";
$username = "'".$conn->real_escape_string($row['Username'])."'";
$display_name = "'".$conn->real_escape_string(
ucfirst(strtolower($row['FirstName'])).' '.ucfirst(strtolower($row['LastName']))
)."'";
// VALUES should be added only if row doesn't exists
if ($rows_returned === 0) {
// VALUES should be append until they reach 1000
while ($i % 1000 !== 0) {
$sqlInsert .= "VALUES($veeva_rep_id,$first,$last,$email,$username,NOW(),NOW(),$display_name,'VEEVA','https://pdone.s3.amazonaws.com/avatar/default_avatar.png',NOW(),NOW())";
++$i;;
}
// QUERY should be output to console to see if it's right or something is wrong
echo "DEBUG: ", $sqlInsert, "\n";
// QUERY should be executed if there are 1000 VALUES ready to add as a batch
/*$rs = $conn->query($sqlInsert);
if ($rs === false) {
echo 'Wrong SQL: '.$sqlInsert.' Error: '.$conn->error, E_USER_ERROR;*/
}
} else {
// UPDATE
echo "UPDATE";
}
}
}
But this line of code: echo "DEBUG: ", $sql, "\n"; is not outputting nothing to console. I must be doing something wrong but I can't find what. Can any help me to build the proper batch query and to execute it each 1000 values append?
Proper output should be:
DEBUG count(data): 1454
DEBUG: SELECT id,lastSyncAt FROM reps WHERE veeva_rep_id='00580000008ReolAAC'
DEBUG: SELECT id,lastSyncAt FROM reps WHERE veeva_rep_id='005800000039SIWAA2'
....
DEBUG: INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt) VALUES(...), VALUES(...), VALUES(...)
Obtained result:
DEBUG count(data): 1454
DEBUG: SELECT id,lastSyncAt FROM reps WHERE veeva_rep_id='00580000008RGg6AAG'
DEBUG: INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt)
DEBUG: SELECT id,lastSyncAt FROM reps WHERE veeva_rep_id='00580000008RQ4CAAW'
DEBUG: INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt)
.... // until reach 1454 results
The table is empty so it should never goes through ELSE condition (UPDATE one).
EDIT
With help from the answer this is how the code looks now:
$data = convertCsvToArray($fileName);
echo "DEBUG count(data): ", count($data), "\n";
$i = 1;
$sqlInsert = "INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt) VALUES";
foreach ($data as $row) {
$sql = "SELECT id,lastSyncAt FROM reps WHERE veeva_rep_id='{$row['Id']}'";
$rs = $conn->query($sql);
if ($rs === false) {
echo 'Wrong SQL: '.$sql.' Error: '.$conn->error, E_USER_ERROR;
} else {
$rows_returned = $rs->num_rows;
$veeva_rep_id = "'".$conn->real_escape_string($row['Id'])."'";
$first = "'".$conn->real_escape_string(ucfirst(strtolower($row['FirstName'])))."'";
$last = "'".$conn->real_escape_string(ucfirst(strtolower($row['LastName'])))."'";
$email = "'".$conn->real_escape_string($row['Email'])."'";
$username = "'".$conn->real_escape_string($row['Username'])."'";
$display_name = "'".$conn->real_escape_string(
ucfirst(strtolower($row['FirstName'])).' '.ucfirst(strtolower($row['LastName']))
)."'";
if ($rows_returned === 0) {
if ($i % 1000 === 0) {
file_put_contents("output.log", $sqlInsert."\n", FILE_APPEND);
$sqlInsert = "INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt) VALUES";
} else {
$sqlInsert .= "($veeva_rep_id,$first,$last,$email,$username,NOW(),NOW(),$display_name,'VEEVA','https://pdone.s3.amazonaws.com/avatar/default_avatar.png',NOW(),NOW()), ";
}
$i++;
} else {
echo "UPDATE";
}
}
}
But still buggy because:
I have got a first empty INSERT query: INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt) VALUES
I have got a second INSERT query with 1000 VALUES() append, but what happened with the rest? The remaining 454?
Can any give me another tip? Help?
Since it looks like you are trying to load data from a CSV file, you might want to consider using LOAD DATA INFILE functionality which is designed specifically for this purpose.
Here is link to documentation: https://dev.mysql.com/doc/refman/5.6/en/load-data.html
consider using INSERT IGNORE INTO table to check if the record already exists. How to 'insert if not exists' in MySQL?
if you haven't already done so, make veeva_rep_id a PRIMARY key so the INSERT IGNORE will work
also check out using PDO for transactions, prepared statements and dynamically generating queries using PDO
PDO Prepared Inserts multiple rows in single query
<?php
$sql = 'INSERT IGNORE INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt) VALUES ';
$insertQuery = array();
$insertData = array();
/*
assuming the array from the csv is like this
$data = array(
0 => array('name' => 'Robert', 'value' => 'some value'),
1 => array('name' => 'Louise', 'value' => 'another value')
);
*/
foreach ($data as $row) {
$insertQuery[] = '(:veeva_rep_id' . $n . ', :first' . $n . ', :last' . $n . ', :email' . $n . ', :username' . $n . ', :lastLoginAt' . $n . ', :lastSyncAt' . $n . ', :display_name' . $n . ', :rep_type' . $n . ', :avatar_url' . $n . ', :createdAt' . $n . ', :updatedAt' . $n . ')';
$insertData['veeva_rep_id' . $n] = $row['name'];
$insertData['first' . $n] = $row['value'];
$insertData['last' . $n] = $row['name'];
$insertData['email' . $n] = $row['value'];
$insertData['username' . $n] = $row['name'];
$insertData['lastLoginAt' . $n] = $row['value'];
$insertData['lastSyncAt' . $n] = $row['value'];
$insertData['display_name' . $n] = $row['name'];
$insertData['rep_type' . $n] = $row['value'];
$insertData['avatar_url' . $n] = $row['value'];
$insertData['createdAt' . $n] = $row['name'];
$insertData['updatedAt' . $n] = $row['value'];
$n++;
}
$db->beginTransaction();
if (!empty($insertQuery) and count($insertQuery)>1000) {
$sql .= implode(', ', $insertQuery);
$stmt = $db->prepare($sql);
$stmt->execute($insertData);
}
$db->commit();
print $sql . PHP_EOL;
let me know if it helps.
You should have something like:
// Try fetching data from table 1
// If there is no record available, then fetch some data from table 2
// and insert that data inito table 1
You just wrote
$sql = "INSERT INTO reps(veeva_rep_id,first,last,email,username,lastLoginAt,lastSyncAt,display_name,rep_type,avatar_url,createdAt,updatedAt) ";
// Processing on each row of data
foreach ($data as $row) {
But from an insert no data is selected and second...you didn't run a select, where comes $data from?
update Use if ($i % 1000 === 0) { instead of while ($i % 1000 !== 0) {
$i = 0;
$sqlInsert = "INSERT INTO reps(veeva_rep_id,first,last,email,...) ";
// Processing on each row of data
foreach ($data as $row) {
$sql = "SELECT id,lastSyncAt FROM reps WHERE veeva_rep_id='{$row['Id']}'";
echo "DEBUG: ", $sql, "\n";
$rs = $conn->query($sql);
if ($rs === false) {
echo 'Wrong SQL: '.$sql.' Error: '.$conn->error, E_USER_ERROR;
} else {
$veeva_rep_id = ...;
$first = ...;
$last = ...;
$email = ...;
// ...
// VALUES should be added only if row doesn't exists
if($rs->num_rows == 0) {
// Insert some data
$i++;
if ($i % 1000 === 0) {
echo "DEBUG: ", $sqlInsert, "\n";
// execSql($sqlInsert);
$sqlInsert = "INSERT INTO reps(veeva_rep_id,first,last,email,...) "; // reset
} else {
$sqlInsert .= "VALUES($veeva_rep_id,$first,$last,$email,...) ";
}
} else {
echo "UPDATE";
}
}
}
I have an foreach loop that is creating a table with new column names from an array collected from a site. At the moment only the first element of the array is being set as a column name, can someone help?
H
Here is my code:
<?php
include 'simple_html_dom.php';
include 'connection.php';
function getsquad($url, $tablename){
$html = file_get_html($url);
$player_fromsite = array();
$space = ' ';
$replacespace = '_';
$player = array();
foreach($html->find('td[align=left]') as $element) {
if ($element->children(0)) { // work only when children exists
array_push($player_fromsite ,$element->children(0)->innertext);
}
}
//$player[] = str_replace($space, $replacespace, $player_fromsite);
//unset($player_fromsite);
$length = count($player);
foreach($player_fromsite as $player_name) {
mysql_query("CREATE TABLE " . $tablename . "(" . str_replace($space, $replacespace, $player_name) . " VARCHAR(30))") or die(mysql_error());
}
echo "Table Created!";
}
$Squad = new squad();
$Squad->getsquad('site', 'Ars');
?>
any spaces are replace with a "_", in the foreach loop
I am not quite sure if I correctly understand what you want: In $player_fromsite you have some strings, which should become columns of the new table with the name from $tablename, whereas all columns are VARCHAR(30).
If so, replace your last foreach loop with the following:
$columns = array();
foreach ($player_fromsite as $player_name) {
$columns[] = '`' . str_replace($space, $resplacespace, $player_name) . '` VARCHAR(30)';
}
$query = 'CREATE TABLE `' . $tablename . '` (' . implode(',', $columns) . ')';
mysql_query($query) or die(mysql_error());
The foreach loop basically prepares all the columns, which afterwards get concatenated to the actual query which is send to the database.
So for exmaple let $player_fromsite = array('foo', 'bar'), you would end up with the query
CREATE TABLE `Ars` (`foo` VARCHAR(30),`bar` VARCHAR(30));
Language: PHP and MySQL
I have some rows in a database that have a column with a value of NULL. I have a function that builds the query when it receive a $params array. Here's the relevant piece of code in that function (to build the WHERE clause):
if (isset($params['where'])) {
$query .= "WHERE ";
$count = count($params['where']);
foreach ($params['where'] as $key => $value) {
$count--;
if ($count) {
$query .= "`$key` ".$value['sign']." '".$value['text']."' AND ";
} else {
$query .= "`$key` ".$value['sign']." '".$value['text']."' ";
}
}
}
The problem is that when I set $value['text'] to null, here's the result:
SELECT * FROM `db_profiles` WHERE `committed` IS '' LIMIT 100
If I set it to $value['text'] = 'NULL', then I get the following:
SELECT * FROM `db_profiles` WHERE `committed` IS 'NULL' LIMIT 100
Which of course produces nothing. Any ideas?
ANSWER
if (isset($params['where'])) {
$where_segment = array ();
foreach ($params['where'] as $key => $value) {
if (is_null($value['text'])) {
$where_segment[] = "`".$key."`"." IS NULL";
} else {
$where_segment[] = "`$key` ".$value['sign']." '".$value['text']."' ";
}
}
$query .= " WHERE ".implode('AND', $where_segment);
}
$string_array = array();
foreach($params['where'] as $field => $value) {
if(is_null($value)) {
$string_array[] = "`$field` IS NULL";
}
else {
$string_array[] = "`$field`='$value'";
}
}
$where_string = "WHERE ".implode(" AND ", $string_array);
$sql = "SELECT * FROM `db_profiles` $where_string";
Notice that if the $value is NULL I omit the '' marks around the NULL phrase.
Please use this query.
SELECT * FROM db_profiles WHERE committed IS NULL LIMIT 100.
Remove the single quotes.
When populating the values to the query, you'll need an explicit check for something like:
if(is_null($value)) {
$query . ' WHERE something IS NULL';
} else {
$query . ' WHERE something \'' . $value . '\'';
}
Note that when you are new to MySQL and PHP you should start to use prepared statements as soon as possible. They are most secure against SQL injection vulnerabilities and easy to use. You can start here to learn to use them.
this should help you without single quotes
SELECT * FROM `db_profiles` WHERE `committed` IS NULL LIMIT 100
you could set your variable like that to null
$value['text'] = NULL;
and then in your query
SELECT * FROM `db_profiles` WHERE `committed` IS $value['text'] LIMIT 100
I have this function
function updateDbRecord($db, $table, $carry, $carryUrl) {
mysql_select_db($db) or die("Could not select database. " . mysql_error());
$resultInsert = mysql_query("SHOW COLUMNS FROM " . $table . " WHERE Field NOT IN ('id')");
$fieldnames=array();
if (mysql_num_rows($resultInsert) > 0) {
while ($row = mysql_fetch_array($resultInsert)) {
$fieldnames[] = $row['Field'];
$arr = array_intersect_key( $_POST, array_flip($fieldnames) ); #check if value is null otherwise do not INSERT
}
}
$set = "";
foreach($arr as $key => $v) {
$val = is_numeric($v) ? $v : "'" . $v . "'";
$set .= $key . '=' . $val . ', ';
}
$sql = sprintf("UPDATE %s SET %s WHERE id='%s'", $table, $set, $_POST['id']);
mysql_query($sql);
if ($carry == 'yes') {
redirect($carryUrl.'?id='.$_REQUEST['id']);
} else { echo "Done!"; }
echo $sql;
}
It outputs for example: UPDATE projects SET project_name='123', project_bold='123', project_content='123', WHERE id='12'
The last comma before where is preventing it from working. Is there a way of avoiding this? Im aware of the function implode, however I am not sure how to employ it in this situation.
Yes,
$sql = substr($sql,'',-1);
I would use
$sql = rtrim($sql, ',');
Either that or instead of appending to a string, append to an array and use implode.
function updateDbRecord($db, $table, $carry, $carryUrl) {
mysql_select_db($db) or die("Could not select database. " . mysql_error());
$resultInsert = mysql_query("SHOW COLUMNS FROM " . $table . " WHERE Field NOT IN ('id')");
$fieldnames=array();
if (mysql_num_rows($resultInsert) > 0) {
while ($row = mysql_fetch_array($resultInsert)) {
$fieldnames[] = $row['Field'];
$array = array_intersect_key( $_POST, array_flip($fieldnames) ); #check if value is null otherwise do not INSERT
}
}
foreach ($array as $key => $value) {
$value = mysql_real_escape_string($value); // this is dedicated to #Jon
$value = "'$value'";
$updates[] = "$key = $value";
}
$implodeArray = implode(', ', $updates);
$sql = sprintf("UPDATE %s SET %s WHERE id='%s'", $table, $implodeArray, $_POST['id']);
mysql_query($sql);
I have a database containing three tables:
practices - 8 fields
patients - 47 fields
exacerbations - 11 fields
The majority of the fields in these tables are recorded in varchar format, other fields include integers, doubles and dates.
I have to transform this data into numerically classified data so that it can be used by a statistician to extrapolate any patterns in the data. To acheive this I will have to convert varchar fields into integers that represent the classification that string belongs to, an example being 'Severity' which has the following possible string values:
Mild
Moderate
Severe
Very Severe
This field in the patients table has a finite list of string values that can appear, other fields have an endless possibility of string values that cannot be classified until they are encountered by my database (unless I implement some form of intelligent approach).
For the time being I am just trying to construct the best approach to converting each field for all entries in each of the 3 tables to numeric values. The pseudo code I have in my head so far is as follows (it's not complete):
function profileDatabase
for each table in database
for each field that is of type varchar
select all distinct values and insert into classfication table for that field
end for
end for
function classifyDatabase
for each table in database
for each field that is of type varchar
// do something efficient to build an insert string to place into new table
end for
end for
Can someone suggest the best way of performing this process so that it is efficient giving that there are currently in excess of 100 practices, 15,000 patients and 55,000 exacerbations in the system. I have no need to implement this in PHP, build I would prefer to do so. Any pointers as to how to structure this would be great as I am not sure my approach the best approach.
This process will have to run every month for the next two years as the database grows to have a total of 100,000 patients.
Maybe http://dev.mysql.com/doc/refman/5.1/en/procedure-analyse.html will help to get an overview to find out how fields are used.
I have managed to build my own solution to this problem which runs in reasonable time. For anyone interested, or anyone who may encounter a similar issue here is the approach I have used:
A PHP script that is run as a cron job by calling php scriptName.php [database-name]. The script builds a classified table for each table name that is within the database (that is not a lookup table for this process). The setting up of each classification creates a new table which mimics the format of the base table but sets all fields to allow NULL values. It then creates blank rows for each of the rows found in the base table. The process then proceeds by analysing each table field by field and updating each row with the correct class for this field.
I am sure I can optimise this function to improve on the current complexity, but for now I shall use this approach until the run-time of the scripts goes outside of an acceptable range.
Script code:
include ("../application.php");
profileDatabase("coco");
classifyDatabase("coco");
function profileDatabase($database) {
mysql_select_db($database);
$query = "SHOW TABLES";
$result = db_query($query);
$dbProfile = array();
while ($obj = mysql_fetch_array($result)) {
if (!preg_match("/_/", $obj[0])) {
$dbProfile[$obj[0]] = profileTable($obj[0]);
}
}
return $dbProfile;
}
function profileTable($table) {
$tblProfile = array();
$query = "DESCRIBE $table";
$result = db_query($query);
while ($obj = mysql_fetch_array($result)) {
$type = substr($obj[1], 0, 7);
//echo $type;
if (preg_match("/varchar/", $obj[1]) && (!preg_match("/Id/", $obj[0]) && !preg_match("/date/", $obj[0]) && !preg_match("/dob/", $obj[0]))) {
$x = createLookup($obj[0], $table);
$arr = array($obj[0], $x);
$tblProfile[] = $arr;
}
}
return $tblProfile;
}
function getDistinctValues($field, $table) {
$distinct = array();
$query = "SELECT DISTINCT $field as 'value', COUNT($field) as 'no' FROM $table GROUP BY $field ORDER BY no DESC";
$result = db_query($query);
while ($obj = mysql_fetch_array($result)) {
$distinct[] = $obj;
}
return $distinct;
}
function createLookup($field, $table) {
$query = "CREATE TABLE IF NOT EXISTS `" . $table . "_" . $field . "`
(
`id` int(5) NOT NULL auto_increment,
`value` varchar(255) NOT NULL,
`no` int(5) NOT NULL,
`map1` int(3) NOT NULL,
`map2` int(3) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1";
db_query($query);
$distinct = getDistinctValues($field, $table);
$count = count($distinct);
foreach ($distinct as $val) {
$val['value'] = addslashes($val['value']);
$rs = db_query("SELECT id FROM " . $table . "_" . $field . " WHERE value = '" . $val['value'] . "' LIMIT 1");
if (mysql_num_rows($rs) == 0) {
$sql = "INSERT INTO " . $table . "_" . $field . " (value,no) VALUES ('" . $val['value'] . "', " . $val['no'] . ")";
} else {
$sql = "UPDATE " . $table . "_" . $field . " (value,no) VALUES ('" . $val['value'] . "', " . $val['no'] . ")";
}
db_query($sql);
}
return $count;
}
function classifyDatabase($database) {
mysql_select_db($database);
$query = "SHOW TABLES";
$result = db_query($query);
$dbProfile = array();
while ($obj = mysql_fetch_array($result)) {
if (!preg_match("/_/", $obj[0])) {
classifyTable($obj[0]);
//echo "Classfied $obj[0]\n";
}
}
}
function classifyTable($table) {
$query = "SHOW TABLES";
$result = db_query($query);
$dbProfile = array();
$setup = true;
while ($obj = mysql_fetch_array($result)) {
if ($obj[0] == "classify_" . $table)
$setup = false;
}
if ($setup) {
setupClassifyTable($table);
//echo "Setup $table\n";
}
$query = "DESCRIBE $table";
$result = db_query($query);
while ($obj = mysql_fetch_array($result)) {
if (preg_match("/varchar/", $obj[1]) && (!preg_match("/Id/", $obj[0]) && !preg_match("/date/", $obj[0]) && !preg_match("/dob/", $obj[0]))) {
$rs = db_query("
SELECT t.entryId, t.$obj[0], COALESCE(tc.map1,99) as 'group' FROM $table t
LEFT JOIN " . $table . "_$obj[0] tc ON t.$obj[0] = tc.value
ORDER BY tc.map1 ASC");
while ($obj2 = mysql_fetch_object($rs)) {
$sql = "UPDATE classify_$table SET $obj[0] = $obj2->group WHERE entryId = $obj2->entryId";
db_query($sql);
}
} else {
if ($obj[0] != "entryId") {
$rs = db_query("
SELECT t.entryId, t.$obj[0] as 'value' FROM $table t");
while ($obj2 = mysql_fetch_object($rs)) {
$sql = "UPDATE classify_$table SET $obj[0] = '" . addslashes($obj2->value) . "' WHERE entryId = $obj2->entryId";
db_query($sql);
}
}
}
}
}
function setupClassifyTable($table) {
$tblProfile = array();
$query = "DESCRIBE $table";
$result = db_query($query);
$create = "CREATE TABLE IF NOT EXISTS `classify_$table` (";
while ($obj = mysql_fetch_array($result)) {
if (preg_match("/varchar/", $obj[1]) && (!preg_match("/Id/", $obj[0]) && !preg_match("/date/", $obj[0]) && !preg_match("/dob/", $obj[0]))) {
//echo $obj[1]. " matches<br/>";
$create .= "$obj[0] int(3) NULL,";
} else {
$create .= "$obj[0] $obj[1] NULL,";
}
}
$create .= "PRIMARY KEY(`entryId`)) ENGINE=MyISAM DEFAULT CHARSET=latin1";
db_query($create);
$result = mysql_query("SELECT entryId FROM $table");
while ($obj = mysql_fetch_object($result)) {
db_query("INSERT IGNORE INTO classify_$table (entryId) VALUES ($obj->entryId)");
}
}
?>