How import big CSV file into MySQL - php

My web application receives comma separated list of phones, which must be imported into the Contacts table. The list can contain from 100,000 to 1,000,000 items. I implemented the stored procedure. But it works still too slow for me. Can you please help me to improve it?
DELIMITER $$
CREATE DEFINER=`root`#`localhost` PROCEDURE `import_phones`
(IN ContactGroupId int, IN strPhones mediumtext, OUT total int, OUT inserted int)
main: BEGIN
declare delim varchar(1) default(',');
declare delimPtr int unsigned default(0); -- pointer to current spliter position
declare startPtr int unsigned default(1);
declare phone1 nvarchar(20);
set total = 0; -- counter of total rows
set inserted = 0; -- counter of inserted rows
if strPhones is null or length(strPhones) < 1 then
leave main;
end if;
drop table if exists insertphones;
create temporary table insertphones(phone nvarchar(20))
engine = memory;
/***
-- split strPhones by delimiter
*/
SET delimPtr = locate(delim, strPhones,startPtr);
loop_label: while(delimPtr > 0) do
insert into insertphones (phone) values (substring(strPhones,startPtr,delimPtr-startPtr));
-- select delimPtr,startPtr, substring(strPhones,startPtr,delimPtr-startPtr);
set startPtr = delimPtr+1;
set delimPtr = locate(delim, strPhones,startPtr);
end while;
if delimPtr = 0 then
insert into insertphones (phone) values (substring(strPhones,startPtr,delimPtr-startPtr));
end if;
-- select delimPtr,startPtr;
select count(*) from insertphones into total;
/***
-- insert phones
*/
insert into contacts (Phone, ContactGroupId)
select distinct(phone), ContactGroupId from insertphones where
phone not in (select Phone from contacts where ContactGroupId = ContactGroupId);
SELECT ROW_COUNT() into inserted ;
-- select total, inserted;
END

Ponder load data infile, like
load data infile 'filename' into tablename lines terminated by ','

Ok. The solution which works much much (about 10x-20x times) faster then the initial version.
$tempFile = tempnam(sys_get_temp_dir(), 'phones');
$tempTable = 'tmp'.$contactGroupId.time();
file_put_contents($tempFile, $phones);
try {
chmod($tempFile, 0777); //required for MySQL import
$sql = sprintf(
"set #total = 0;
set #inserted = 0;
drop table if exists %s;
create temporary table %s(phone nvarchar(20)) engine = memory;
LOAD DATA INFILE '%s' INTO TABLE %s FIELDS TERMINATED BY '' LINES TERMINATED BY ',';
SELECT count(*) FROM %s INTO #total;
DELETE FROM %s WHERE length(phone) < 10;
UPDATE %s SET phone = CONCAT('7',phone) WHERE length(phone) = 10;
UPDATE %s SET phone = INSERT(phone,1,1,'7') WHERE phone like '8%%' and length(phone) = 11;
insert into contacts (Phone, ContactGroupId) select distinct(phone), %d from %s where phone not in (select Phone from contacts where ContactGroupId = %d);
SELECT ROW_COUNT() into #inserted;"
, $tempTable, $tempTable, str_replace('\\','\\\\',$tempFile), $tempTable, $tempTable, $tempTable, $tempTable, $tempTable, $contactGroupId, $tempTable, $contactGroupId );
$command = Yii::app()->db->createCommand($sql);
$command->execute();
$command = false;
$result = Yii::app()->db->createCommand("select #total as total, #inserted as inserted")->queryRow();
$contactCounter['all'] += $result['total'];
$contactCounter['created'] += $result['inserted'];

Related

Is it possible to pass multiple input values in stored procedure parameter?

I am working on a fantasy baseball optimizer and I have a stored procedure where when the user selects players it will return the best possible player, but the problem is, the sp only does it for one player at a time. I wanted to know if there was a way the sp could do it for at most nine players at one time.
Visual Reference
Here's the code for the stored procedure:
DELIMITER $$
CREATE DEFINER=`u998875936_chri`#`%` PROCEDURE `Optimizer9`(
PlayerName varchar(30),
PlayerPosition varchar(2),
PlayerSalary int,
PlayerFPPG Numeric (20,17),
CapRemaining int,
OUT ReturnPlayerName varchar(30),
OUT ReturnCap int
)
BEGIN
Declare Count int;
Declare lclSal int;
Declare lclPlayerName varchar(30);
Declare lclReturnPlayerName varchar(30);
Declare lclCap int;
Set lclPlayerName='';
Set lclReturnPlayerName='';
Set lclSal=0;
Set lclCap = CapRemaining;
Set Count=0;
create temporary Table IF NOT EXISTS TempPlayer9(
Player varchar(30),
Pos varchar(2),
Sal int,
Points Numeric (20,17)
);
Insert into TempPlayer9 (Player,Pos,Sal,Points)
SELECT Nickname
,Position
,Salary
,FPPG
FROM playerList
WHERE POSITION = PlayerPosition
and FPPG > PlayerFPPG
order by FPPG DESC;
Set Count=(Select Count(*) from TempPlayer9);
While Count > 0
DO
Set lclPlayerName=(Select Player from TempPlayer9 LIMIT 1);
Set lclSal=(Select Sal from TempPlayer9 LIMIT 1);
IF (lclsal- PlayerSalary) < lclCap
THEN
Set lclCap = lclCap - (lclSal- PlayerSalary);
Set lclReturnPlayerName = lclPlayerName;
Set Count=0;
DELETE from TempPlayer9 Where Player = lclPlayerName;
Set Count=Count-1 ;
END IF;
END WHILE;
Set ReturnPlayerName = lclReturnPlayerName;
Set ReturnCap = lclCap;
END$$
DELIMITER ;
PHP Code:
<?php
$remain=35000;
$player= $_POST['player'];
$pos= $_POST['pos'];
$sal= $_POST['sal'];
$points= $_POST['fppg'];
$cap= $remain-$sal;
$sql = $mysqli->query("call Optimizer11('$player', '$pos','$sal' , '$points', '$cap', #ReturnPlayerName, #ReturnCap)");
$results = $mysqli->query ("select #ReturnPlayerName as Player,#ReturnCap AS SalRemaining");
$rows = mysqli_fetch_array($results);
print_r($rows);
?>

How to get result from stored procedure, the procedure runs fine but php returns null

I have an stored procedure running on my mysql server with this code:
DELIMITER $$
CREATE DEFINER=`present`#`%` PROCEDURE `InsertNewEventOrderItems`(IN `UserIDParam` INT, IN `Time` VARCHAR(255), IN `EventIDParam` INT, OUT `Result` VARCHAR(255))
MODIFIES SQL DATA
BEGIN
DECLARE OrderIDVar INT DEFAULT 0;
IF (EXISTS(SELECT * FROM _newapp_shop_orders where UserID=UserIDParam AND Status='Open')) THEN
SELECT OrderID INTO OrderIDVar FROM _newapp_shop_orders WHERE UserID=UserIDParam AND Status='Open';
IF (NOT EXISTS(SELECT * FROM _newapp_shop_order_items WHERE OrderID=OrderIDVar AND EventID=EventIDParam)) THEN
-- User has open order so create a new order item!
SELECT OrderID INTO OrderIDVar FROM _newapp_shop_orders WHERE UserID=UserIDParam AND Status='Open';
INSERT INTO _newapp_shop_order_items (OrderID, EventID, ActivityID, Price, Amount) VALUES (OrderIDVar, EventIDParam, NULL, (SELECT Price FROM _newapp_shop_products_events WHERE EventID=EventIDParam), 1);
-- SELECT 'InsertOrderItem' INTO Result;
SELECT ROW_COUNT() INTO Result;
ELSE
UPDATE _newapp_shop_order_items SET Amount=Amount + 1 WHERE OrderID=OrderIDVar AND EventID=EventIDParam;
-- SELECT 'UpdateOrderItem' INTO Result;
SELECT ROW_COUNT() INTO Result;
END IF;
ELSE
-- User has no open order so create a new order!
INSERT INTO _newapp_shop_orders (UserID, Created, Modified) VALUES (UserIDParam, Time, Time);
SELECT OrderID INTO OrderIDVar from _newapp_shop_orders where UserID=UserIDParam AND Status='Open';
IF (NOT EXISTS(SELECT * FROM _newapp_shop_order_items WHERE OrderID=OrderIDVar AND EventID=EventIDParam)) THEN
-- User has made a new order item so create a new order item!
INSERT INTO _newapp_shop_order_items (OrderID, EventID, ActivityID, Price, Amount) VALUES (OrderIDVar, EventIDParam, NULL, (SELECT Price FROM _newapp_shop_products_events WHERE EventID=EventIDParam), 1);
-- SELECT 'InsertOrderItem' INTO Result;
SELECT ROW_COUNT() INTO Result;
ELSE
-- User has made an existing order item so update the order item!
UPDATE _newapp_shop_order_items SET Amount=Amount + 1 WHERE OrderID=OrderIDVar AND EventID = EventIDParam;
-- SELECT 'UpdateOrderItem' INTO Result;
SELECT ROW_COUNT() INTO Result;
END IF;
END IF;
END$$
DELIMITER ;
It inserts a new order, order item or updates an existing item, it returns the modified rows to check if the insert was succesfull. When i execute the procedure in phpmyadmin it result the inserted rows fine and it inserts the proper rows in the belonging tables. But when i run this procedure from php, it returns just NULL, this is my php code:
public function pushItemToCart($userID, $item)
{
$itemList = json_decode($item);
if ($itemList->Type == 'events') {
$sql = 'CALL `InsertNewEventOrderItems`(?, ?, ?, #Result)';
} elseif ($itemList->Type == 'activities') {
$sql = 'CALL `InsertNewActivityOrderItems`(?, ?, ?, #Result)';
}
if ($query = $this->DB->prepare($sql)) {
$nowFormat = mktime(
date('H'),
date('i'),
date('s'),
date('m'),
date('d'),
date('Y')
);
$query->bind_param('isi', $userID, $nowFormat, $itemList->Id);
$query->execute();
$query->next_result();
$select = $this->DB->query('SELECT #Result;');
//$query->execute();
$result = $select->fetch_assoc();
$query->close();
echo json_encode($result['#Result']);
exit();
} else {
$error = $this->DB->errno . ' ' . $this->DB->error;
echo json_encode('405'); // 1054 Unknown column 'foo' in 'field list'
exit();
}
}
So i expect the result to be 1 or 2 or 0
I have solved it by removing the second execute and fully checking the procedure on errors. Correct code in the main post.

php Num rows of stored procedure MySQL

I have a stored procedure in MySQL. I call my procedure in php as
$qry = $mysqli->prepare("CALL seen_table()");
i tried to get the resulting rows by
$row = $qry->num-rows;
but its resulting 0 even there is a resulting set.Then I tried also to put output parameter in my procedure where in inside of my proc is ...
SELECT COUNT(*) INTO cnt FROM TBL
...then this is my codes
$qry = $mysqli->prepare("CALL seen_table(#cnt)");
$qry1 = $mysqli->query("SELECT #cnt");
$row = $qry1->num_rows;
then now its always results 1 even there is no count. when i try to execute CALL and SELECT #cnt in Mysql . if there is no count. the result will be
|#cnt|
|(null)|
does null really count as one?please help. thanks a lot.
EDIT: Added seen_table Procedure codes
DELIMITER $$
USE `xiroosco_mundoxiro`$$
DROP PROCEDURE IF EXISTS `seen_table`$$
CREATE DEFINER=`xiroosco`#`103.16.170.%` PROCEDURE `seen_table`(bound
VARCHAR(255),IN cmntId INT,IN cmntViewID VARCHAR(255),OUT viewCNT INT)
BEGIN
DECLARE seen_ID INT DEFAULT 0;
DECLARE seen_notifica_ID INT DEFAULT 0;
DECLARE seen_viewers TEXT;
DECLARE occurance INT DEFAULT 0;
DECLARE i INT DEFAULT 0;
DECLARE splitted_value INT;
DECLARE done INT DEFAULT 0;
DECLARE cur1 CURSOR FOR SELECT seen.seen_ID, seen.seen_notifica_ID,
seen.seen_viewers
FROM seen
WHERE seen.seen_notifica_ID = cmntId;
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
DROP TEMPORARY TABLE IF EXISTS seen2;
CREATE TEMPORARY TABLE seen2(
`seen_ID` INT NOT NULL,
`seen_notifica_ID` INT NOT NULL,
`seen_viewers` VARCHAR(255) NOT NULL
) ENGINE=MEMORY;
OPEN cur1;
read_loop: LOOP
FETCH cur1 INTO seen_ID,seen_notifica_ID, seen_viewers;
IF done THEN
LEAVE read_loop;
END IF;
SET occurance = (SELECT LENGTH(seen_viewers) -
LENGTH(REPLACE(seen_viewers, bound, '')) +1);
SET i=1;
WHILE i <= occurance DO
SET splitted_value = (SELECT
REPLACE(SUBSTRING(SUBSTRING_INDEX(seen_viewers, bound, i),
LENGTH(SUBSTRING_INDEX(seen_viewers, bound, i - 1)) + 1), ',', ''));
INSERT INTO seen2 VALUES (seen_ID,seen_notifica_ID, splitted_value);
SET i = i + 1;
END WHILE;
END LOOP;
IF cmntViewID = "*" THEN
SELECT * FROM seen2 GROUP BY seen2.seen_viewers;
SELECT COUNT(*) INTO viewCNT FROM seen2;
ELSE
SELECT * FROM seen2 WHERE seen2.seen_viewers = cmntViewID GROUP BY
seen2.seen_viewers;
SELECT seen_ID INTO viewCNT FROM seen2 WHERE seen2.seen_viewers =
cmntViewID GROUP BY seen2.seen_viewers;
END IF;
CLOSE cur1;
END$$
DELIMITER ;
this is how i call my procedure example
CALL seen_table (',',2995,'356',#count);
NULL counts as one, but as far as I can see it is against the ANSI standard: If there are not results, there should also be no NULL returned.
To get a row count returned from a procedure (or any result set) in MySQL, there are information functions.
I do not know which select you want to return the count, so just giving you an example:
{your procedure before this part}
IF cmntViewID = "*" THEN
SELECT SQL_CALC_FOUND_ROWS * FROM seen2 GROUP BY seen2.seen_viewers;
SELECT COUNT(*) INTO viewCNT FROM seen2;
ELSE
SELECT * FROM seen2 WHERE seen2.seen_viewers = cmntViewID GROUP BY
seen2.seen_viewers;
SELECT seen_ID INTO viewCNT FROM seen2 WHERE seen2.seen_viewers =
cmntViewID GROUP BY seen2.seen_viewers;
END IF;
{your procedure after this part}
Then execute:
CALL seen_table();
SELECT FOUND_ROWS();
That will return the number of rows in the "SELECT SQL_CALC_FOUND_ROWS * FROM seen2 GROUP BY seen2.seen_viewers;" query.

MySQL Iterate through elements of a split string and do things with them

I have a cell in one of my MySQL community edition 5.1 tables. The contents are always certain number(s). If there is more than one number, then it is delimited by an ; semi-colon.
For example:
| Column |
1ar8fj
99dkek;adjidk3;dajdid
divdae;dadjid;
NULL
dkadjcud;dasd;adfkvdo
dkjfakj
...
I need make some code that takes each column value, splits it up by the ; and then uses each value after it was split up to do another query, and output the results.
I know I can do this with PHP but I don't need to make this into a webpage, so I was wondering if this is possible write in MySQL syntax? The PHP code would look something like this:
<?php
$result = $mysqli->query('select column from table;');
while ($row = $result->fetch_array($result)){
$id_numbers = explode($row[0],';');
foreach($id_numbers as $key => $val){
// do another query
$result2 = $mysqli->query('select * from table2 where col_val = "'.$val.'"');
while ($row2 = $result2->fetch_array($result2){
echo $row2[0].'<br>';
}
}
}
?>
Is this possible directly in MySQL syntax?
Thanks!!!
PHEW. Okay. I finally got it working, but here's a solution as a stored procedure that takes a string as an input for the delimiter and is runs on the given table called testtable
--Procedure: sprecursplit
--DROP PROCEDURE IF EXISTS sprecursplit;
DELIMITER |
CREATE PROCEDURE sprecursplit
(
IN delim nvarchar(255)
)
BEGIN
declare tdone tinyint unsigned default(0);
declare depth int unsigned default(1);
declare datas nvarchar(255) default('');
declare done tinyint unsigned default(0);
declare dlength int unsigned default(1);
declare hlength int unsigned default(0);
declare pos int unsigned default(1);
declare runpos int unsigned default(1);
declare slice nvarchar(255) default('');
drop table if exists allsubs;
create temporary table allsubs(id int unsigned auto_increment, val nvarchar(255), primary key (id))engine = memory;
while tdone <> 1 do
if depth <= (select count(*) from testtable) then
select t.datastring into datas from testtable t where t.id = depth limit 1;
if length(datas) > 0 then
set dlength = length(delim);
set hlength = length(datas);
set pos = 1;
set runpos = 1;
set slice = '';
set done = 0;
if hlength > 0 then
while done <> 1 do
if runpos > hlength then
set done = 1;
else
set pos = locate(delim, substring(datas from runpos));
if pos <> 1 then
if pos > 1 then
set slice = substring(datas from runpos for (pos - 1));
else
set slice = substring(datas from runpos);
end if;
insert into allsubs (val) values (slice);
end if;
if pos = 0 then
set runpos = runpos + hlength;
else
set runpos = runpos + pos + dlength - 1;
end if;
end if;
end while;
end if;
end if;
set depth = depth + 1;
else
set tdone = 1;
end if;
end while;
select * from allsubs;
drop table allsubs;
END|
DELIMITER ;

Echoing out a field and a value from one row while running a query that ranks all rows

I have a MySQL query called $sqlStr5 that ranks rows by a metric called totalScore2. One of the fields that $sqlStr5 returns is called username.
I would like to echo out the rank and the value of totalScore2 where username equals a variable called $u.
How can I do this?
Below is what I have so far.
Thanks in advance,
John
$result = mysql_query($sqlStr5);
$count = 1;
$arr = array();
while ($row = mysql_fetch_array($result)) {
echo '<div class="sitename1edit2a">'.$count++.'.</div>';
echo '<div class="sitename1edit2">'.number_format(($row["totalScore2"])).'</div>';
}
This should work:
$result = mysql_query($sqlStr5);
$count = 1;
while($row = mysql_fetch_array($result))
{
if($u == $row['username'])
{
echo '<div class="sitename1edit2a">'.$count.'</div>';
echo '<div class="sitename1edit2">'.number_format($row["totalScore2"]).'</div>';
}
$count++;
}
Note however that this is not the most efficient way. It should be possible to make a SQL query that returns the rank and the total score for user $u.
The following solution uses a stored procedure to work out a user's rank based on a simple ranking system which you could replace with your own more complex one.
call get_user_rank(<user_id>)
I've added another stored procedure which lists the top ten ranks and flags whether the user_id passed in is included in the top ten as I remember that being a requirement from another question of yours.
call list_top_ten_ranks(<user_id>)
Testing (call these sprocs from your php)
select count(*) from users;
count(*)
========
250000
call get_user_rank(54193);
-- 0:00:00.300: Query OK
call get_user_rank(1);
-- 0:00:00.291: Query OK
call list_top_ten_ranks(54193);
-- 0:00:00.208: Query OK
call list_top_ten_ranks(1);
-- 0:00:00.215: Query OK
PHP
$result = $conn->query(sprintf("call get_user_rank(%d)", 1));
$row = $result->fetch_assoc();
$result->close();
echo sprintf("user = %s rank = %s points = %s<br/>",
$row["user_id"],$row["rank"],$row["points"]);
$conn->close();
Hope some of this proves useful.
Script
-- TABLES
drop table if exists users;
create table users
(
user_id int unsigned not null auto_increment primary key,
points smallint unsigned not null default 0
)
engine=innodb;
-- PROCEDURES
drop procedure if exists get_user_rank;
delimiter #
create procedure get_user_rank
(
p_user_id int unsigned
)
proc_main:begin
-- replace this simple ranking method by your own
set #rank = 0;
create temporary table tmp engine=memory
select
#rank:=#rank+1 as rank,
u.user_id,
u.points
from
users u
order by
u.points desc, u.user_id;
select * from tmp where user_id = p_user_id;
drop temporary table if exists tmp;
end proc_main #
delimiter ;
drop procedure if exists list_top_ten_ranks;
delimiter #
create procedure list_top_ten_ranks
(
p_user_id int unsigned
)
proc_main:begin
-- replace this simple ranking method by your own
set #rank = 0;
set #in_top_ten = 0;
create temporary table tmp engine=memory
select
#rank:=#rank+1 as rank,
u.user_id,
u.points
from
users u
order by
u.points desc, u.user_id
limit 10;
if exists (select 1 from tmp where user_id = p_user_id) then
set #in_top_ten = 1;
end if;
select tmp.*, #in_top_ten as in_top_ten from tmp order by tmp.rank;
drop temporary table if exists tmp;
end proc_main #
delimiter ;

Categories