Grouping multiple rows on a condition using while loop - mysql

In MySQL I have a table that looks like this:
This is a little complex as mysql doesnot support ranking window functions and it does not even support CTEs so I'm trying to implement it using while loop
If there is any relationship between the imported and section then they should be grouped together. Depending on the relationship they should be grouped as 1,2,3,4,....
But in this scenario my rn is always 1 and not sure why its not incrementing. Can you help me review this query?
I'm not sure how to go around this.
Example creation script:
create table temp1 (
id int, imported int, section int, rn int, checked int default 0
);
insert into temp1(id, section, rn) values (204, 718, 0);
insert into temp1(id, imported, section, rn) values (997,718,034,0);
insert into temp1(id, imported, section, rn) values (998,034,055,0);
insert into temp1(id, imported, section, rn) values (111,453,234,0);
insert into temp1(id, section, rn) values (908, 453,0);
insert into temp1(id, imported, section, rn) values (231,234,890,0);
insert into temp1(id, section, rn) values (342, 567,0);
My End Result should look like:
I'have tried with while loop too creating a stored procedure:
DROP PROCEDURE IF EXISTS sp_recursiveimport;
Delimiter $$
CREATE PROCEDURE sp_recursiveimport() -- (IN rnX integer)
BEGIN
DECLARE n INT DEFAULT 0; DECLARE i,j,k INT DEFAULT 0; SELECT COUNT(*) FROM temp1 INTO n;
SET i=0; set #rn = 1; -- set #k = 0;
WHILE i<n DO
set j = 0; select i;
set #sec = (select ifnull(section,0) FROM temp1 LIMIT i,1);
set #imp = (select ifnull(imported,0) FROM temp1 LIMIT i,1); select #imp, #sec;
update1: while j<n do select j;
-- if j=0 then
if (select ifnull(imported,0) from temp1 limit j,1) = #sec and (select checked from temp1 limit j,1) = 0 then
set #update = concat('update temp1 set rn = 1, checked = 1 where imported = ',#sec); select #update; PREPARE stmt_name FROM #update; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set #update1 = concat('update temp1 set rn = 1, checked = 1 where section = ',#sec); select #update1; PREPARE stmt_name FROM #update1; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set k = j;
end if;
if (select ifnull(section,0) from temp1 limit j,1) = #imp and (select checked from temp1 limit j,1) = 0 then
set #update3 = concat('update temp1 set rn = 1, checked = 1 where section = ',#imp); select #update3; PREPARE stmt_name FROM #update3; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set #update4 = concat('update temp1 set rn = 1, checked = 1 where imported = ',#imp); select #update4; PREPARE stmt_name FROM #update4; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set k = j;
end if;
-- set #sec = (select ifnull(imported,0) from temp1 limit k,1);
-- set #imp = (select ifnull(section,0) from temp1 limit k,1); select #sec, #imp;
set j= j+1;
end while update1;
set i = i + 1;
END WHILE;
END;
$$
delimiter;
Not sure why its not working.

Related

Stored procedure, using variable no result

The variable #total in DESC LIMIT ? isn't working. If I manually set DESC LIMIT 3 then the sp runs fine and returns rows. I also tried placing the SET total in between PREPARE.
Here is the stored procedure:
DELIMITER //
CREATE PROCEDURE getTotalNET()
BEGIN
DECLARE total INT;
SET total := (SELECT COUNT(*) as item_count FROM items WHERE hostid = '12345' and key_ LIKE '%_net%' AND STATUS = '0' );
PREPARE STMT FROM
" SELECT DISTINCT itemid, clock, VALUE, ns FROM history_uint WHERE itemid IN (SELECT itemid FROM items WHERE hostid = '12345' and key_ LIKE '%_net%' AND STATUS = '0' ) AND clock >= UNIX_TIMESTAMP(NOW() - INTERVAL 120 SECOND) ORDER BY clock DESC LIMIT ?";
EXECUTE STMT USING #total ;
END //
DELIMITER ;

Stored procedure is too slow in mysql

I have a routine. But it' s too slow. How can I improve the query?
My records: http://www.sqlfiddle.com/#!9/14cceb/1/0
My query:
CREATE DEFINER = 'root'#'localhost'
PROCEDURE example.ssa()
BEGIN
drop table if exists gps_table;
drop table if exists exam_datas;
CREATE TEMPORARY TABLE gps_table(ID int PRIMARY KEY AUTO_INCREMENT,timei
int,
trun_date_time datetime, tadd_meter int, tin_here int null);
insert into gps_table(timei,trun_date_time,tadd_meter,tin_here) select
imei, run_date_time, add_meter, in_here from example_table;
CREATE TEMPORARY TABLE exam_datas(ID int PRIMARY KEY AUTO_INCREMENT,vimei
int, vbas_run_date_time datetime, vbit_run_date_time datetime, vdifff int);
select tin_here from gps_table limit 1 into #onceki_durum;
select count(id) from gps_table into #kayit_sayisi;
set #i = 1;
set #min_mes = 0;
set #max_mes = 0;
set #frst_id = 0;
set #imei = 0;
set #run_date_time = '0000-00-00 00:00:00';
set #run_date_time2 = '0000-00-00 00:00:00';
myloop: WHILE (#i <= #kayit_sayisi) DO
select tin_here from gps_table where id = #i into #in_here_true;
if (#in_here_true = 1) then
select id,trun_date_time, tadd_meter from gps_table where id = #i into #frst_id,#run_date_time2, #min_mes;
select id from gps_table where id > #frst_id and tin_here =0 order by id asc limit 1 INTO #id;
SET #id = #id-1;
select id, timei, trun_date_time, tadd_meter from gps_table
where id = #id and tin_here =1 limit 1 into #i, #imei, #run_date_time, #max_mes;
if(#i-#frst_id>3) then
set #i:=#i+1;
insert into exam_datas(vimei,vbas_run_date_time,vbit_run_date_time,vdifff) Values (#imei, #run_date_time2, #run_date_time, #max_mes-#min_mes);
SELECT * FROM exam_datas;
SET #asd =1;
elseif 1=1 then
set #i:=#i+1;
End if;
ELSEIF 1=1
THEN SET #i:=#i+1;
End if;
IF (#i = #kayit_sayisi)
THEN set #tamam =1; LEAVE myloop;
END IF;
END WHILE myloop;
select DISTINCT * from exam_datas;
drop table if exists exam_datas;
drop table if exists gps_table;
END
I need: id= 6 first true and id= 11 last_true
firs_trure - last_true = 304-290= 14
id=14 first true and id=18 last_true
firs_true - last_true = 332-324= 8
This routine is too slow.
MySql version is 5.7 and There are 2 milions record in the table.
UPDATE:
Query is here. HERE
Thank you #LukStorms
It's possible to get such results in 1 query.
Thus avoiding a WHILE loop over records.
This example works without using window functions. Just using variables inside the query to calculate a rank. Which is then used to get the minimums and maximums of the groups.
select
imei,
min(run_date_time) as start_dt,
max(run_date_time) as stop_dt,
max(add_meter) - min(add_meter) as diff
from
(
select imei, id, run_date_time, add_meter, in_here,
case
when #prev_imei = imei and #prev_ih = in_here then #rnk
when #rnk := #rnk + 1 then #rnk
end as rnk,
#prev_imei := imei as prev_imei,
#prev_ih := in_here as prev_ih
from example_table t
cross join (select #rnk := 0, #prev_ih := null, #prev_imei := null) vars
order by imei, id, run_date_time
) q
where in_here = 1
group by imei, rnk
having count(*) > 4
order by imei, min(id);
In the procedure such query can be used to fill that final temporary table.
A test on db<>fiddle here

Relation between rows ranking

I have a table that looks like this:
If there is any relationship between the imported and section then they should be grouped together. Depending on the relationship they should be grouped as 1,2,3,4,....
I tried a query that looks like this:
select sec.section,sec.id, sec.imported, sec.id,
case when imp.imported = sec.section or imp.imported is null then 1 ELSE
2 end as rn
from
( select section, id, imported from temp1) sec
left outer join
(
select imported, Section from temp1
) imp on imp.imported = sec.section
But in this scenario my rn is always 1. Can you help me review this query?
I'm not sure how to go around this. Do we need to use a while loop and do it or can this be done using a query?
Example creation script:
create table temp1 (
id int, imported int, section int, rn int, checked int default 0
);
insert into temp1(id, section, rn) values (204, 718, 0);
insert into temp1(id, imported, section, rn) values (997,718,034,0);
insert into temp1(id, imported, section, rn) values (998,034,055,0);
insert into temp1(id, imported, section, rn) values (111,453,234,0);
insert into temp1(id, section, rn) values (908, 453,0);
insert into temp1(id, imported, section, rn) values (231,234,890,0);
insert into temp1(id, section, rn) values (342, 567,0);
My End Result should look like:
I'have tried with while loop too creating a stored procedure:
DROP PROCEDURE IF EXISTS sp_recursiveimport;
Delimiter $$
CREATE PROCEDURE sp_recursiveimport() -- (IN rnX integer)
BEGIN
DECLARE n INT DEFAULT 0; DECLARE i,j,k INT DEFAULT 0; SELECT COUNT(*) FROM temp1 INTO n;
SET i=0; set #rn = 1; -- set #k = 0;
WHILE i<n DO
set j = 0; select i;
set #sec = (select ifnull(section,0) FROM temp1 LIMIT i,1);
set #imp = (select ifnull(imported,0) FROM temp1 LIMIT i,1); select #imp, #sec;
update1: while j<n do select j;
-- if j=0 then
if (select ifnull(imported,0) from temp1 limit j,1) = #sec and (select checked from temp1 limit j,1) = 0 then
set #update = concat('update temp1 set rn = 1, checked = 1 where imported = ',#sec); select #update; PREPARE stmt_name FROM #update; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set #update1 = concat('update temp1 set rn = 1, checked = 1 where section = ',#sec); select #update1; PREPARE stmt_name FROM #update1; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set k = j;
end if;
if (select ifnull(section,0) from temp1 limit j,1) = #imp and (select checked from temp1 limit j,1) = 0 then
set #update3 = concat('update temp1 set rn = 1, checked = 1 where section = ',#imp); select #update3; PREPARE stmt_name FROM #update3; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set #update4 = concat('update temp1 set rn = 1, checked = 1 where imported = ',#imp); select #update4; PREPARE stmt_name FROM #update4; EXECUTE Stmt_name; DEALLOCATE prepare stmt_name;
set k = j;
end if;
-- set #sec = (select ifnull(imported,0) from temp1 limit k,1);
-- set #imp = (select ifnull(section,0) from temp1 limit k,1); select #sec, #imp;
set j= j+1;
end while update1;
set i = i + 1;
END WHILE;
END;
$$
delimiter;
Not sure why its not working.
This is not an answer to the question since, honestly, I don't know how to write this query in MySQL 5.x.
Anyway, at least I wanted to document the answer using recursive CTEs, available on MySQL 8.0 or newer. Here it is:
with recursive sect as (
select id, imported, section, row_number() over() as rn
from temp1 where imported is null
union all
select t.id, t.imported, t.section, s.rn
from temp1 t
join sect s on t.imported = s.section
)
select * from sect order by rn;
Result:
id imported section rn
--- -------- ------- --
998 34 55 1
204 <null> 718 1
997 718 34 1
231 234 890 2
908 <null> 453 2
111 453 234 2
342 <null> 567 3

Enhancing MySQL Stored Procedure performance

i want to enhance performance of following Mysql Stored Procedure.
i am trying to XML with 100 items. and it is Taking almost 90 seconds. and Its too much.
Please check my following SP.
DELIMITER $$
DROP PROCEDURE IF EXISTS `kshitij`.`Insert_date` $$
CREATE DEFINER=`root`#`localhost` PROCEDURE `Insert_date`()
BEGIN
DECLARE itemstart INTEGER default 1;
DECLARE valuestart INTEGER default 1;
DECLARE itemcount INTEGER;
DECLARE valuecount INTEGER;
SET #xml = '<items>
<size>2</size>
<item>
<value columntype="0" columnid="23">Single Line Text_0</value>
<value columntype="1" columnid="24">Multi Line Text_0</value>
<value columntype="2" columnid="25">Number_0</value>
<value columntype="3" columnid="26">Link_0</value>
<value columntype="4" columnid="27">Image_0</value>
<value columntype="5" columnid="28">Date time_0</value>
</item>
TRYING WITH 100 ITEMS.
</items>';
SET itemcount = (SELECT ExtractValue(#xml, 'count(//item)'));
WHILE itemstart < itemcount
DO
SET valuecount = (SELECT ExtractValue(#xml, 'count(/items/item[itemstart]/value)'));
SET valuestart = 1;
WHILE valuestart < valuecount
DO
IF((SELECT ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columntype')) = '0') THEN
INSERT INTO singleline_text(value) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]'));
INSERT INTO sheet_items(c_id, value_id) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columnid'), (SELECT MAX(id) FROM singleline_text));
END IF;
IF((SELECT ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columntype')) = '1') THEN
INSERT INTO multiline_text(value) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]'));
INSERT INTO sheet_items(c_id, value_id) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columnid'), (SELECT MAX(id) FROM multiline_text));
END IF;
IF((SELECT ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columntype')) = '2') THEN
INSERT INTO number(value) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]'));
INSERT INTO sheet_items(c_id, value_id) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columnid'), (SELECT MAX(id) FROM number));
END IF;
IF((SELECT ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columntype')) = '3') THEN
INSERT INTO link(value) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]'));
INSERT INTO sheet_items(c_id, value_id) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columnid'), (SELECT MAX(id) FROM link));
END IF;
IF((SELECT ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columntype')) = '4') THEN
INSERT INTO image(value) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]'));
INSERT INTO sheet_items(c_id, value_id) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columnid'), (SELECT MAX(id) FROM image));
END IF;
IF((SELECT ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columntype')) = '5') THEN
INSERT INTO date_time(value) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]'));
INSERT INTO sheet_items(c_id, value_id) VALUES(ExtractValue(#xml, '/items/item[$itemstart]/value[$valuestart]/#columnid'), (SELECT MAX(id) FROM date_time));
END IF;
SET valuestart = valuestart + 1;
END WHILE;
SET itemstart = itemstart + 1;
end while;
END $$
DELIMITER ;
Thanks.

How could I simplify this mysql statement?

I use follow mysql statement to get some info from mysql via php.
( SELECT *
FROM mytable
WHERE qid NOT IN ({$used['used']})
AND level = 1
ORDER BY RAND()
LIMIT 5)
UNION
( SELECT *
FROM app_mytable _qt
WHERE qid NOT IN ({$used['used']})
AND level = 2
ORDER BY RAND()
LIMIT 5)
UNION
( SELECT *
FROM app_mytable _qt
WHERE qid NOT IN ({$used['used']})
AND level = 3
ORDER BY RAND()
LIMIT 5)
UNION
( SELECT *
FROM app_mytable _qt
WHERE qid NOT IN ({$used['used']})
AND level = 4
ORDER BY RAND()
LIMIT 5)
$used['used'] is a set of qid that likes 23,31,653,147,146,134,6.....
How could I simplify this mysql statement?
You can use user variables to keep running totals by groups. This is untested but something like the following should work:
select *,
#running:=#previous:=NULL
from (
select *,
#running:=if(#previous=inside.level,#running,0)+1 as TOTAL,
#previous:=inside.level
from (
select *
from mytable
where qid NOT IN ({$used['used']})
order by level, rand()
) as inside
)as outside
where TOTAL < 5;
Some dynamic sql can eliminate the copy pasting for ya... that way you can easily extend this out for as many levels as you like, just mod the max_level.
DECLARE current_level INT; SET current_level = 1;
DECLARE max_level INT; SET max_level = 4;
DECLARE full_sql VARCHAR(1000); SET full_sql = '';
DECLARE base_sql VARCHAR(1000); SET base_sql = '(SELECT * FROM mytable WHERE qid NOT IN ({$used['used']}) AND level = ? ORDER BY RAND() limit 5) ';
WHILE current_level <= max_level
SET full_sql = CONCAT(full_sql, REPLACE(base_sql, '?', current_level));
IF current_level < max_level THEN
SET full_sql = CONCAT(full_sql, ' UNION ');
END IF
SET current_level = current_level + 1;
END WHILE
PREPARE s1 FROM full_sql;
EXECUTE s1;
DEALLOCATE PREPARE s1;