mysql: how to outfile csv - limit x number of rows? - mysql

I have some queries like this
(SELECT 'title', 'created_at', 'modified_at')
UNION
(select title, created_at, modified_at from table1 order by created_at
INTO OUTFILE 'result.csv'
CHARACTER SET utf8
FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' ESCAPED BY '"' LINES TERMINATED BY '\r\n');
and the the result is more than 10000 rows.
Is there a way to split this into multiple files?
I would like to limit the number of rows.
I know that I can do something like this:
select * from table1 limit 0, 1000 into outfile result1.csv
select * from table1 limit 1000, 1000 into outfile result2.csv
select * from table1 limit 2000, 1000 into outfile result3.csv
...
since there's more than 10000 rows, I am wondering if I can use some kind of loop.
Any advice and help will be greatly appreciated!
::EDIT::
I wrote repeat procedure but it gives me a syntax error.
DELIMITER ;;
CREATE PROCEDURE `reapeatloop`()
BEGIN
declare x int;
declare filename varchar(255);
declare lowlimit int;
SET x = 0;
REPEAT
set lowlimit = x*1000;
(select * from table1 limit lowlimit, 1000
INTO OUTFILE concat('post',x,'.csv')
CHARACTER SET utf8
FIELDS TERMINATED BY ',' ENCLOSED BY '"' ESCAPED BY '"' LINES
TERMINATED BY '\r\n');
SET x = x + 1;
UNTIL x > 19 END REPEAT;
END;;
DELIMITER ;
Inside this query, I used local variable to define csv and limit value and this gives syntax error. can anyone see what's wrong with this query?
as soon as I change this variable to some number, or string, it creates procedure successfully.

You have some mistake in your procedure. Specially check the SELECT INTO OUTFILE.
Be careful about the position of FROM.
In your case try this:
DROP PROCEDURE reapeatloop;
DELIMITER $$
CREATE PROCEDURE `reapeatloop`()
BEGIN
declare x int;
declare filename varchar(255);
declare lowlimit int;
SET x = 0;
REPEAT
set lowlimit = x*1000;
SET #fullOutputPath := CONCAT('"D:/post',x,'.csv"');
set #q1 := concat("select * INTO OUTFILE ", #fullOutputPath,
" CHARACTER SET utf8 FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\r\n'
from table1 limit ", lowlimit, ",1000");
select #q1;
prepare s1 from #q1;
execute s1;deallocate prepare s1;
SET x = x + 1;
UNTIL x > 19 END REPEAT;
END$$
DELIMITER ;

Related

MySQL export into Outfile issue

I created a Stored procedure with a cursor in order to get the info from a few tables into a temp table and then export the result into multiple files, one for each Client ID, the cursor and the loop get all the info right but I'm having syntax issues when creating each file
BEGIN
DECLARE id_cli INT;
DECLARE cur_id_cli CURSOR FOR SELECT id FROM cliente ORDER BY id;
OPEN cur_id_cli;
read_loop: LOOP
FETCH cur_id_cli INTO id_cli;
DROP TEMPORARY TABLE IF EXISTS tmp_reporte_enviadas_sucursal_mensual;
CREATE TEMPORARY TABLE tmp_reporte_enviadas_sucursal_mensual
SELECT * FROM
((SELECT 'Promo_id', 'Mensaje', 'Sucursal_id', 'Direccion') UNION ALL (SELECT
p.id,
p.mensaje,
s.id,
s.direccion
FROM
Usuario_Promo AS up, Promo_Sucursal AS ps, Cliente_Sucursal AS cs, Cliente AS c, Promo AS p, Sucursal AS s
WHERE
p.id = ps.Promo_id
AND up.promo_id = ps.id
AND up.recibido = 1
AND ps.Cliente_Sucursal_id = cs.id
AND cs.cliente_id = id_cli
AND DATE(up.fecha_recibido) BETWEEN (NOW() - INTERVAL 30 DAY) AND NOW()
AND s.id = cs.sucursal_id
ORDER BY p.id)) AS tmp_reporte_enviadas_sucursal_mensual;
SET #idCli = CAST(id_cli AS CHAR);
SET #fullOutputPath = CONCAT('D:/Octagon/Apps/Flyermob/Clientes/Reportes_Sql/reporte_enviadas_sucursal_mensual_',#idCli,'.csv');
SELECT * FROM
INTO OUTFILE #fullOutputPath
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"'
FROM tmp_reporte_enviadas_sucursal_mensual;
SELECT * FROM tmp_reporte_enviadas_sucursal_mensual;
END LOOP;
CLOSE cur_id_cli;
END
here's where I'm having the issue:
SELECT * FROM
INTO OUTFILE #fullOutputPath
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"'
FROM tmp_reporte_enviadas_sucursal_mensual;
Whenever I try to save, it says that there's a syntax problem near #fullOutputPath if I remove this section, the procedure run perfectly and I get all the results for each Client Id.
Just to get it out of the way id_cli is a primary key, so there's no chance of repeating and I have full permissions on #fullOutputPath.
Try this:
SELECT * FROM tmp_reporte_enviadas_sucursal_mensual
INTO OUTFILE #fullOutputPath
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"';
I managed to make it work with
SET #idCli = CAST(id_cli AS CHAR);
SET #fullOutputPath = CONCAT('flyermob/cliente/reporte_enviadas_sucursal_mensual_cliente_',#idCli,'.csv');
set #q1 := concat("SELECT * INTO OUTFILE '",#fullOutputPath,
"' FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"'
FROM tmp_reporte_enviadas_sucursal_mensual");
prepare s1 from #q1;
execute s1;
deallocate prepare s1;
It creates the file in the MySql default folder

Using Prepare and Execute

I have a table that has training records. Each record has a field with an Agency value. I have another table of just agency values. I want to export the records into CSV files for each agency. There are over 2 million records so I don't want to export the whole table and do it manually.
I've created a stored procedure that pulls a value from the agency_codes table using a cursor and uses that value in a select statement for the WHERE clause and part of the INTO OUTFILE name.
The procedure works but only for the first two values in the agency_codes table. On the third (ACB) value it gives the error Unknown column 'ACB' in 'where clause' I'm confused why it worked with the first two values then stops with the third.
Here is the procedure:
DELIMITER $$
DROP PROCEDURE IF EXISTS export_csv $$
CREATE PROCEDURE export_csv()
BEGIN
DECLARE agency_name VARCHAR(255);
DECLARE exit_loop BOOLEAN;
DECLARE agency_cursor CURSOR FOR
SELECT agency FROM agency_codes;
DECLARE CONTINUE HANDLER FOR NOT FOUND SET exit_loop = TRUE;
OPEN agency_cursor;
agency_loop: LOOP
FETCH agency_cursor INTO agency_name;
SET #sql_text = Concat("(select 'class_code','course_code','course_name','course_type','username','grade_type','score','letter_grade','is_passed','completion_date','completion_status','registration_date','registration_entry_status','registration_type','comment','first_name','last_name','class_name','agency') Union (select class_code,course_code,course_name,course_type,username,grade_type,score,letter_grade,is_passed,completion_date,completion_status,registration_date,registration_entry_status,registration_type,comment,first_name,last_name,class_name,agency from hrdis_oru where hrdis_oru.agency =", agency_name," into outfile 'C:/HDD/",agency_name,".csv' fields enclosed by '\"' terminated by ',' escaped by '\"' lines terminated by '\r\n')");
prepare s1 from #sql_text;
execute s1;
deallocate prepare s1;
IF exit_loop THEN
CLOSE agency_cursor;
LEAVE agency_loop;
END IF;
END LOOP agency_loop;
END $$
DELIMITER ;
The first few values in my Agency table are:
17
303
ACB
Actuary
agr
AIM
Any help would be great. Thanks.
If you think about what values #sql_text will hold on each iteration, the problem should be immediately obvious. Adding some whitespace for clarity:
(
select 'class_code','course_code','course_name','course_type','username',
'grade_type','score','letter_grade','is_passed','completion_date',
'completion_status','registration_date','registration_entry_status',
'registration_type','comment','first_name','last_name','class_name',
'agency'
) Union (
select class_code,course_code,course_name,course_type,username,
grade_type,score,letter_grade,is_passed,completion_date,
completion_status,registration_date,registration_entry_status,
registration_type,comment,first_name,last_name,class_name,
agency
from hrdis_oru
where hrdis_oru.agency =ACB
into outfile 'C:/HDD/ACB.csv'
fields enclosed by '\"'
terminated by ','
escaped by '\"'
lines terminated by '\r\n'
)
Note in particular, where htdis_oru.agency =ACB.
Since ACB has not been quoted, MySQL parses it as a schema object identifier and complains when it cannot find any such-named object (this isn't the case with the purely numeric agency names, since they are parsed as integers that subsequently get cast to strings during expression evaluation).
For MySQL to correctly parse non-numeric values as string literals, they must be quoted:
... where hrdis_oru.agency ='", agency_name, "' ...
^ ^
Naturally, this presents a problem should agency_name contain the ' string quotation character—any such occurrences must of course be escaped. MySQL handily provides a QUOTE() function for precisely this purpose:
... where hrdis_oru.agency =", QUOTE(agency_name), " ...
However, to guard against possible SQL injection attacks, you really should instead parameterise your prepared statement:
FETCH agency_cursor INTO #agency_name;
(You no longer need to DECLARE agency_name); then:
... where hrdis_oru.agency = ? into outfile CONCAT('C:/HDD/', ?, '.csv') ...
Followed by:
PREPARE s1 FROM #sql_text;
EXECUTE s1 USING #agency_name, #agency_name;
DEALLOCATE PREPARE s1;
Note that you can now also PREPARE the statement before entering the loop and simply EXECUTE it (with appropriate values) inside the loop—this should yield a slight performance improvement. Remember to DEALLOCATE after exiting the loop.
One final note: you should inspect exit_loop immediately after the FETCH command—otherwise you will end up attempting to execute the SELECT ... INTO OUTFILE statement a final time when there are no more agencies.
It may also be worth noting that, in this case, you have no particular need to use prepared statements at all. You could simply do the following:
CREATE PROCEDURE export_csv() BEGIN
DECLARE agency_name VARCHAR(255);
DECLARE agency_cursor CURSOR FOR SELECT agency FROM agency_codes;
DECLARE EXIT HANDLER FOR NOT FOUND CLOSE agency_cursor;
OPEN agency_cursor;
LOOP
FETCH agency_cursor INTO agency_name;
SELECT 'class_code','course_code','course_name','course_type','username',
'grade_type','score','letter_grade','is_passed','completion_date',
'completion_status','registration_date','registration_entry_status',
'registration_type','comment','first_name','last_name','class_name'
UNION ALL
SELECT class_code,course_code,course_name,course_type,username,
grade_type,score,letter_grade,is_passed,completion_date,
completion_status,registration_date,registration_entry_status,
registration_type,comment,first_name,last_name,class_name
FROM hrdis_oru
WHERE agency = agency_name
INTO OUTFILE CONCAT('C:/HDD/', agency_name, '.csv')
FIELDS ENCLOSED BY '"'
TERMINATED BY ','
ESCAPED BY '"'
LINES TERMINATED BY '\r\n';
END LOOP;
END

Mysql select multiple id's

I need to execute queries like this:
SELECT draw,SUM(uplata) as uplata, SUM(wonamount) as isplata,
SUM(uplata)- SUM(wonamount) as stanje,
COUNT(*) as ukupno_tiketa FROM macau.tickets
WHERE shop ='105' and status != 1 and draw = 1;
I need to execute that query 100 times changing only draw = 1 to draw = 2, draw = 3, etc...
How can i do this to get a new row for each select statement in mysql workbench to be able to export everything to csv file.??
First you need to write a store procedure with a while loop:
drop procedure if exists load_test_data;
delimiter #
create procedure load_test_data()
begin
declare v_max int unsigned default 100;
declare v_counter int unsigned default 0;
start transaction;
truncate table new_table;
while v_counter < v_max do
insert into new_table SELECT draw,SUM(uplata) as uplata, SUM(wonamount)
as isplata, SUM(uplata)- SUM(wonamount)
as stanje, COUNT(*) as ukupno_tiketa
FROM macau.tickets WHERE
shop ='105' and status != 1 and draw = v_counter;
set v_counter=v_counter+1;
end while;
commit;
end #
delimiter ;
and later you can call this procedure to run it:
call load_test_data();
And you can dump your new_table to a csv file:
SELECT *
FROM new_table
INTO OUTFILE '[path]/File.csv'
FIELDS TERMINATED BY ','
ENCLOSED BY '"'
LINES TERMINATED BY '\n'
NOTE: Replace [path] with the folder that you want the file to be in it. Keep in mind that the folder should be writable and all full permissions needs to be set. Read this

CONCAT string and INT in MYSQL and export big mysql table to small csv files

I am new in MYSQL programming. I want to select into outfile a very big table into several csv files using MYSQL loop script. My script is as following:
BEGIN
SET #t_lines=0;
SET #t_count=0;
SET #t_filepath='/home/ab/path/table_name_02212013_';
WHILE t_lines<=5000000
SET #t_filename=CONCAT(#t_filepath,CAST(#t_count as CHAR));
select * into outfile #t_filename fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' from table_name limit #t_lines,#t_lines+300000;
SET #t_lines= #t_lines+300000;
SET #t_count= #t_count+1;
END WHILE;
COMMIT;
END
I got a syntax error in line:
SET #t_filename=CONCAT(#t_filepath,CAST(#t_count as CHAR));
The syntax error caused by WHILE ... DO. Thanks for quick reply.
Here I still have syntax error at "#t_lines,#t_lines+300000" after "limit".
I figure it out, seems "limit" not allowed "#t_lines+300000" to give the line range. It can be fixed by giving a new variable:
BEGIN
DECLARE t_lines INT DEFAULT 0;
DECLARE t_count INT DEFAULT 0;
DECLARE t_endlines INT DEFAULT 300000;
DECLARE t_linerange INT DEFAULT 300000;
SET #t_filepath='/home/ab/path/table_name_02212013_';
WHILE t_lines<=5000000 DO
SET #t_filename=CONCAT(#t_filepath,CAST(#t_count as CHAR));
select * into outfile '#t_filename' fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' from table_name limit t_lines,t_endlines;
SET t_lines= t_lines+t_linerange;
SET t_endlines= t_endlines+t_linerange
SET t_count= t_count+1;
END WHILE;
COMMIT;
END
Thanks very much
The syntax for a WHILE loop in MySQL requires a DO;
WHILE t_lines<=5000000 DO
...
END WHILE;
The filename argument for INTO OUTFILE must be a literal string. You can't use variables or expressions as the filename.
The arguments to LIMIT must be integer constants. You can use parameter placeholders (?), or stored procedure parameters or local procedure variables (the ones you DECLARE). But you can't use session variables (the ones with # prefix), nor can you use expressions.
You're going to have to create the query as an SQL string, interpolating the values into the string.
Then use that SQL string with PREPARE and EXECUTE.
Try :
convert(columnName, char)
-->
BEGIN
SET #t_lines=0;
SET #t_count=0;
SET #t_filepath='/home/ab/path/table_name_02212013_';
WHILE t_lines<=5000000
SET #t_filename=CONCAT(#t_filepath,convert(#t_count, CHAR));
select * into outfile #t_filename fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' from table_name limit #t_lines,#t_lines+300000;
SET #t_lines= #t_lines+300000;
SET #t_count= #t_count+1;
END WHILE;
COMMIT;
END

get current date in stored procedure in mysql

I have created a stored procedure in mysql
DELIMITER //
CREATE PROCEDURE simpleproc ()
BEGIN
SELECT YEAR(dtm),
WEEK(dtm),
b.x_title,
COUNT(b.x_id)
INTO OUTFILE '/appdata/reports01/result.csv'
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '"'
LINES TERMINATED BY '\n'
FROM tbla a
JOIN tblb b ON b.x_id = a.x_id
WHERE plan_grp_id IN (10031, 10034)
AND dtm > '20110101'
GROUP BY YEAR(dtm), WEEK(dtm), a.x_id;
END //
delimiter ;
I need to do two things
INTO OUTFILE '/appdata/reports01/result.csv' must be like "INTO OUTFILE '/appdata/reports01/result-current-timestamp.csv'
dtm > '20110101' must be dtm > CURDATE() + 0 .
This:
SELECT CURDATE() + 0
...gets me the format '20110101', but I can't use that in the query like dtm > CURDATE() + 0
Use variables.
DECLARE outfile_name varchar(300);
Get the current timestamp and append it to the remaining part of the string.
Use INTO keyword to get the output of the SELECT statement into a variable.
Use CONCAT() for concatinating 2 strings.