I have some queries like this
(SELECT 'title', 'created_at', 'modified_at')
UNION
(select title, created_at, modified_at from table1 order by created_at
INTO OUTFILE 'result.csv'
CHARACTER SET utf8
FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' ESCAPED BY '"' LINES TERMINATED BY '\r\n');
and the the result is more than 10000 rows.
Is there a way to split this into multiple files?
I would like to limit the number of rows.
I know that I can do something like this:
select * from table1 limit 0, 1000 into outfile result1.csv
select * from table1 limit 1000, 1000 into outfile result2.csv
select * from table1 limit 2000, 1000 into outfile result3.csv
...
since there's more than 10000 rows, I am wondering if I can use some kind of loop.
Any advice and help will be greatly appreciated!
::EDIT::
I wrote repeat procedure but it gives me a syntax error.
DELIMITER ;;
CREATE PROCEDURE `reapeatloop`()
BEGIN
declare x int;
declare filename varchar(255);
declare lowlimit int;
SET x = 0;
REPEAT
set lowlimit = x*1000;
(select * from table1 limit lowlimit, 1000
INTO OUTFILE concat('post',x,'.csv')
CHARACTER SET utf8
FIELDS TERMINATED BY ',' ENCLOSED BY '"' ESCAPED BY '"' LINES
TERMINATED BY '\r\n');
SET x = x + 1;
UNTIL x > 19 END REPEAT;
END;;
DELIMITER ;
Inside this query, I used local variable to define csv and limit value and this gives syntax error. can anyone see what's wrong with this query?
as soon as I change this variable to some number, or string, it creates procedure successfully.
You have some mistake in your procedure. Specially check the SELECT INTO OUTFILE.
Be careful about the position of FROM.
In your case try this:
DROP PROCEDURE reapeatloop;
DELIMITER $$
CREATE PROCEDURE `reapeatloop`()
BEGIN
declare x int;
declare filename varchar(255);
declare lowlimit int;
SET x = 0;
REPEAT
set lowlimit = x*1000;
SET #fullOutputPath := CONCAT('"D:/post',x,'.csv"');
set #q1 := concat("select * INTO OUTFILE ", #fullOutputPath,
" CHARACTER SET utf8 FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\r\n'
from table1 limit ", lowlimit, ",1000");
select #q1;
prepare s1 from #q1;
execute s1;deallocate prepare s1;
SET x = x + 1;
UNTIL x > 19 END REPEAT;
END$$
DELIMITER ;
I created a Stored procedure with a cursor in order to get the info from a few tables into a temp table and then export the result into multiple files, one for each Client ID, the cursor and the loop get all the info right but I'm having syntax issues when creating each file
BEGIN
DECLARE id_cli INT;
DECLARE cur_id_cli CURSOR FOR SELECT id FROM cliente ORDER BY id;
OPEN cur_id_cli;
read_loop: LOOP
FETCH cur_id_cli INTO id_cli;
DROP TEMPORARY TABLE IF EXISTS tmp_reporte_enviadas_sucursal_mensual;
CREATE TEMPORARY TABLE tmp_reporte_enviadas_sucursal_mensual
SELECT * FROM
((SELECT 'Promo_id', 'Mensaje', 'Sucursal_id', 'Direccion') UNION ALL (SELECT
p.id,
p.mensaje,
s.id,
s.direccion
FROM
Usuario_Promo AS up, Promo_Sucursal AS ps, Cliente_Sucursal AS cs, Cliente AS c, Promo AS p, Sucursal AS s
WHERE
p.id = ps.Promo_id
AND up.promo_id = ps.id
AND up.recibido = 1
AND ps.Cliente_Sucursal_id = cs.id
AND cs.cliente_id = id_cli
AND DATE(up.fecha_recibido) BETWEEN (NOW() - INTERVAL 30 DAY) AND NOW()
AND s.id = cs.sucursal_id
ORDER BY p.id)) AS tmp_reporte_enviadas_sucursal_mensual;
SET #idCli = CAST(id_cli AS CHAR);
SET #fullOutputPath = CONCAT('D:/Octagon/Apps/Flyermob/Clientes/Reportes_Sql/reporte_enviadas_sucursal_mensual_',#idCli,'.csv');
SELECT * FROM
INTO OUTFILE #fullOutputPath
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"'
FROM tmp_reporte_enviadas_sucursal_mensual;
SELECT * FROM tmp_reporte_enviadas_sucursal_mensual;
END LOOP;
CLOSE cur_id_cli;
END
here's where I'm having the issue:
SELECT * FROM
INTO OUTFILE #fullOutputPath
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"'
FROM tmp_reporte_enviadas_sucursal_mensual;
Whenever I try to save, it says that there's a syntax problem near #fullOutputPath if I remove this section, the procedure run perfectly and I get all the results for each Client Id.
Just to get it out of the way id_cli is a primary key, so there's no chance of repeating and I have full permissions on #fullOutputPath.
Try this:
SELECT * FROM tmp_reporte_enviadas_sucursal_mensual
INTO OUTFILE #fullOutputPath
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"';
I managed to make it work with
SET #idCli = CAST(id_cli AS CHAR);
SET #fullOutputPath = CONCAT('flyermob/cliente/reporte_enviadas_sucursal_mensual_cliente_',#idCli,'.csv');
set #q1 := concat("SELECT * INTO OUTFILE '",#fullOutputPath,
"' FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '\"'
FROM tmp_reporte_enviadas_sucursal_mensual");
prepare s1 from #q1;
execute s1;
deallocate prepare s1;
It creates the file in the MySql default folder
this below query is working fine for me to export the data from table to csv file but i want handle like if query returns no record then in 'filename.csv' file should contain 'no data found' message for users
-- file name as timestamp
SET #fileName = DATE_FORMAT(NOW(),'%Y-%m-%d-%H:%i:%s');
SET #FOLDER = '/tmp/';
SET #EXT = '.csv';
SET #CMD = CONCAT("SELECT id,name,salary,salaryDate FROM emp1 where name ='some_name' INTO OUTFILE '"
,#FOLDER,#fileName,#EXT,
"' FIELDS ENCLOSED BY '\"' TERMINATED BY ',' ESCAPED BY '\"'",
" LINES TERMINATED BY '\n';");
PREPARE statement FROM #CMD;
EXECUTE statement;
where do i need to change ? Any one can help me ?
You should create stored procedure. Check row numbers using COUNT function, then output one of results you need, for example -
CREATE PROCEDURE procedure1()
BEGIN
IF (SELECT COUNT(*) FROM emp1 where name ='some_name') = 0 THEN
SELECT 'no data found' INTO OUTFILE 'file_name.csv';
ELSE
your code here - SELECT INTO OUTFILE
END IF;
END
I have a table that has training records. Each record has a field with an Agency value. I have another table of just agency values. I want to export the records into CSV files for each agency. There are over 2 million records so I don't want to export the whole table and do it manually.
I've created a stored procedure that pulls a value from the agency_codes table using a cursor and uses that value in a select statement for the WHERE clause and part of the INTO OUTFILE name.
The procedure works but only for the first two values in the agency_codes table. On the third (ACB) value it gives the error Unknown column 'ACB' in 'where clause' I'm confused why it worked with the first two values then stops with the third.
Here is the procedure:
DELIMITER $$
DROP PROCEDURE IF EXISTS export_csv $$
CREATE PROCEDURE export_csv()
BEGIN
DECLARE agency_name VARCHAR(255);
DECLARE exit_loop BOOLEAN;
DECLARE agency_cursor CURSOR FOR
SELECT agency FROM agency_codes;
DECLARE CONTINUE HANDLER FOR NOT FOUND SET exit_loop = TRUE;
OPEN agency_cursor;
agency_loop: LOOP
FETCH agency_cursor INTO agency_name;
SET #sql_text = Concat("(select 'class_code','course_code','course_name','course_type','username','grade_type','score','letter_grade','is_passed','completion_date','completion_status','registration_date','registration_entry_status','registration_type','comment','first_name','last_name','class_name','agency') Union (select class_code,course_code,course_name,course_type,username,grade_type,score,letter_grade,is_passed,completion_date,completion_status,registration_date,registration_entry_status,registration_type,comment,first_name,last_name,class_name,agency from hrdis_oru where hrdis_oru.agency =", agency_name," into outfile 'C:/HDD/",agency_name,".csv' fields enclosed by '\"' terminated by ',' escaped by '\"' lines terminated by '\r\n')");
prepare s1 from #sql_text;
execute s1;
deallocate prepare s1;
IF exit_loop THEN
CLOSE agency_cursor;
LEAVE agency_loop;
END IF;
END LOOP agency_loop;
END $$
DELIMITER ;
The first few values in my Agency table are:
17
303
ACB
Actuary
agr
AIM
Any help would be great. Thanks.
If you think about what values #sql_text will hold on each iteration, the problem should be immediately obvious. Adding some whitespace for clarity:
(
select 'class_code','course_code','course_name','course_type','username',
'grade_type','score','letter_grade','is_passed','completion_date',
'completion_status','registration_date','registration_entry_status',
'registration_type','comment','first_name','last_name','class_name',
'agency'
) Union (
select class_code,course_code,course_name,course_type,username,
grade_type,score,letter_grade,is_passed,completion_date,
completion_status,registration_date,registration_entry_status,
registration_type,comment,first_name,last_name,class_name,
agency
from hrdis_oru
where hrdis_oru.agency =ACB
into outfile 'C:/HDD/ACB.csv'
fields enclosed by '\"'
terminated by ','
escaped by '\"'
lines terminated by '\r\n'
)
Note in particular, where htdis_oru.agency =ACB.
Since ACB has not been quoted, MySQL parses it as a schema object identifier and complains when it cannot find any such-named object (this isn't the case with the purely numeric agency names, since they are parsed as integers that subsequently get cast to strings during expression evaluation).
For MySQL to correctly parse non-numeric values as string literals, they must be quoted:
... where hrdis_oru.agency ='", agency_name, "' ...
^ ^
Naturally, this presents a problem should agency_name contain the ' string quotation character—any such occurrences must of course be escaped. MySQL handily provides a QUOTE() function for precisely this purpose:
... where hrdis_oru.agency =", QUOTE(agency_name), " ...
However, to guard against possible SQL injection attacks, you really should instead parameterise your prepared statement:
FETCH agency_cursor INTO #agency_name;
(You no longer need to DECLARE agency_name); then:
... where hrdis_oru.agency = ? into outfile CONCAT('C:/HDD/', ?, '.csv') ...
Followed by:
PREPARE s1 FROM #sql_text;
EXECUTE s1 USING #agency_name, #agency_name;
DEALLOCATE PREPARE s1;
Note that you can now also PREPARE the statement before entering the loop and simply EXECUTE it (with appropriate values) inside the loop—this should yield a slight performance improvement. Remember to DEALLOCATE after exiting the loop.
One final note: you should inspect exit_loop immediately after the FETCH command—otherwise you will end up attempting to execute the SELECT ... INTO OUTFILE statement a final time when there are no more agencies.
It may also be worth noting that, in this case, you have no particular need to use prepared statements at all. You could simply do the following:
CREATE PROCEDURE export_csv() BEGIN
DECLARE agency_name VARCHAR(255);
DECLARE agency_cursor CURSOR FOR SELECT agency FROM agency_codes;
DECLARE EXIT HANDLER FOR NOT FOUND CLOSE agency_cursor;
OPEN agency_cursor;
LOOP
FETCH agency_cursor INTO agency_name;
SELECT 'class_code','course_code','course_name','course_type','username',
'grade_type','score','letter_grade','is_passed','completion_date',
'completion_status','registration_date','registration_entry_status',
'registration_type','comment','first_name','last_name','class_name'
UNION ALL
SELECT class_code,course_code,course_name,course_type,username,
grade_type,score,letter_grade,is_passed,completion_date,
completion_status,registration_date,registration_entry_status,
registration_type,comment,first_name,last_name,class_name
FROM hrdis_oru
WHERE agency = agency_name
INTO OUTFILE CONCAT('C:/HDD/', agency_name, '.csv')
FIELDS ENCLOSED BY '"'
TERMINATED BY ','
ESCAPED BY '"'
LINES TERMINATED BY '\r\n';
END LOOP;
END
I am new in MYSQL programming. I want to select into outfile a very big table into several csv files using MYSQL loop script. My script is as following:
BEGIN
SET #t_lines=0;
SET #t_count=0;
SET #t_filepath='/home/ab/path/table_name_02212013_';
WHILE t_lines<=5000000
SET #t_filename=CONCAT(#t_filepath,CAST(#t_count as CHAR));
select * into outfile #t_filename fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' from table_name limit #t_lines,#t_lines+300000;
SET #t_lines= #t_lines+300000;
SET #t_count= #t_count+1;
END WHILE;
COMMIT;
END
I got a syntax error in line:
SET #t_filename=CONCAT(#t_filepath,CAST(#t_count as CHAR));
The syntax error caused by WHILE ... DO. Thanks for quick reply.
Here I still have syntax error at "#t_lines,#t_lines+300000" after "limit".
I figure it out, seems "limit" not allowed "#t_lines+300000" to give the line range. It can be fixed by giving a new variable:
BEGIN
DECLARE t_lines INT DEFAULT 0;
DECLARE t_count INT DEFAULT 0;
DECLARE t_endlines INT DEFAULT 300000;
DECLARE t_linerange INT DEFAULT 300000;
SET #t_filepath='/home/ab/path/table_name_02212013_';
WHILE t_lines<=5000000 DO
SET #t_filename=CONCAT(#t_filepath,CAST(#t_count as CHAR));
select * into outfile '#t_filename' fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' from table_name limit t_lines,t_endlines;
SET t_lines= t_lines+t_linerange;
SET t_endlines= t_endlines+t_linerange
SET t_count= t_count+1;
END WHILE;
COMMIT;
END
Thanks very much
The syntax for a WHILE loop in MySQL requires a DO;
WHILE t_lines<=5000000 DO
...
END WHILE;
The filename argument for INTO OUTFILE must be a literal string. You can't use variables or expressions as the filename.
The arguments to LIMIT must be integer constants. You can use parameter placeholders (?), or stored procedure parameters or local procedure variables (the ones you DECLARE). But you can't use session variables (the ones with # prefix), nor can you use expressions.
You're going to have to create the query as an SQL string, interpolating the values into the string.
Then use that SQL string with PREPARE and EXECUTE.
Try :
convert(columnName, char)
-->
BEGIN
SET #t_lines=0;
SET #t_count=0;
SET #t_filepath='/home/ab/path/table_name_02212013_';
WHILE t_lines<=5000000
SET #t_filename=CONCAT(#t_filepath,convert(#t_count, CHAR));
select * into outfile #t_filename fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' from table_name limit #t_lines,#t_lines+300000;
SET #t_lines= #t_lines+300000;
SET #t_count= #t_count+1;
END WHILE;
COMMIT;
END