Memory Efficient reading + writing of .csv and .xlsx data - mysql

I am looking to implement memory efficient reading and writing of .xlsx/.csv files.
I am currently using phpExcel and am having trouble with large files. (Known issue)
I found the following library for reading: https://github.com/nuovo/spreadsheet-reader
This seems that it will work.
For writing files currently I create an array and then loop the array to output the file. Is this why it uses a lot of memory? What is the best way to not use a lot of memory when writing csv/xlsx when getting data from mysql database using code igniter?
This is what I do now:
/* added for excel expert */
function excel_export() {
$data = $this->Customer->get_all($this->Customer->count_all())->result_object();
$this->load->helper('report');
$rows = array();
$header_row = $this->_excel_get_header_row();
$header_row[] = lang('customers_customer_id');
$rows[] = $header_row;
foreach ($data as $r) {
$row = array(
$r->first_name,
$r->last_name,
$r->email,
$r->phone_number,
$r->address_1,
$r->address_2,
$r->city,
$r->state,
$r->zip,
$r->country,
$r->comments,
$r->account_number,
$r->taxable ? 'y' : '',
$r->company_name,
$r->person_id
);
$rows[] = $row;
}
$content = array_to_spreadsheet($rows);
force_download('customers_export.'.($this->config->item('spreadsheet_format') == 'XLSX' ? 'xlsx' : 'csv'), $content);
exit;
}
function array_to_spreadsheet($arr)
{
$CI =& get_instance();
$objPHPExcel = new PHPExcel();
//Default all cells to text
$objPHPExcel->getDefaultStyle()->getNumberFormat()->setFormatCode(PHPExcel_Style_NumberFormat::FORMAT_TEXT);
for($k = 0;$k < count($arr);$k++)
{
for($j = 0;$j < count($arr[$k]); $j++)
{
$objPHPExcel->getActiveSheet()->setCellValueExplicitByColumnAndRow($j, $k+1, $arr[$k][$j]);
}
}
if ($CI->config->item('spreadsheet_format') == 'XLSX')
{
$objWriter = new PHPExcel_Writer_Excel2007($objPHPExcel);
}
else
{
$objWriter = new PHPExcel_Writer_CSV($objPHPExcel);
}
ob_start();
$objWriter->save('php://output');
$excelOutput = ob_get_clean();
return $excelOutput;
}

Related

mkdir(): File existed (Yii2)

I am getting this error when trying to create new directory. What I don't understand is that when I use $folderGallery = mkdir($dir.$modelGallery->gallery_id, 0777, true);, it does not work, but when I change to $modelGallery->gallery_name, it is successfully created.
$session = Yii::$app->session;
$modelGallery= new Gallery();
//die(count($modelEulogy));
if ($modelGallery->load(Yii::$app->request->post()))
{
if($modelGallery->validate())
{
$dir = 'img/'.$session['UserID'].'/album/';
//die($modelGallery->gallery_id);
if(!file_exists($dir))
$folderGallery = mkdir($dir.$modelGallery->gallery_id, 0777, true);
else
$folderGallery = mkdir($dir.$modelGallery->gallery_id, 0777, true);
//$model->gallery_id = $id;
$modelGallery->user_id = $session['UserID'];
$modelGallery->date = date("Y-m-d H:i:s");
$modelGallery->save();
}
return $this->redirect(['index', 'gallery_id'=>$id]);
}
You must check the existing directory
$path = $dir.$modelGallery->gallery_id;
if (!is_dir($path)) {
mkdir($dir.$modelGallery->gallery_id, 0777, true);
}
You may use FileHelper::createDirectory() - it will do any necessary checks for you. You should also use absolute paths to avoid ambiguous path resolution:
$dir = '#webroot/img/'.$session['UserID'].'/album/';
FileHelper::createDirectory(Yii::getAlias($dir . $modelGallery->gallery_id, 0777, true));

Creating Json file from mysql

i can't get more than one return in this json. when the original query returns 90k results.
i can't figure out what's hapening.
also the return i get isn't organized as it should. it return the following
{"material":["R8190300000","0"],"grid":["R8190300000","0"]}
sorry to ask this i have been looking for an answer but couln't get it in the internet.
<?php
$link = mysqli_connect("localhost","blablabla","blablabla","blablabla");
if (mysqli_connect_error()) {
die("Could not connect to database");
}
$query =" SELECT material,grid FROM ZATPN";
if( $result = mysqli_query( $link, $query)){
while ($row = mysqli_fetch_row($result)) {
$resultado['material']=$row;
$resultado['grid']=$row;
}
} else {
echo"doesnt work";
}
file_put_contents("data.json", json_encode($resultado));
?>
The problem is that you are overriding the value for the array keys:
$resultado['material']=$row;
$resultado['grid']=$row;
At the end you will have only the last 2 rows; I suggest you to use something like:
$resultado['material'][] = $row;
$resultado['grid'][] = $row;
This will save you pair rows in $resultado['grid'] and unpaired rows in $resultado['material'];
After the information in comments you can use this code:
$allResults = array();
while ($object = mysqli_fetch_object($result)) {
$resultado['id'] = $object->id;
$resultado['name'] = $object->name;
$resultado['item'] = $object->item;
$resultado['price'] = $object->price;
$allResults[] = $resultado;
}
file_put_contents("data.json", json_encode($allResults));

Excel Not Saved as Numeric Format (MySQL to Excel)

I have a script that allows me to convert a database from MySQL to Excel .xls format, It was a success,
here is the code
<?php
// DB TABLE Exporter
//
// How to use:
//
// Place this file in a safe place, edit the info just below here
// browse to the file, enjoy!
// CHANGE THIS STUFF FOR WHAT YOU NEED TO DO
$cdate = date("Y-m-d");
$dbhost = "localhost";
$dbuser = "-";
$dbpass = "-";
$dbname = "-";
$dbtable = "-";
$filename = "C:\xxx";
// END CHANGING STUFF
// first thing that we are going to do is make some functions for writing out
// and excel file. These functions do some hex writing and to be honest I got
// them from some where else but hey it works so I am not going to question it
// just reuse
// This one makes the beginning of the xls file
function xlsBOF() {
echo pack("ssssss", 0x809, 0x8, 0x0, 0x10, 0x0, 0x0);
return;
}
// This one makes the end of the xls file
function xlsEOF() {
echo pack("ss", 0x0A, 0x00);
return;
}
// this will write text in the cell you specify
function xlsWriteLabel($Row, $Col, $Value ) {
$L = strlen($Value);
echo pack("ssssss", 0x204, 8 + $L, $Row, $Col, 0x0, $L);
echo $Value;
return;
}
// make the connection an DB query
$dbc = mysql_connect( $dbhost , $dbuser , $dbpass ) or die( mysql_error() );
mysql_select_db( $dbname );
$q = "SELECT * FROM ".$dbtable." ";
$qr = mysql_query( $q ) or die( mysql_error() );
//start the object
ob_start();
// start the file
xlsBOF();
// these will be used for keeping things in order.
$col = 0;
$row = 0;
// This tells us that we are on the first row
$first = true;
while( $qrow = mysql_fetch_assoc( $qr ) )
{
// Ok we are on the first row
// lets make some headers of sorts
if( $first )
{
// di comment karena ini ngasih label tabelnya, sepertinya nggak butuh
//foreach( $qrow as $k => $v )
//{
// // take the key and make label
// // make it uppper case and replace _ with ' '
// xlsWriteLabel( $row, $col, strtoupper( ereg_replace( "_" , " " , $k ) ) );
// $col++;
//}
// prepare for the first real data row
$col = 0;
$row = 0;//$row++; // nyoba
$first = false;
}
// go through the data
foreach( $qrow as $k => $v )
{
// write it out
xlsWriteLabel( $row, $col, $v );
$col++;
}
// reset col and goto next row
$col = 0;
$row++;
}
xlsEOF();
//write the contents of the object to a file
file_put_contents($filename, ob_get_clean());
?>
this code produced a .xls file. I used Matlab to read my .xls file through readxls function, but it didn't recognize the value inside the .xls file as a numeric data, so my script on matlab couldn't make a matrix from reading my xls file. I had to convert it manually inside excel, there were some pop-ups near the value that offers me to convert it into numbers
If your goal is to take data from MySQL and put it in Matlab it might be easier to directly connect to MySQL from Matlab rather than sending the data through the xls format.
http://www.mathworks.com/help/database/ug/database.fetch.html

codeigniter - convert html to pdf

I have a little problem. I have html page and I want to convert to pdf. My index page has a list that will get to the database and click on "Download PDF", I put this list in a PDF file.
My controller:
<?php
class pdf_c extends CI_Controller{
function __construct() {
parent::__construct();
$this->load->helper(array('url', 'mediatutorialpdf'));
}
function index($download_pdf = ''){
$ret = '';
$ID = 1;
$pdf_filename = 'user_info_'.$ID.'.pdf';
$link_download = ($download_pdf == TRUE)?'':anchor(base_url().'index.php/true', 'Download PDF');
$query = $this->db->query("SELECT * FROM `ci_pdf_user` WHERE `id` = '{$ID}' LIMIT 1");
if($query->num_rows() > 0)
{
$user_info = $query->row_array();
}
$data_header = array(
'title' => 'Convert codeigniter to pdf'
);
$data_userinfo = array(
'user_info' => $user_info,
'link_download' => $link_download
);
$header = $this->load->view('header',$data_header, true);
$user_info = $this->load->view('user_table', $data_userinfo, true);
$footer = $this->load->view('footer','', true);
$output = $header.$user_info.$footer;
if($download_pdf == TRUE)
{
generate_pdf($output, $pdf_filename);
}
else
{
echo $output;
}
}
}
?>
The problem is when I click the button "Download PDF" should redirect me to the function index () and get the $ download_pdf = true. And so called generate_pdf function () that will generate the PDF.
I think the problem is in the variable $ link_download, but can not solve the problem.
Thanks
I think that you could try with:
function index(pdf = 0)...
Then check that optional parameter with:
$pdf = $this->uri->segment(2, 0); //not sure, should be 2? try it...`
And then, if $pdf=='1' (send nummber rather than string 'true') ...etc,etc...

code igniter active records - help streamlining process

I am currently using the below code to get a list of uuid's then split them into groups of 1000, then insert those groups into the database.
This works fine except this has to work on at times, over a million uuid's
The issue is this uses a massive amount of memory, so I need help to streamline this process to use less memory...
public function send_daily_email($dealId) {
set_time_limit(0);
$deal = $this->ci->deal->get($dealId);
if ($deal == false)
throw new exception('Unknown Deal Specified');
$users = $this->db->select('uuid')->from('userRegionLink')->where('regionId', $deal->region)->get();
if ($users->num_rows() == 0)
throw new exception('No users in region');
$message = $this->ci->load->view('emails/daily', array('name' => $deal->title, 'content' => $deal->snippet), true);
$uuids = array();
foreach ($users->result() as $u)
$uuids[] = $u->uuid;
$uuids = array_chunk($uuids, 1000);
$sendId = 0;
foreach ($uuids as $batch) {
$count = count($batch);
$this->db->set('dealId', $dealId)->set('content', $message)->set('regionId', $deal->region)->set('recipients', $count)->set('created', 'NOW()', false)->set('status', 'Creating');
if ($sendId === 0) {
$this->db->insert('dealEmailParent');
$sendId = $this->db->insert_id();
$this->db->set('sendId', $sendId)->where('id', $sendId)->update('dealEmailParent');
}
else
$this->db->set('sendId', $sendId)->insert('dealEmailParent');
$insert = array();
foreach ($batch as $uuid)
$insert[] = array('parentId' => $sendId, 'uuid' => $uuid);
$this->db->insert_batch('dealEmailChild', $insert);
}
}
I hate to say this, but from what I know about CodeIgniter, the only way it knows to fetch results is to fetch the entire resultset at once, even if you only need one row, or even if you want to fetch a row at a time and do some processing. It doesn't operate with cursors as the native mysql(i)/PDO functionality does.
For this large a dataset, I'd suggest sticking to the native PHP database functions and foregoing CodeIgniter's active record database classes.
This reworking can insert 1,000,000 "users" in under a minute without any memory limits :)
public function create_daily_email($dealId)
{
$time_start = microtime(true);
set_time_limit(0);
$deal = $this->ci->deal->get($dealId);
if ($deal == false)
throw new exception('Unknown Deal Specified');
$message = $this->ci->load->view('emails/daily', array('name' => $deal->title, 'content' => $deal->snippet), true);
$start = 0;
$end = 50000;
$q = $this->db->select('uuid')->from('userRegionLink')->where('regionId', $deal->region)->limit($end, $start)->get();
$sendId = 0;
while ($q->num_rows() != 0) {
//do stuff
$uuids = array();
foreach ($q->result() as $u)
$uuids[] = $u->uuid;
$uuids = array_chunk($uuids, 1000);
foreach ($uuids as $batch) {
$count = count($batch);
$this->db->set('dealId', $dealId)->set('content', $message)->set('regionId', $deal->region)->set('recipients', $count)->set('created', 'NOW()', false)->set('status', 'Creating');
if ($sendId === 0) {
$this->db->insert('dealEmailParent');
$sendId = $this->db->insert_id();
$this->db->set('sendId', $sendId)->where('id', $sendId)->update('dealEmailParent');
$parentId = $sendId;
}
else {
$this->db->set('sendId', $sendId)->insert('dealEmailParent');
$parentId = $this->db->insert_id();
}
$insert = array();
foreach ($batch as $uuid) {
$insert[] = array(
'parentId' => $parentId,
'uuid' => $uuid
);
}
$this->db->insert_batch('dealEmailChild', $insert);
}
//stop stuff
unset($q);
unset($uuids);
unset($insert);
$start = $start + $end;
$q = $this->db->select('uuid')->from('userRegionLink')->where('regionId', $deal->region)->limit($end, $start)->get();
}
$this->db->set('status', 'Pending')->where('sendId', $sendId)->update('dealEmailParent');
$time_end = microtime(true);
$time = $time_end - $time_start;
die("Did nothing in $time seconds");
}