code igniter active records - help streamlining process - mysql

Question

I am currently using the below code to get a list of uuid's then split them into groups of 1000, then insert those groups into the database.
This works fine except this has to work on at times, over a million uuid's
The issue is this uses a massive amount of memory, so I need help to streamline this process to use less memory...
public function send_daily_email($dealId) {
set_time_limit(0);
$deal = $this->ci->deal->get($dealId);
if ($deal == false)
throw new exception('Unknown Deal Specified');
$users = $this->db->select('uuid')->from('userRegionLink')->where('regionId', $deal->region)->get();
if ($users->num_rows() == 0)
throw new exception('No users in region');
$message = $this->ci->load->view('emails/daily', array('name' => $deal->title, 'content' => $deal->snippet), true);
$uuids = array();
foreach ($users->result() as $u)
$uuids[] = $u->uuid;
$uuids = array_chunk($uuids, 1000);
$sendId = 0;
foreach ($uuids as $batch) {
$count = count($batch);
$this->db->set('dealId', $dealId)->set('content', $message)->set('regionId', $deal->region)->set('recipients', $count)->set('created', 'NOW()', false)->set('status', 'Creating');
if ($sendId === 0) {
$this->db->insert('dealEmailParent');
$sendId = $this->db->insert_id();
$this->db->set('sendId', $sendId)->where('id', $sendId)->update('dealEmailParent');
}
else
$this->db->set('sendId', $sendId)->insert('dealEmailParent');
$insert = array();
foreach ($batch as $uuid)
$insert[] = array('parentId' => $sendId, 'uuid' => $uuid);
$this->db->insert_batch('dealEmailChild', $insert);
}
}

Answer 1

I hate to say this, but from what I know about CodeIgniter, the only way it knows to fetch results is to fetch the entire resultset at once, even if you only need one row, or even if you want to fetch a row at a time and do some processing. It doesn't operate with cursors as the native mysql(i)/PDO functionality does.
For this large a dataset, I'd suggest sticking to the native PHP database functions and foregoing CodeIgniter's active record database classes.

Answer 2

This reworking can insert 1,000,000 "users" in under a minute without any memory limits :)
public function create_daily_email($dealId)
{
$time_start = microtime(true);
set_time_limit(0);
$deal = $this->ci->deal->get($dealId);
if ($deal == false)
throw new exception('Unknown Deal Specified');
$message = $this->ci->load->view('emails/daily', array('name' => $deal->title, 'content' => $deal->snippet), true);
$start = 0;
$end = 50000;
$q = $this->db->select('uuid')->from('userRegionLink')->where('regionId', $deal->region)->limit($end, $start)->get();
$sendId = 0;
while ($q->num_rows() != 0) {
//do stuff
$uuids = array();
foreach ($q->result() as $u)
$uuids[] = $u->uuid;
$uuids = array_chunk($uuids, 1000);
foreach ($uuids as $batch) {
$count = count($batch);
$this->db->set('dealId', $dealId)->set('content', $message)->set('regionId', $deal->region)->set('recipients', $count)->set('created', 'NOW()', false)->set('status', 'Creating');
if ($sendId === 0) {
$this->db->insert('dealEmailParent');
$sendId = $this->db->insert_id();
$this->db->set('sendId', $sendId)->where('id', $sendId)->update('dealEmailParent');
$parentId = $sendId;
}
else {
$this->db->set('sendId', $sendId)->insert('dealEmailParent');
$parentId = $this->db->insert_id();
}
$insert = array();
foreach ($batch as $uuid) {
$insert[] = array(
'parentId' => $parentId,
'uuid' => $uuid
);
}
$this->db->insert_batch('dealEmailChild', $insert);
}
//stop stuff
unset($q);
unset($uuids);
unset($insert);
$start = $start + $end;
$q = $this->db->select('uuid')->from('userRegionLink')->where('regionId', $deal->region)->limit($end, $start)->get();
}
$this->db->set('status', 'Pending')->where('sendId', $sendId)->update('dealEmailParent');
$time_end = microtime(true);
$time = $time_end - $time_start;
die("Did nothing in $time seconds");
}