Order by slow on joined large table - mysql

Question

I have 3 large table around 1 table have 1,000,000 rows other 2 have 500,000 rows.
When I use ORDER BY in query it takes 15 seconds (without ORDER BY 0.001s).
I already try add index on invoice.created_at and invoice.invoice_id
But still take a lot of time.
SELECT
`i`.*,
GROUP_CONCAT(DISTINCT ii.item_id SEPARATOR ',') AS item_id,
GROUP_CONCAT(DISTINCT it.transaction_id SEPARATOR ',') AS transactions,
SUM(DISTINCT IF(ii.status = 1, ii.subtotal, 0)) AS subtotal,
SUM(DISTINCT it.subtotal) AS total_paid_amount,
SUM(DISTINCT it.additional_fee) AS total_additional_fee_amount,
SUM(ii.quantity) AS total_quantity,
(total_amount - SUM(DISTINCT COALESCE(it.amount, 0))) AS total_balance
FROM
`invoices` `i`
LEFT JOIN
`invoices_items` `ii` ON `ii`.`invoice_id` = `i`.`invoice_id`
LEFT JOIN
`invoices_transactions` `it` ON `it`.`invoice_id` = `i`.`invoice_id`
AND `it`.`process_status` = 1
AND `it`.`status` = 1
WHERE
`i`.`status` = '1'
GROUP BY `i`.`invoice_id`
ORDER BY `i`.`created_at` DESC
LIMIT 50
Query with EXPLAIN:
+----+-------------+-------+------------+-------+--------------------------------------------------+------------+---------+------------------------------+--------+----------+----------------------------------------------+
| id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+-------------+-------+------------+-------+--------------------------------------------------+------------+---------+------------------------------+--------+----------+----------------------------------------------+
| 1 | SIMPLE | i | NULL | index | PRIMARY,customer_id,code,invoice_id_2,created_at | PRIMARY | 4 | NULL | 473309 | 10.00 | Using where; Using temporary; Using filesort |
| 1 | SIMPLE | ii | NULL | ref | invoice_id | invoice_id | 5 | test.i.invoice_id | 2 | 100.00 | NULL |
| 1 | SIMPLE | it | NULL | ref | invoice_id,status | invoice_id | 5 | test.i.invoice_id | 1 | 100.00 | Using where |
+----+-------------+-------+------------+-------+--------------------------------------------------+------------+---------+------------------------------+--------+----------+----------------------------------------------+
I try to remove SUM ,GROUP_CONCAT and WHERE CASE,
But still need 10 sec
SELECT
`i`.*
FROM
`invoices` `i`
LEFT JOIN
`invoices_items` `ii` ON `ii`.`invoice_id` = `i`.`invoice_id`
LEFT JOIN
`invoices_transactions` `it` ON `it`.`invoice_id` = `i`.`invoice_id`
GROUP BY `i`.`invoice_id`
ORDER BY `i`.`created_at` DESC
LIMIT 50
Create Table:
CREATE TABLE `invoices` (
`invoice_id` int NOT NULL AUTO_INCREMENT,
`warehouse_id` tinyint DEFAULT NULL,
`invoice_type` varchar(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i NOT NULL DEFAULT 'C',
`invoice_date` date DEFAULT NULL,
`customer_id` int DEFAULT NULL,
`contact_name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`contact_no` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`contact_email` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`address` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`code` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`total_amount` deimal(10,2) DEFAULT '0.00',
`total_cost` deimal(20,2) DEFAULT NULL,
`delivery_type` tinyint DEFAULT '0',
`remark` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i,
`payment_status` tinyint(1) NOT NULL DEFAULT '3',
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` datetime DEFAULT NULL,
`updated_by` int NOT NULL,
`confirmed_at` datetime DEFAULT NULL,
`status` tinyint(1) NOT NULL DEFAULT '2'
PRIMARY KEY (`invoice_id`),
KEY `customer_id` (`customer_id`),
KEY `code` (`code`),
KEY `invoice_id_2` (`invoice_id`,`created_at`),
KEY `created_at` (`created_at`)
) ENGINE=InnoDB AUTO_INCREMENT=513697 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_i;
CREATE TABLE `invoices_items` (
`item_id` int NOT NULL AUTO_INCREMENT,
`invoice_id` int NOT NULL,
`warehouse_id` int DEFAULT NULL,
`product_id` int DEFAULT NULL,
`variant_id` int DEFAULT NULL,
`name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`quantity` int DEFAULT '1',
`unit_price` deimal(10,2) DEFAULT NULL,
`cost` deimal(10,2) DEFAULT NULL,
`subtotal` deimal(10,2) DEFAULT NULL,
`serial_no` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`StockoutDate` datetime DEFAULT NULL,
`ReturnDate` datetime DEFAULT NULL,
`Return_StockLocationID` tinyint DEFAULT NULL,
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` datetime DEFAULT NULL,
`status` tinyint(1) NOT NULL DEFAULT '1',
`priority` int NOT NULL DEFAULT '0',
PRIMARY KEY (`item_id`),
KEY `invoice_id` (`invoice_id`),
KEY `variant_id` (`variant_id`),
KEY `quantity` (`quantity`),
KEY `unit_price` (`unit_price`),
KEY `subtotal` (`subtotal`),
KEY `created_at` (`created_at`),
KEY `updated_at` (`updated_at`),
KEY `status` (`status`),
KEY `priority` (`priority`),
KEY `variant_id_2` (`variant_id`,`quantity`),
KEY `variant_id_3` (`variant_id`,`name`,`quantity`),
KEY `product_id` (`product_id`),
KEY `StockoutDate` (`StockoutDate`)
) ENGINE=InnoDB AUTO_INCREMENT=1200951 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_i;
CREATE TABLE `invoices_transactions` (
`transaction_id` int NOT NULL AUTO_INCREMENT,
`invoice_id` int DEFAULT NULL,
`warehouse_id` int DEFAULT NULL,
`invoice_date` date DEFAULT NULL,
`payment_id` int DEFAULT NULL,
`balance` deimal(10,2) DEFAULT NULL,
`amount` deimal(10,2) DEFAULT NULL,
`additional_rate` deimal(10,2) DEFAULT NULL,
`additional_fee` deimal(10,2) DEFAULT NULL,
`subtotal` deimal(10,2) DEFAULT NULL,
`process_status` int DEFAULT '1',
`remark` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`payment_status` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` datetime DEFAULT NULL,
`status` tinyint NOT NULL DEFAULT '1',
`CreatedByID` int DEFAULT NULL,
`ModifiedByID` int DEFAULT NULL,
`priority` int DEFAULT '0',
PRIMARY KEY (`transaction_id`),
KEY `invoice_id` (`invoice_id`),
KEY `payment_status` (`payment_status`),
KEY `created_at` (`created_at`),
KEY `updated_at` (`updated_at`),
KEY `status` (`status`),
KEY `amount` (`amount`),
KEY `additional_fee` (`additional_fee`),
KEY `subtotal` (`subtotal`),
KEY `transaction_id` (`transaction_id`,`amount`,`additional_fee`,`subtotal`),
KEY `invoice_id_2` (`invoice_id`,`process_status`,`status`),
KEY `process_status` (`process_status`),
KEY `transaction_id_2` (`transaction_id`,`invoice_id`,`amount`,`additional_fee`,`subtotal`,`process_status`,`status`)
) ENGINE=InnoDB AUTO_INCREMENT=543606 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_i;

Answer 1

Let's find the 50 invoices first, then reach for the rest of the stuff:
SELECT ... (all the current stuff)
FROM ( SELECT invoice_id
FROM invoices
WHERE status = 1
ORDER BY created_at DESC
LIMIT 50
) AS ids
JOIN invoices AS i USING(invoice_id)
LEFT JOIN ... ON ...
LEFT JOIN ... ON ...
GROUP BY invoice_id
ORDER BY created_id DESC
These composite indexes may help:
i: (status, created_at, invoice_id)
it: (status, process_status, invoice_id)
ii: (invoice_id, quantity, subtotal, status, item_id)
Redundant index:
KEY `variant_id` (`variant_id`) -- since this is the prefix of others
Because of PRIMARY KEY(transaction_id), the keys starting with transaction_id are redundant.
Summary:
I provided a "covering index" with the columns in the right order to allow finding the 50 ids directly from that index.
The rest of the work is limited to the 50 rows (or sets of rows, since I suspect that the rest are many:1 with respect to invoices.)
Your EXPLAIN shows lots of "Rows" and a "filesort" -- indicating that it gathered everything, then did the group by, then sorted, and finally, peeled of 50.
I suspect the "filesort" was really two sorts. See EXPLAIN FORMAT=JSON ... to find out.