MYSQL, slow performance group by with multiples tables with millions rows - mysql

I have three table with more than 1 millions of rows and one of my query take more than 30 seconds with a date range of 3 months. I have tried multiples index and I think I have respect index orders. I don't know what to do go get a better performance.
Here my query, create table statement, in my explain
SELECT
MAX(table2.id) as item_id,
MAX(table2.sampled_inspected) as sample_size,
table1.group_col AS ass_group_id
FROM
table1
INNER JOIN table2 ON table1.id = table2.assignment_id
LEFT OUTER JOIN table3 ON table1.po_id = table3.id
LEFT JOIN table4 ON table4.id = table3.supplier_id
LEFT JOIN table5 ON table5.id = table3.project_id
WHERE table1.recycled=0
AND table2.inspection_status_id > 0
AND table2.inspection_status_id != 50
AND table3.importer_id = 215 AND table5.recycled = 0 AND table3.project_id IN ( '2062','2063','2064','2065' ) AND table2.inspection_completed_date >= DATE('2019-10-01') AND table2.inspection_completed_date <= DATE('2020-01-29')
GROUP BY ass_group_id
and here my 5 tables
CREATE TABLE `table1` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`recycled` int(1) NOT NULL DEFAULT '0',
`group_col` varchar(25) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`po_id` int(11) unsigned DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `index_assignment_po` (`po_id`),
KEY `recycled` (`recycled`,`group_col`),
KEY `group_col` (`group_col`)
) ENGINE=InnoDB AUTO_INCREMENT=3324705 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `table2` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`assignment_id` int(10) unsigned DEFAULT NULL,
`sampling_size` int(11) DEFAULT NULL,
`notes` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`photo_report` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`inspection_status_id` int(11) DEFAULT '0',
`inspection_result_id` int(1) NOT NULL DEFAULT '0',
`inspection_completed_date` datetime DEFAULT NULL,
`recycled` int(1) NOT NULL DEFAULT '0',
`qty_to_inspect` decimal(11,2) DEFAULT NULL,
`defect_rate` decimal(16,13) DEFAULT NULL,
`sampled_inspected` int(11) unsigned DEFAULT NULL,
`inspection_report_id` int(11) unsigned DEFAULT NULL,
`poline_id` int(11) unsigned DEFAULT NULL,
`item_status_id` int(11) NOT NULL DEFAULT '0',
`production_line` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `FK_wsxix_assignments` (`assignment_id`),
KEY `index_inspection_report_reportid` (`inspection_report_id`),
KEY `index_ai_inspection_completed_date` (`inspection_completed_date`),
KEY `index_item_poline` (`poline_id`),
KEY `index_item_poline_assignment` (`poline_id`,`assignment_id`),
KEY `wsxix_assignments_items_inspection_status_id` (`inspection_status_id`),
KEY `wsxix_assignments_items_inspection_result_id` (`inspection_result_id`),
KEY `inspection_completed_date` (`inspection_completed_date`,`inspection_result_id`,`inspection_status_id`)
) ENGINE=InnoDB AUTO_INCREMENT=3324705 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `table3` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`buyer_id` int(11) unsigned DEFAULT NULL,
`client_id` int(11) unsigned DEFAULT NULL,
`merchandiser_user_id` int(11) unsigned DEFAULT NULL,
`factory_id` int(11) unsigned DEFAULT NULL,
`dc_id` int(11) unsigned DEFAULT NULL,
`parent_id` int(10) unsigned DEFAULT NULL,
`supplier_id` int(11) unsigned NOT NULL,
`importer_id` int(11) unsigned NOT NULL,
`project_id` int(11) unsigned NOT NULL,
`tier_two_id` int(11) unsigned DEFAULT NULL,
`store_id` int(11) unsigned DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `buyer_id` (`buyer_id`),
KEY `client_id` (`client_id`),
KEY `merchandiser_user_id` (`merchandiser_user_id`),
KEY `FK_po_factory_companies_id` (`factory_id`),
KEY `FK_po_dc_companies_id` (`dc_id`),
KEY `index_po_parentid` (`parent_id`),
KEY `index_purchaseorders_cie` (`supplier_id`),
KEY `index_poimporter_cie` (`importer_id`),
KEY `index_purchaseorders_project` (`project_id`),
KEY `facory_importer_id` (`factory_id`,`importer_id`),
KEY `posu` (`project_id`,`supplier_id`),
KEY `store_id` (`store_id`),
KEY `tier_two_id` (`tier_two_id`)
) ENGINE=InnoDB AUTO_INCREMENT=34959 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `table4` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`parent_company_id` int(11) DEFAULT NULL,
`name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `parent_company_id` (`parent_company_id`)
) ENGINE=InnoDB AUTO_INCREMENT=272 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `table5` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
`recycled` int(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `recycled` (`recycled`)
) ENGINE=InnoDB AUTO_INCREMENT=2066 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

Add these indexes:
table2: (assignment_id, inspection_completed_date)
table2: (inspection_completed_date, assignment_id)
table3: (importer_id, project_id, supplier_id, id)
table3: (project_id, importer_id, supplier_id, id)
table5: (recycled, id)
table1: (recycled, po_id, id, group_col)
Remove this; it adds nothing, but slows down the query:
LEFT JOIN table4 ON table4.id = table3.supplier_id

Related

Does MySQL different engine types (InnoDB and Aria) affect on query with large amount of LEFT JOIN?

I have one heavy MySQL query with 12 LEFT JOIN and noticed that some of tables have InnoDB engine and some Aria.
I've attached EXPLAIN of this query and pointed engine types at image with corresponding letters I and A.
Does fact of different engines affects on query speed and how seriously? Does changing of engines types helps to optimize query execution?
What else I could do to optimize this query?
Query (if needed)
SELECT user.id, user.first_name, user.last_name, user.birthday, user.email, user.phone, user.address_id,
user.alt_address_id, user.type, user.level_id, user_level.level, user.consecutive_orders, user.orders_count,
user.code_id, user.period, user.preferences, user.is_pick_up, user.pickup_address_id, user.is_active,
user.first_delivery_date, user.change_delivery_date, user.eway_id, user.date_created,
user.rest_referral_discount as `rest_referral_discount`, user.rest_code_discount as `rest_code_discount`,
user.rest_code_box_discount as `rest_code_box_discount`, user.rest_code_mp_discount as `rest_code_mp_discount`,
user.rest_code_tup_discount as `rest_code_tup_discount`, user.nf_pantry_list, user.nf_next_weeks_menu,
user.nf_paused_reminder, user.nf_welcome, user.nf_expected_delivery_time,
user.nf_delivery, address_pickup.address as `pickup_address`, address.address, address.unit,
address.instructions, location.id location_id, location.postcode, location.suburb,
location_state.code state_code, delivery_area.delivery_area_window_id delivery_area_window_id,
delivery_area_by_location.fee delivery_fee_value, delivery_area_window.day delivery_day,
delivery_area_window.day_name_full delivery_day_name_full, delivery_area_window.window delivery_window,
user.delivery_area_id, user.alt_delivery_area_id, if (user.is_pick_up, 0, delivery_area.fee) delivery_fee,
if (user.type = 1 || user.type = 2, 5, 0) ondemand_fee, code.code code_name, code.amount code_amount,
code.value code_value, code.is_permanent is_code_permanent, code.repeat code_repeat, code.times code_times,
code.apply_to code_apply_to, code.valid_from code_valid_from, code.valid_until code_valid_until,
if (code_used.count > 0, 1, 0) is_code_used, if (code_used.count > 0, code_used.count, 0) code_used_count,
if (user.is_pick_up, IFNULL(delivery_area_by_location.topup_available, 0), IFNULL(delivery_area.topup_available, 0)) topup_available,
(SELECT `order`.`id` FROM `order` LEFT JOIN `gift` ON gift.order_id = order.id
WHERE (`order`.`delivery_date` BETWEEN '2021-02-07' and '2021-02-13') and (user.id = order.user_id) and (`gift`.`id` IS NULL)
and (`order`.`status` = 10) ORDER BY `order`.`date_created` DESC LIMIT 1) as `order_id`,
(SELECT `order`.`delivery_date` FROM `order`
LEFT JOIN `gift`
ON gift.order_id = order.id
WHERE (user.id = order.user_id) and (`gift`.`id` IS NULL) and (`order`.`status` = 10) ORDER BY `order`.`date_created` DESC LIMIT 1) as `last_order_delivery_date`
FROM `user`
LEFT JOIN `address`
ON user.address_id = address.id
LEFT JOIN `address_pickup`
ON user.pickup_address_id = address_pickup.id
LEFT JOIN `delivery_area`
ON user.delivery_area_id = delivery_area.id
LEFT JOIN `delivery_area_window`
ON delivery_area_window.id = delivery_area.delivery_area_window_id
LEFT JOIN `user_level`
ON user.level_id = user_level.id
LEFT JOIN `location`
ON address.location_id = location.id
LEFT JOIN `location_state`
ON location.state_id = location_state.id
LEFT JOIN `delivery_area` `delivery_area_by_location` ON delivery_area_by_location.location_id = location.id
LEFT JOIN (SELECT count(*) count, `user_id`, `code_id` FROM `order` WHERE (code_id > 0) and (`order`.`status` = 10) GROUP BY `user_id`, `code_id`) `code_used`
ON ((user.id = code_used.user_id) and (code_used.code_id = user.code_id))
LEFT JOIN `code`
ON user.code_id = code.id WHERE (`user`.`type` = 0) and (`user`.`status` = 10)
GROUP BY `user`.`id` ORDER BY `last_name` LIMIT 15
UPD: CREATE TABLE
CREATE TABLE `address` (
`id` int(11) UNSIGNED NOT NULL,
`location_id` int(11) UNSIGNED NOT NULL,
`address` text NOT NULL,
`unit` char(4) NOT NULL,
`instructions` text NOT NULL,
`regular` tinyint(1) NOT NULL DEFAULT '1',
`topup` tinyint(1) NOT NULL DEFAULT '1',
`date_from` date DEFAULT NULL,
`date_to` date DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `address_pickup` (
`id` tinyint(3) UNSIGNED NOT NULL,
`address` text NOT NULL
) ENGINE=Aria DEFAULT CHARSET=utf8;
CREATE TABLE `code` (
`id` smallint(6) UNSIGNED NOT NULL,
`code` varchar(255) NOT NULL,
`amount` decimal(7,2) NOT NULL,
`value` tinyint(1) NOT NULL DEFAULT '0' COMMENT '0 - number; 1 - percent',
`is_mswa` tinyint(4) DEFAULT '0',
`is_permanent` tinyint(4) DEFAULT '0',
`repeat` tinyint(4) DEFAULT '0',
`apply_to` tinyint(4) DEFAULT '0' COMMENT '0 - both; 1 - box; 2 - mp;',
`times` int(11) NOT NULL DEFAULT '0',
`date_created` datetime NOT NULL,
`date_updated` datetime NOT NULL,
`valid_from` date NOT NULL,
`valid_until` date DEFAULT NULL,
`author` varchar(255) NOT NULL,
`checkout_text` text NOT NULL,
`status` smallint(6) UNSIGNED NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `delivery_area` (
`id` int(10) UNSIGNED NOT NULL,
`location_id` int(11) NOT NULL,
`fee` decimal(7,2) NOT NULL DEFAULT '0.00',
`delivery_area_window_id` tinyint(4) UNSIGNED NOT NULL DEFAULT '1',
`topup_available` tinyint(1) NOT NULL DEFAULT '1'
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `delivery_area_window` (
`id` tinyint(4) UNSIGNED NOT NULL,
`day` tinyint(3) UNSIGNED NOT NULL,
`day_name` varchar(255) NOT NULL,
`day_name_full` varchar(255) NOT NULL,
`window` varchar(255) NOT NULL DEFAULT '',
`pickup` tinyint(4) NOT NULL DEFAULT '0'
) ENGINE=Aria DEFAULT CHARSET=utf8;
CREATE TABLE `gift` (
`id` int(11) UNSIGNED NOT NULL,
`user_id` int(11) UNSIGNED NOT NULL,
`recipient_id` int(11) UNSIGNED NOT NULL,
`type` tinyint(1) NOT NULL COMMENT '0 - card; 1 - box;',
`message` text NOT NULL,
`token` char(24) DEFAULT NULL,
`order_id` int(11) UNSIGNED NOT NULL,
`is_redeemed` tinyint(1) NOT NULL,
`date_exp` date DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `location` (
`id` int(10) UNSIGNED NOT NULL,
`postcode` int(4) UNSIGNED NOT NULL,
`suburb` varchar(45) NOT NULL,
`dc` varchar(45) NOT NULL,
`state_id` tinyint(3) UNSIGNED NOT NULL,
`type_id` tinyint(4) UNSIGNED NOT NULL,
`lat` double DEFAULT NULL,
`lon` double DEFAULT NULL
) ENGINE=Aria DEFAULT CHARSET=utf8;
CREATE TABLE `location_state` (
`id` tinyint(4) UNSIGNED NOT NULL,
`code` char(3) NOT NULL,
`state` varchar(255) NOT NULL
) ENGINE=Aria DEFAULT CHARSET=utf8;
CREATE TABLE `order` (
`id` int(11) UNSIGNED NOT NULL,
`user_id` int(11) UNSIGNED NOT NULL,
`shipping_address_id` int(11) UNSIGNED NOT NULL,
`topup_address_id` int(10) UNSIGNED NOT NULL,
`pickup_address_id` tinyint(3) UNSIGNED DEFAULT NULL,
`preferences` int(11) UNSIGNED NOT NULL,
`referral_discount` decimal(7,2) NOT NULL DEFAULT '0.00',
`code_id` int(11) UNSIGNED NOT NULL,
`code_discount` decimal(7,2) NOT NULL DEFAULT '0.00',
`pickup_discount` decimal(7,2) NOT NULL DEFAULT '0.00',
`admin_discount` decimal(7,2) NOT NULL DEFAULT '0.00',
`level_discount` decimal(7,2) NOT NULL DEFAULT '0.00',
`delivery_fee` decimal(7,2) NOT NULL DEFAULT '0.00',
`topup_delivery_fee` decimal(7,2) NOT NULL,
`ondemand_fee` decimal(7,2) NOT NULL DEFAULT '0.00',
`amount` decimal(7,2) NOT NULL DEFAULT '0.00',
`retry_payment` tinyint(3) UNSIGNED NOT NULL DEFAULT '0' COMMENT '1-wait answer; 2-recharge with sms; 3-recharge without sms',
`retry_payment_sms` int(11) UNSIGNED NOT NULL DEFAULT '0' COMMENT '0 - processed; >0 - sms id',
`retry_payment_email` int(10) UNSIGNED NOT NULL DEFAULT '0',
`eway_errors` text NOT NULL,
`date_created` datetime NOT NULL,
`date_updated` datetime NOT NULL,
`delivery_date` date DEFAULT NULL,
`status` smallint(6) UNSIGNED NOT NULL DEFAULT '10',
`processing_status` smallint(6) UNSIGNED NOT NULL,
`processing_status_edited` tinyint(1) UNSIGNED NOT NULL DEFAULT '0',
`has_topup` tinyint(1) UNSIGNED NOT NULL DEFAULT '0',
`is_topup_only` tinyint(1) UNSIGNED NOT NULL DEFAULT '0'
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT;
CREATE TABLE `user` (
`id` int(11) UNSIGNED NOT NULL,
`first_name` varchar(255) NOT NULL,
`last_name` varchar(255) NOT NULL,
`birthday` date DEFAULT NULL,
`auth_key` varchar(32) NOT NULL,
`password_hash` varchar(255) NOT NULL,
`password_reset_token` varchar(255) DEFAULT NULL,
`email` varchar(255) NOT NULL,
`phone` varchar(255) DEFAULT NULL,
`custom_ref_code` varchar(255) DEFAULT NULL,
`eway_id` bigint(20) UNSIGNED DEFAULT NULL COMMENT 'eWAY Customer Token ID',
`active_campaign_id` int(11) UNSIGNED DEFAULT NULL COMMENT 'ActiveCampaign subscriber_id',
`level_id` tinyint(4) UNSIGNED NOT NULL DEFAULT '1',
`address_id` int(11) UNSIGNED NOT NULL DEFAULT '0',
`delivery_area_id` int(10) UNSIGNED DEFAULT NULL,
`alt_address_id` int(10) UNSIGNED DEFAULT NULL,
`alt_delivery_area_id` int(10) UNSIGNED DEFAULT NULL,
`code_id` int(11) UNSIGNED NOT NULL DEFAULT '0',
`period` tinyint(4) UNSIGNED NOT NULL DEFAULT '1' COMMENT '1 - Once a Week; 2 - Fortnightly',
`preferences` int(11) UNSIGNED NOT NULL DEFAULT '0',
`is_pick_up` tinyint(1) UNSIGNED NOT NULL DEFAULT '0',
`pickup_address_id` tinyint(3) UNSIGNED DEFAULT NULL,
`is_active` tinyint(4) NOT NULL DEFAULT '1' COMMENT '0 - Inactive; 1 - Active; -1 - Blocked',
`status` smallint(6) NOT NULL DEFAULT '10',
`type` tinyint(4) UNSIGNED NOT NULL COMMENT '0 - Set And Forget; 1 - On Demand; 2 - Trial; 3 - Potential;',
`first_delivery_date` date DEFAULT NULL,
`last_delivery_date` date DEFAULT NULL,
`change_delivery_date` date DEFAULT NULL,
`nf_pantry_list` tinyint(1) NOT NULL DEFAULT '1',
`nf_next_weeks_menu` tinyint(1) NOT NULL DEFAULT '1',
`nf_paused_reminder` tinyint(1) NOT NULL DEFAULT '1',
`nf_expected_delivery_time` tinyint(1) NOT NULL DEFAULT '1',
`nf_delivery` tinyint(1) NOT NULL DEFAULT '1',
`nf_welcome` tinyint(1) NOT NULL DEFAULT '1',
`date_created` datetime NOT NULL,
`date_updated` datetime NOT NULL,
`consecutive_orders` smallint(5) UNSIGNED DEFAULT NULL,
`orders_count` smallint(5) UNSIGNED NOT NULL DEFAULT '0',
`rest_referral_discount` decimal(7,2) UNSIGNED NOT NULL,
`rest_code_discount` decimal(7,2) UNSIGNED NOT NULL,
`rest_code_box_discount` decimal(7,2) UNSIGNED NOT NULL,
`rest_code_mp_discount` decimal(7,2) UNSIGNED NOT NULL,
`rest_code_tup_discount` decimal(7,2) UNSIGNED NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `user_level` (
`id` int(11) NOT NULL,
`level` varchar(255) NOT NULL,
`icon` varchar(255) NOT NULL,
`description` text NOT NULL,
`from` int(11) NOT NULL DEFAULT '0',
`to` int(11) NOT NULL DEFAULT '0',
`discount` int(11) NOT NULL DEFAULT '0'
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
And indexes
--
-- Indexes table `address`
--
ALTER TABLE `address`
ADD PRIMARY KEY (`id`),
ADD KEY `location_id` (`location_id`),
ADD KEY `regular` (`regular`),
ADD KEY `topup` (`topup`),
ADD KEY `date_to` (`date_to`),
ADD KEY `date_from` (`date_from`);
--
-- Indexes table `address_pickup`
--
ALTER TABLE `address_pickup`
ADD PRIMARY KEY (`id`);
--
-- Indexes table `code`
--
ALTER TABLE `code`
ADD PRIMARY KEY (`id`),
ADD KEY `status` (`status`),
ADD KEY `valid_from` (`valid_from`),
ADD KEY `valid_until` (`valid_until`),
ADD KEY `date_created` (`date_created`),
ADD KEY `code` (`code`) USING BTREE,
ADD KEY `is_mswa` (`is_mswa`);
--
-- Indexes table `delivery_area`
--
ALTER TABLE `delivery_area`
ADD PRIMARY KEY (`id`),
ADD KEY `day` (`delivery_area_window_id`),
ADD KEY `location_id` (`location_id`) USING BTREE,
ADD KEY `friday_topup_available` (`topup_available`);
--
-- Indexes table `delivery_area_window`
--
ALTER TABLE `delivery_area_window`
ADD PRIMARY KEY (`id`),
ADD KEY `day` (`day`),
ADD KEY `pickup_only` (`pickup`);
--
-- Indexes table `gift`
--
ALTER TABLE `gift`
ADD PRIMARY KEY (`id`),
ADD UNIQUE KEY `token` (`token`),
ADD KEY `user_id` (`user_id`),
ADD KEY `recipient_id` (`recipient_id`),
ADD KEY `order_id` (`order_id`),
ADD KEY `is_redeemed` (`is_redeemed`);
--
-- Indexes table `location`
--
ALTER TABLE `location`
ADD PRIMARY KEY (`id`),
ADD KEY `idx_lon` (`lon`),
ADD KEY `idx_lat` (`lat`),
ADD KEY `type_id` (`type_id`),
ADD KEY `state_id` (`state_id`),
ADD KEY `postcode` (`postcode`),
ADD KEY `suburb_2` (`suburb`),
ADD KEY `postcode_suburb` (`postcode`,`suburb`) USING BTREE;
ALTER TABLE `location` ADD FULLTEXT KEY `suburb` (`suburb`);
--
-- Indexes table `location_state`
--
ALTER TABLE `location_state`
ADD PRIMARY KEY (`id`);
--
-- Indexes table `order`
--
ALTER TABLE `order`
ADD PRIMARY KEY (`id`),
ADD KEY `client_id` (`user_id`),
ADD KEY `status` (`status`),
ADD KEY `shipping_address_id` (`shipping_address_id`),
ADD KEY `preferences` (`preferences`),
ADD KEY `processing_status` (`processing_status`),
ADD KEY `date_created` (`date_created`),
ADD KEY `delivery_date` (`delivery_date`),
ADD KEY `pickup_address_id` (`pickup_address_id`),
ADD KEY `processing_status_edited` (`processing_status_edited`),
ADD KEY `retry_payment_sms` (`retry_payment_sms`),
ADD KEY `retry_payment` (`retry_payment`),
ADD KEY `retry_payment_email` (`retry_payment_email`),
ADD KEY `has_topup` (`has_topup`),
ADD KEY `is_topup_only` (`is_topup_only`),
ADD KEY `friday_address_id` (`topup_address_id`),
ADD KEY `code_id` (`code_id`),
ADD KEY `user_id` (`user_id`);
--
-- Indexes table `user`
--
ALTER TABLE `user`
ADD PRIMARY KEY (`id`),
ADD UNIQUE KEY `password_reset_token` (`password_reset_token`),
ADD KEY `level_id` (`level_id`),
ADD KEY `eway_id` (`eway_id`),
ADD KEY `password_hash` (`password_hash`),
ADD KEY `auth_key` (`auth_key`),
ADD KEY `email` (`email`),
ADD KEY `period` (`period`),
ADD KEY `preferences` (`preferences`),
ADD KEY `is_pick_up` (`is_pick_up`),
ADD KEY `is_active` (`is_active`),
ADD KEY `date_created` (`date_created`),
ADD KEY `phone` (`phone`) USING BTREE,
ADD KEY `active_campaign_id` (`active_campaign_id`),
ADD KEY `custom_ref_code` (`custom_ref_code`),
ADD KEY `consecutive_orders` (`consecutive_orders`),
ADD KEY `pickup_address_id` (`pickup_address_id`),
ADD KEY `rest_referral_discount` (`rest_referral_discount`),
ADD KEY `rest_code_discount` (`rest_code_discount`),
ADD KEY `rest_code_box_discount` (`rest_code_box_discount`),
ADD KEY `rest_code_mp_discount` (`rest_code_mp_discount`),
ADD KEY `delivery_area_id` (`delivery_area_id`),
ADD KEY `rest_code_tup_discount` (`rest_code_tup_discount`),
ADD KEY `orders_count` (`orders_count`),
ADD KEY `alt_delivery_address_id` (`alt_address_id`),
ADD KEY `alt_delivery_area_id` (`alt_delivery_area_id`),
ADD KEY `address_id` (`address_id`),
ADD KEY `code_id` (`code_id`),
ADD KEY `status` (`status`),
ADD KEY `type` (`type`),
ADD KEY `last_name` (`last_name`);
--
-- Indexes table `user_level`
--
ALTER TABLE `user_level`
ADD PRIMARY KEY (`id`);
--
-- AUTO_INCREMENT
--
ALTER TABLE `address`
MODIFY `id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `address_pickup`
MODIFY `id` tinyint(3) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `code`
MODIFY `id` smallint(6) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `delivery_area`
MODIFY `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `delivery_area_window`
MODIFY `id` tinyint(4) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `gift`
MODIFY `id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `location`
MODIFY `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `location_state`
MODIFY `id` tinyint(4) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `order`
MODIFY `id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `user`
MODIFY `id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT;
ALTER TABLE `user_level`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
COMMIT;
For table order:
INDEX(status, code_id, order_id)
INDEX(status, order_id, code_id)
(I can't tell which would be better; the Optimizer can decide.)
Don't use LEFT JOIN when the 'right' table is not optional. It makes it tedious for a human (and the Optimizer) to figure the intent.
User needs
INDEX(type, status, code_id, code, id, last_name)
It is almost always beneficial to DROP INDEX(a) when you ADD INDEX(a,b). I bring that up because you probably have some one-column indexes.
If you need further help, please provide SHOW CREATE TABLE. I suspect that the query could be turned inside-out to great benefit. This involves finding the 15 ids first; then after that do all the JOINs.
That might have this as the first "derived" table since it generates only 15 rows, not the 9976 estimated in the Explain:
FROM ( SELECT id, `code`, code_id, last_name
FROM user
WHERE `type` = 0
AND `status` = 10
GROUP BY `id`
ORDER BY `last_name`
LIMIT 15
) AS u
JOIN ...

"1062 Duplicate entry" on SELECT

I've got a SELECT query that's returning the following error:
SQLSTATE[23000]: Integrity constraint violation: 1062 Duplicate entry 'user_id-name email#address.com--2018-11-04 01:24:38' for key ''
(I replaced the user_id, name and email for privacy)
SELECT users.user_id, users.fullname, users.email, users.phone_formatted, users.date_created,
MAX(cards.card_id) AS latest_card_id, MAX(cards.date_created) AS latest_date_card, MAX(punches.date_created) AS date_lastvisited
FROM users
INNER JOIN cards ON cards.user_id = users.user_id
INNER JOIN punches ON punches.card_id = cards.card_id
WHERE punches.business_id=1626
GROUP BY users.user_id, users.fullname, users.email, users.phone_formatted, users.date_created
ORDER BY users.fullname;
Here are the table definitions for cards, punches and users:
CREATE TABLE `cards` (
`card_id` int(11) NOT NULL AUTO_INCREMENT,
`user_id` int(11) DEFAULT NULL,
`business_id` int(11) DEFAULT NULL,
`printjob_id` int(11) DEFAULT NULL,
`temp` varchar(256) DEFAULT NULL,
`active` tinyint(1) NOT NULL DEFAULT '1',
`whitelabel_id` int(11) NOT NULL DEFAULT '0',
`date_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`card_id`),
KEY `printjob_id` (`printjob_id`),
KEY `active` (`active`),
KEY `user_id` (`user_id`),
KEY `business_id` (`business_id`)
) ENGINE=InnoDB AUTO_INCREMENT=576475 DEFAULT CHARSET=latin1;
CREATE TABLE `punches` (
`punch_id` int(11) NOT NULL AUTO_INCREMENT,
`punch_count` int(11) unsigned NOT NULL DEFAULT '1',
`employeebonus` tinyint(1) NOT NULL DEFAULT '0',
`details` text,
`card_id` int(11) DEFAULT NULL,
`behaviour` set('basicpunch','employeebonus','visitsregularly','broughtfriend','mytreat','firstcustomer','dailyrepeater','visitslocations','facebookshare','mostfrequent','longabsence','registerer','employeepromoter','luckiest','enroll','opportunist') NOT NULL DEFAULT 'basicpunch',
`location_id` int(11) DEFAULT NULL,
`business_id` int(11) DEFAULT NULL,
`employee_id` int(11) DEFAULT NULL,
`v1` float DEFAULT NULL,
`eventcounter` smallint(5) unsigned NOT NULL DEFAULT '0',
`date_local` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`date_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`punch_id`),
KEY `business_id` (`business_id`),
KEY `employee_id` (`employee_id`),
KEY `card_id` (`card_id`),
KEY `date_created` (`date_created`),
KEY `behaviour_id` (`behaviour`),
KEY `location_id` (`location_id`),
KEY `employeebonus` (`employeebonus`),
KEY `punch_count` (`punch_count`)
) ENGINE=InnoDB AUTO_INCREMENT=807402 DEFAULT CHARSET=latin1;
CREATE TABLE `users` (
`user_id` int(11) NOT NULL AUTO_INCREMENT,
`fullname` varchar(60) DEFAULT NULL,
`email` varchar(50) NOT NULL,
`undeliverable` tinyint(4) DEFAULT '0',
`phone_formatted` varchar(25) DEFAULT NULL,
`phone_unformatted` varchar(15) DEFAULT NULL,
`password_hash` char(64) DEFAULT NULL,
`password_salt` char(6) DEFAULT NULL,
`reset_hash` binary(32) DEFAULT NULL,
`date_reset` datetime DEFAULT NULL,
`stripecustomer_id` int(11) DEFAULT NULL,
`stripe_customer_id` varchar(40) DEFAULT NULL,
`stripe_customer_access_id` varchar(40) DEFAULT NULL,
`problemwithpayment` tinyint(1) NOT NULL DEFAULT '0',
`enterprisemanager_whitelabel_id` int(11) NOT NULL DEFAULT '0',
`god` tinyint(1) NOT NULL DEFAULT '0',
`date_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`),
KEY `email` (`email`),
KEY `phone_unformatted` (`phone_unformatted`),
KEY `date_reset` (`date_reset`),
KEY `undeliverable` (`undeliverable`)
) ENGINE=InnoDB AUTO_INCREMENT=16809 DEFAULT CHARSET=latin1;
I don't understand how I can get this error with a SELECT query. When I remove the MAX() functions, I no longer get the error.

Subquery returning null

Got this confusing issue where I need to get data from several tables and join them into one result.
This query was working fine:
select simcard.*,customer.name as cuname,mi,ma,co, tot from simcard, customer,
(SELECT s.sim_id AS ssim_id, min(datetime) AS mi, max(datetime) AS ma, FLOOR(sum(t.WEIGHT) / 1000) AS tot,
(SELECT count(datetime)
FROM transactions t
WHERE (t.SIM_ID = ssim_id) AND t.ROWTYPE LIKE '$D' GROUP BY t.sim_id) as co
FROM simcard s, transactions t
WHERE (s.sim_id = t.sim_id) AND t.ROWTYPE LIKE '$Z'
AND ( s.customer_id =1 ) GROUP BY s.sim_id) as T
WHERE (sim_id = ssim_id) AND (simcard.customer_id = customer.id) GROUP BY simcard.SIM_ID
But when I add a new customer and new simcard to that customer, it will return empty. I'm guessing this is because there is not transaction with that sim_id in the transaction able.
I've tried to left join but I'm just getting errors.
I've narrowed it down to the first subquery, removing the second one did not result in an non-empty result, but because both subqueries uses the transaction table I'm guessing that both will need some left join.
Structure:
DROP TABLE IF EXISTS `customer`;
CREATE TABLE IF NOT EXISTS `customer` (
`ID` int(10) NOT NULL AUTO_INCREMENT,
`NAME` varchar(40) DEFAULT NULL,
`API` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`ID`),
KEY `CUSTOMER_ID` (`ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Tabellstruktur `simcard`
--
DROP TABLE IF EXISTS `simcard`;
CREATE TABLE IF NOT EXISTS `simcard` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`SIM_ID` char(20) DEFAULT NULL,
`CUSTOMER_ID` char(10) DEFAULT NULL,
`SCALES_ID` char(10) DEFAULT NULL,
`NAME` varchar(40) DEFAULT NULL,
`ACTIVE` char(1) DEFAULT 'N',
`EMAIL` varchar(255) DEFAULT NULL,
`TARGET_WEIGHT` varchar(255) DEFAULT NULL,
`LICENSE` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`ID`),
KEY `SIM_ID` (`SIM_ID`),
KEY `CUSTOMER_ID` (`CUSTOMER_ID`),
KEY `SCALES_ID` (`SCALES_ID`),
KEY `SIM_ID_2` (`SIM_ID`,`CUSTOMER_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Tabellstruktur `transactions`
--
DROP TABLE IF EXISTS `transactions`;
CREATE TABLE IF NOT EXISTS `transactions` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`SCALES_ID` char(10) DEFAULT NULL,
`SIM_ID` char(20) DEFAULT NULL,
`ROWTYPE` char(2) DEFAULT NULL,
`DATETIME` datetime DEFAULT NULL,
`TRANSACTIONTIME` int(11) DEFAULT NULL,
`NAME` varchar(255) DEFAULT NULL,
`TRANSACTIONNUMBER` int(11) DEFAULT NULL,
`DATA0` varchar(255) DEFAULT NULL,
`DATA1` varchar(255) DEFAULT NULL,
`DATA2` varchar(255) DEFAULT NULL,
`DATA3` varchar(255) DEFAULT NULL,
`DATA4` varchar(255) DEFAULT NULL,
`DATA5` varchar(255) DEFAULT NULL,
`DATA6` varchar(255) DEFAULT NULL,
`DATA7` varchar(255) DEFAULT NULL,
`DATA8` varchar(255) DEFAULT NULL,
`MEMORY` varchar(255) DEFAULT NULL,
`MATERIAL` varchar(255) DEFAULT NULL,
`DENSITY` int(11) DEFAULT NULL,
`VEHICLETYPE` char(1) DEFAULT NULL,
`TOOLNAME` varchar(255) DEFAULT NULL,
`WEIGHT` int(11) DEFAULT NULL,
`TRANSACTIONID` int(11) DEFAULT NULL,
`group_id` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`ID`),
KEY `SIM_ID` (`SIM_ID`),
KEY `TRANSACTIONID` (`TRANSACTIONID`),
KEY `SCALES_ID` (`SCALES_ID`),
KEY `SIM_ID_2` (`SIM_ID`,`TRANSACTIONID`),
KEY `idx_transactions_ROWTYPE` (`ROWTYPE`),
KEY `idx_transactions_DATETIME` (`DATETIME`),
KEY `idx_transactions_DATA0` (`DATA0`),
KEY `idx_transactions_DATA1` (`DATA1`),
KEY `idx_transactions_DATA2` (`DATA2`),
KEY `idx_transactions_DATA3` (`DATA3`),
KEY `idx_transactions_DATA4` (`DATA4`),
KEY `idx_transactions_DATA5` (`DATA5`),
KEY `idx_transactions_DATA6` (`DATA6`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
COMMIT;
Expected result:
ID SIM_ID CUSTOMER_ID SCALES_ID NAME ACTIVE EMAIL TARGET_WEIGHT LICENSE cuname mi ma co tot
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
8 10100421287868 1 61 M-61M Y NULL NULL 0 testcustomer 2018-04-26 00:00:00 2018-08-08 00:00:00 14908 529446

MySQL insert using transaction

I have following structure on mysql database:
sqlfiddle
What I want to do is:
To select DISTINCT industry from Company table
To insert into Industry table first and get auto incremented ID
With this ID to insert again into IndustryTranslation table and set "language"="en"
To insert Company's id and newly generated Industry's id into MapCompanyIndustry table
I know that it's not possible with one statement. But definitely it's possible with transaction. Can't figure out how to achieve this result with one transaction.
Any suggestions?
Schema
CREATE TABLE `Industry` (
`id` int(4) unsigned NOT NULL AUTO_INCREMENT,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `IndustryTranslation` (
`industryID` int(4) unsigned NOT NULL,
`language` varchar(5) NOT NULL,
`name` varchar(255) NOT NULL,
`confirmed` tinyint(1) DEFAULT '0',
PRIMARY KEY (`industryID`,`language`),
KEY `language` (`language`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `Company` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL,
`imageUri` varchar(255) DEFAULT NULL,
`countryID` int(3) unsigned DEFAULT NULL,
`phone` varchar(255) DEFAULT NULL,
`email` varchar(255) DEFAULT NULL,
`verified` tinyint(1) DEFAULT NULL,
`industry` varchar(255) DEFAULT NULL,
`headquarters` varchar(255) DEFAULT NULL,
`uri` varchar(255) DEFAULT NULL,
`createdAt` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`updatedAt` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `countryID` (`countryID`)
) ENGINE=InnoDB AUTO_INCREMENT=4004 DEFAULT CHARSET=utf8;
CREATE TABLE `MapCompanyIndustry` (
`companyID` int(10) unsigned NOT NULL,
`industryID` int(4) unsigned NOT NULL,
PRIMARY KEY (`companyID`,`industryID`),
KEY `industryID` (`industryID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

MySQL - Slow Multiple subquery & GROUP BY

When running the following query using GROUP BY it takes way too much time.
SELECT specialities.id AS ID_DEPARTMENT,
specialities.name AS ID_DEPARTMENT_NAME,
agenda.idagenda AS ID_SERVICE,
agenda.name AS ID_SERVICE_NAME,
supervisor.clients_waiting AS CWaiting,
IFNULL(supervisor.clients_resent_waiting_area, 0) AS CWaiting_Resent_Area,
supervisor.clients_attending AS CAttending,
supervisor.clients_attended AS CAttended,
(SELECT SUM(TIME_TO_SEC(TIMEDIFF(NOW(), time_waiting)) / CWaiting)
FROM supervisor_time_data
WHERE supervisor_time_data.id_service = supervisor.id_service) AS TME,
(SELECT SUM(TIME_TO_SEC(TIMEDIFF(NOW(), time_attending)) / CAttending)
FROM supervisor_time_data
WHERE supervisor_time_data.id_service = supervisor.id_service) AS TMA,
(SELECT TIME_TO_SEC(MAX(TIMEDIFF(NOW(), time_waiting)))
FROM supervisor_time_data
WHERE supervisor_time_data.id_service = supervisor.id_service) AS MTE,
(SELECT TIME_TO_SEC(MAX(TIMEDIFF(NOW(), time_attending)))
FROM supervisor_time_data
WHERE supervisor_time_data.id_service = supervisor.id_service) AS MTA,
supervisor.tme_accumulated AS TME_ACCUMULATED,
supervisor.tma_accumulated AS TMA_ACCUMULATED
FROM supervisor, supervisor_time_data, agenda, specialities
WHERE supervisor.id_service = agenda.id
AND supervisor_time_data.id_service = supervisor.id_service
AND agenda.idspeciality = specialities.id
AND supervisor.booked_or_sequential = 0
AND supervisor.id_service IN (1,2,3)
GROUP BY supervisor.id_service
ORDER BY agenda.name ASC;
It takes over 3 seconds, when commenting the GROUP BY line it takes 8 ms.
Any ideas on how I can optimise this query?
Thanks
EDIT:
CREATE TABLE `supervisor` (
`id` int(9) NOT NULL AUTO_INCREMENT,
`id_department` int(6) DEFAULT NULL,
`id_service` int(9) DEFAULT NULL,
`clients_waiting` int(6) DEFAULT '0',
`clients_attending` int(6) DEFAULT '0',
`clients_attended` int(6) DEFAULT '0',
`tma_accumulated` int(9) DEFAULT '0',
`tme_accumulated` int(9) DEFAULT '0',
`clients_resent_waiting_area` int(6) DEFAULT NULL,
`booked_or_sequential` tinyint(1) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `id_service` (`id_service`),
KEY `booked_or_sequential` (`booked_or_sequential`),
KEY `clients_waiting` (`clients_waiting`)
) ENGINE=MyISAM AUTO_INCREMENT=172 DEFAULT CHARSET=latin1;
.
CREATE TABLE `supervisor_time_data` (
`id` int(9) NOT NULL AUTO_INCREMENT,
`id_ogs` int(32) DEFAULT NULL,
`booked_or_sequential` tinyint(1) DEFAULT NULL,
`time_waiting` datetime DEFAULT NULL,
`time_attending` datetime DEFAULT NULL,
`status` int(2) DEFAULT '0',
`id_service` int(6) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `id_service` (`id_service`),
KEY `time_waiting` (`time_waiting`),
KEY `time_attending` (`time_attending`),
KEY `booked_or_sequential` (`booked_or_sequential`)
) ENGINE=MyISAM AUTO_INCREMENT=2281 DEFAULT CHARSET=latin1;
.
CREATE TABLE `agenda` (
`id` int(3) NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL,
`idagenda` varchar(255) DEFAULT NULL,
`iduser` int(3) DEFAULT NULL,
`date_created` datetime DEFAULT NULL,
`agendatype` tinyint(4) DEFAULT NULL,
`idspeciality` int(6) DEFAULT NULL,
`denomination` varchar(255) DEFAULT NULL,
`ticket_count` int(3) DEFAULT NULL,
`waiting` int(3) DEFAULT NULL,
`ticket_start` int(3) DEFAULT NULL,
`ticket_end` int(3) DEFAULT NULL,
`ticket_letter` varchar(12) DEFAULT NULL,
`idcenter` int(9) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idagenda` (`idagenda`),
KEY `idspeciality` (`idspeciality`)
) ENGINE=MyISAM AUTO_INCREMENT=2228 DEFAULT CHARSET=latin1;
.
CREATE TABLE `specialities` (
`id` int(6) NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL,
`date_created` datetime DEFAULT NULL,
`idwaitingarea` int(3) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=149 DEFAULT CHARSET=latin1;
Here is your queries with joins. Post your schema if this does not work.
SELECT
specialities.id AS ID_DEPARTMENT,
specialities.name AS ID_DEPARTMENT_NAME,
agenda.idagenda AS ID_SERVICE,
agenda.name AS ID_SERVICE_NAME,
supervisor.clients_waiting AS CWaiting,
IFNULL(supervisor.clients_resent_waiting_area, 0) AS CWaiting_Resent_Area,
supervisor.clients_attending AS CAttending,
supervisor.clients_attended AS CAttended,
SUM(TIME_TO_SEC(TIMEDIFF(NOW(), supervisor_time_data.time_waiting)) / supervisor.clients_waiting) AS TME,
SUM(TIME_TO_SEC(TIMEDIFF(NOW(), supervisor_time_data.time_attending)) / supervisor.clients_attending) AS TMA,
TIME_TO_SEC(MAX(TIMEDIFF(NOW(), supervisor_time_data.time_waiting))) AS MTE,
TIME_TO_SEC(MAX(TIMEDIFF(NOW(), supervisor_time_data.time_attending))) AS MTA,
supervisor.tme_accumulated AS TME_ACCUMULATED,
supervisor.tma_accumulated AS TMA_ACCUMULATED
FROM supervisor
LEFT JOIN supervisor_time_data
ON supervisor_time_data.id_service = supervisor.id_service
LEFT JOIN agenda
ON supervisor.id_service = agenda.id
LEFT JOIN specialities
ON agenda.idspeciality = specialities.id
WHERE supervisor.booked_or_sequential = 0
AND supervisor.id_service IN(1,2,3)
GROUP BY supervisor.id_service
ORDER BY agenda.name ASC;