slow performance of query and scanning many rows - mysql

The below query is taking 6.18 min to execute for one row and the cardinality value for the exception_type = 1 is 3, I don't know how to improve the performance.
Query;
select count(1) as rage_tap
from summary_funnel_1066 s
join user_tasks_metadata utm on utm.asi = s.asi
join user_tasks ut on ut.id = utm.user_task_id
where s.seq_no = 1
and s.created_at between '2022-09-27 00:00:00' and '2022-10-27 00:00:00'
and ut.exception_type = 1
explain plan;
*************************** 1. row ***************************
id: 1
select_type: SIMPLE
table: ut
partitions: NULL
type: ALL
possible_keys: PRIMARY
key: NULL
key_len: NULL
ref: NULL
rows: 129554700
filtered: 10.00
Extra: Using where
*************************** 2. row ***************************
id: 1
select_type: SIMPLE
table: utm
partitions: NULL
type: ref
possible_keys: usertask_fk_idx,asi_idx,id_asi,asi
key: usertask_fk_idx
key_len: 8
ref: ue_stage.ut.id
rows: 1
filtered: 100.00
Extra: NULL
*************************** 3. row ***************************
id: 1
select_type: SIMPLE
table: s
partitions: NULL
type: eq_ref
possible_keys: PRIMARY,unique_asi_seq_no,seq_no_date,created_at,idx_combo
key: unique_asi_seq_no
key_len: 12
ref: ue_stage.utm.asi,const
rows: 1
filtered: 50.00
Extra: Using where; Using index
table structure;
Create Table: CREATE TABLE `summary_funnel_1066` (
`funnel_id` int DEFAULT NULL,
`app_id` int DEFAULT NULL,
`platform` int DEFAULT NULL,
`app_version_id` int NOT NULL,
`seq_no` int NOT NULL,
`property_id` bigint DEFAULT NULL,
`property_name` varchar(255) DEFAULT NULL,
`property_type` varchar(50) DEFAULT NULL,
`asi` bigint NOT NULL,
`created_at` datetime NOT NULL,
`capture_time_relative` decimal(15,4) DEFAULT NULL,
`last_event_id` bigint DEFAULT NULL,
`last_event_name` varchar(100) DEFAULT NULL,
`last_message_id` bigint DEFAULT NULL,
`last_message_name` varchar(100) DEFAULT NULL,
`last_tag_id` bigint DEFAULT NULL,
`last_tag_name` varchar(100) DEFAULT NULL,
`is_crash` tinyint DEFAULT NULL,
`is_anr` tinyint DEFAULT NULL,
`is_ragetap` tinyint DEFAULT NULL,
`last_error_type_id` bigint DEFAULT NULL,
`last_error_type` varchar(100) DEFAULT NULL,
`screen_id` bigint DEFAULT NULL,
`screen_name` varchar(100) DEFAULT NULL,
`last_screen_id` bigint DEFAULT NULL,
`last_screen_name` varchar(100) DEFAULT NULL,
`user_task_id` bigint DEFAULT NULL,
`ue_id` bigint DEFAULT NULL,
PRIMARY KEY (`asi`,`seq_no`,`created_at`,`app_version_id`),
UNIQUE KEY `unique_asi_seq_no` (`asi`,`seq_no`),
KEY `seq_no_date` (`seq_no`,`created_at`),
KEY `last_ids` (`last_screen_id`,`last_event_id`),
KEY `created_at` (`created_at`),
KEY `idx_combo` (`seq_no`,`property_id`,`property_name`,`created_at`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
Table: user_tasks_metadata
Create Table: CREATE TABLE `user_tasks_metadata` (
`id` bigint NOT NULL AUTO_INCREMENT,
`user_task_id` bigint NOT NULL,
`device_id` bigint NOT NULL,
`custom_user_id` bigint DEFAULT NULL,
`asi` bigint NOT NULL DEFAULT '0',
`session_id` varchar(300) DEFAULT NULL,
`model` bigint DEFAULT NULL,
`api_level` varchar(300) DEFAULT NULL,
`app_version_id` bigint NOT NULL DEFAULT '0',
`os_version` bigint DEFAULT NULL,
`location` bigint DEFAULT NULL,
`connection_speed` varchar(10) DEFAULT NULL,
`network_operator` varchar(100) CHARACTER SET utf8mb3 COLLATE utf8_general_ci DEFAULT NULL,
`config_response` tinyint DEFAULT '1',
`total_internal_memory` double(12,5) DEFAULT NULL,
`available_internal_memory` double(12,5) DEFAULT NULL,
`total_ram` double(12,5) DEFAULT NULL,
`available_ram` double(12,5) DEFAULT NULL,
`framework` varchar(45) DEFAULT '',
`ue_sdk_version` mediumint DEFAULT NULL,
`crash_type` bigint DEFAULT NULL,
`created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`user_profile_id` bigint DEFAULT NULL,
`associated_custom_user_id` bigint DEFAULT NULL,
`first_usr_interaction` bigint DEFAULT NULL,
`app_launch_type` varchar(45) DEFAULT '',
`app_launch_time` bigint DEFAULT '0',
PRIMARY KEY (`id`),
KEY `session_metadata_filter_idx` (`custom_user_id`,`device_id`),
KEY `usertask_fk_idx` (`user_task_id`),
KEY `idx_app_version` (`app_version_id`),
KEY `asi_idx` (`asi`),
KEY `device_id` (`device_id`),
KEY `user_profile_id` (`user_profile_id`),
KEY `id_asi` (`user_task_id`,`asi`),
KEY `asi` (`asi`)
) ENGINE=InnoDB AUTO_INCREMENT=2252872743 DEFAULT CHARSET=latin1
Table: user_tasks
Create Table: CREATE TABLE `user_tasks` (
`id` bigint NOT NULL AUTO_INCREMENT,
`app_id` bigint NOT NULL,
`status` tinyint NOT NULL DEFAULT '0',
`app_version` varchar(100) DEFAULT NULL,
`platform` tinyint NOT NULL DEFAULT '1',
`exception_type` tinyint NOT NULL DEFAULT '0',
`error_count` smallint NOT NULL DEFAULT '0',
`crash_type` varchar(300) DEFAULT NULL,
`crash_log` varchar(300) DEFAULT NULL,
`avg_signal_level` int DEFAULT '0',
`is_read` tinyint(1) NOT NULL DEFAULT '0',
`is_important` tinyint(1) NOT NULL DEFAULT '0',
`is_video_available` tinyint(1) NOT NULL DEFAULT '0',
`is_video_played` tinyint(1) NOT NULL DEFAULT '0',
`is_ex` tinyint(1) NOT NULL DEFAULT '0',
`is_ragetap` tinyint(1) NOT NULL DEFAULT '0',
`session_start_time` datetime DEFAULT NULL,
`network_type` tinyint NOT NULL DEFAULT '0',
`s3_video_url` varchar(255) DEFAULT NULL,
`image_format` tinyint DEFAULT '0',
`ue_release_version` smallint NOT NULL DEFAULT '0',
`created_at` datetime NOT NULL,
`updated_at` datetime DEFAULT NULL,
`batch_created_at` datetime DEFAULT NULL,
`sys_creation_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `session_filter_idx_2` (`app_id`,`platform`,`created_at`,`exception_type`,`app_version`),
KEY `batch_created_idx` (`app_id`,`platform`,`batch_created_at`),
KEY `app_id_created_at` (`app_id`,`created_at`),
KEY `id_app_id` (`app_id`,`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1636224717 DEFAULT CHARSET=latin1
is there any way possible to improve the performance of the query?
fdsfdfjdnfkjdkjfnkjdnjkfndkjnfjkndjkfnkjdnsfkjndsjknf

Indexes:
utm: INDEX(asi, user_task_id)
s: INDEX(seq_no, created_at, asi)

Related

how to rewrite a join query with CTE?

here I have tried to rewrite the query with cte cuz of good readability but when I try to rewrite the data is mismatched how to solve the problem for this?
Query;
select count(1) as rage_tap
from ue_summary.summary_funnel_1066 s
join user_tasks_metadata utm on utm.asi = s.asi
join user_tasks ut on ut.id = utm.user_task_id
where s.seq_no = 1
and s.created_at between '2022-09-27 00:00:00' and '2022-10-27 00:00:00'
and ut.is_ragetap = 1
Explain plan ;
*************************** 1. row ***************************
id: 1
select_type: SIMPLE
table: ut
partitions: NULL
type: ref
possible_keys: PRIMARY,idx_ir
key: idx_ir
key_len: 1
ref: const
rows: 8413412
filtered: 100.00
Extra: Using index
*************************** 2. row ***************************
id: 1
select_type: SIMPLE
table: utm
partitions: NULL
type: ref
possible_keys: id_asi,asi
key: id_asi
key_len: 8
ref: ue_stage.ut.id
rows: 1
filtered: 100.00
Extra: Using index
*************************** 3. row ***************************
id: 1
select_type: SIMPLE
table: s
partitions: NULL
type: eq_ref
possible_keys: PRIMARY,unique_asi_seq_no,seq_no_date,created_at,idx_combo,idx_seq_created_asi
key: unique_asi_seq_no
key_len: 12
ref: ue_stage.utm.asi,const
rows: 1
filtered: 50.00
Extra: Using where; Using index
Table structure;
Create Table: CREATE TABLE `summary_funnel_1066` (
`funnel_id` int DEFAULT NULL,
`app_id` int DEFAULT NULL,
`platform` int DEFAULT NULL,
`app_version_id` int NOT NULL,
`seq_no` int NOT NULL,
`property_id` bigint DEFAULT NULL,
`property_name` varchar(255) DEFAULT NULL,
`property_type` varchar(50) DEFAULT NULL,
`asi` bigint NOT NULL,
`created_at` datetime NOT NULL,
`capture_time_relative` decimal(15,4) DEFAULT NULL,
`last_event_id` bigint DEFAULT NULL,
`last_event_name` varchar(100) DEFAULT NULL,
`last_message_id` bigint DEFAULT NULL,
`last_message_name` varchar(100) DEFAULT NULL,
`last_tag_id` bigint DEFAULT NULL,
`last_tag_name` varchar(100) DEFAULT NULL,
`is_crash` tinyint DEFAULT NULL,
`is_anr` tinyint DEFAULT NULL,
`is_ragetap` tinyint DEFAULT NULL,
`last_error_type_id` bigint DEFAULT NULL,
`last_error_type` varchar(100) DEFAULT NULL,
`screen_id` bigint DEFAULT NULL,
`screen_name` varchar(100) DEFAULT NULL,
`last_screen_id` bigint DEFAULT NULL,
`last_screen_name` varchar(100) DEFAULT NULL,
`user_task_id` bigint DEFAULT NULL,
`ue_id` bigint DEFAULT NULL,
PRIMARY KEY (`asi`,`seq_no`,`created_at`,`app_version_id`),
UNIQUE KEY `unique_asi_seq_no` (`asi`,`seq_no`),
KEY `seq_no_date` (`seq_no`,`created_at`),
KEY `last_ids` (`last_screen_id`,`last_event_id`),
KEY `idx_seq_created_asi`(seq_no,created_at,asi),
KEY `created_at` (`created_at`),
KEY `idx_combo` (`seq_no`,`property_id`,`property_name`,`created_at`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
Table: user_tasks_metadata
Create Table: CREATE TABLE `user_tasks_metadata` (
`id` bigint NOT NULL AUTO_INCREMENT,
`user_task_id` bigint NOT NULL,
`device_id` bigint NOT NULL,
`custom_user_id` bigint DEFAULT NULL,
`asi` bigint NOT NULL DEFAULT '0',
`session_id` varchar(300) DEFAULT NULL,
`model` bigint DEFAULT NULL,
`api_level` varchar(300) DEFAULT NULL,
`app_version_id` bigint NOT NULL DEFAULT '0',
`os_version` bigint DEFAULT NULL,
`location` bigint DEFAULT NULL,
`connection_speed` varchar(10) DEFAULT NULL,
`network_operator` varchar(100) CHARACTER SET utf8mb3 COLLATE utf8_general_ci DEFAULT NULL,
`config_response` tinyint DEFAULT '1',
`total_internal_memory` double(12,5) DEFAULT NULL,
`available_internal_memory` double(12,5) DEFAULT NULL,
`total_ram` double(12,5) DEFAULT NULL,
`available_ram` double(12,5) DEFAULT NULL,
`framework` varchar(45) DEFAULT '',
`ue_sdk_version` mediumint DEFAULT NULL,
`crash_type` bigint DEFAULT NULL,
`created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`user_profile_id` bigint DEFAULT NULL,
`associated_custom_user_id` bigint DEFAULT NULL,
`first_usr_interaction` bigint DEFAULT NULL,
`app_launch_type` varchar(45) DEFAULT '',
`app_launch_time` bigint DEFAULT '0',
PRIMARY KEY (`id`),
KEY `session_metadata_filter_idx` (`custom_user_id`,`device_id`),
KEY `usertask_fk_idx` (`user_task_id`),
KEY `idx_app_version` (`app_version_id`),
KEY `asi_idx` (`asi`),
KEY `device_id` (`device_id`),
KEY `user_profile_id` (`user_profile_id`),
KEY `id_asi` (`user_task_id`,`asi`),
KEY `asi` (`asi`)
) ENGINE=InnoDB AUTO_INCREMENT=2252872743 DEFAULT CHARSET=latin1
Table: user_tasks
Create Table: CREATE TABLE `user_tasks` (
`id` bigint NOT NULL AUTO_INCREMENT,
`app_id` bigint NOT NULL,
`status` tinyint NOT NULL DEFAULT '0',
`app_version` varchar(100) DEFAULT NULL,
`platform` tinyint NOT NULL DEFAULT '1',
`exception_type` tinyint NOT NULL DEFAULT '0',
`error_count` smallint NOT NULL DEFAULT '0',
`crash_type` varchar(300) DEFAULT NULL,
`crash_log` varchar(300) DEFAULT NULL,
`avg_signal_level` int DEFAULT '0',
`is_read` tinyint(1) NOT NULL DEFAULT '0',
`is_important` tinyint(1) NOT NULL DEFAULT '0',
`is_video_available` tinyint(1) NOT NULL DEFAULT '0',
`is_video_played` tinyint(1) NOT NULL DEFAULT '0',
`is_ex` tinyint(1) NOT NULL DEFAULT '0',
`is_ragetap` tinyint(1) NOT NULL DEFAULT '0',
`session_start_time` datetime DEFAULT NULL,
`network_type` tinyint NOT NULL DEFAULT '0',
`s3_video_url` varchar(255) DEFAULT NULL,
`image_format` tinyint DEFAULT '0',
`ue_release_version` smallint NOT NULL DEFAULT '0',
`created_at` datetime NOT NULL,
`updated_at` datetime DEFAULT NULL,
`batch_created_at` datetime DEFAULT NULL,
`sys_creation_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `session_filter_idx_2` (`app_id`,`platform`,`created_at`,`exception_type`,`app_version`),
KEY `batch_created_idx` (`app_id`,`platform`,`batch_created_at`),
KEY `app_id_created_at` (`app_id`,`created_at`),
KEY `id_app_id` (`app_id`),
KEY `idx_ir` (`is_ragetap`)
) ENGINE=InnoDB AUTO_INCREMENT=1648177712 DEFAULT CHARSET=latin1
rewritten query;
with cte1 as (
select asi,count(1) as rage_tap
from ue_summary.summary_funnel_1066
where s.seq_no = 1
and s.created_at between '2022-09-27 00:00:00' and '2022-10-27 00:00:00'
),
cte2 as (
select id, count(*) 'rage_tap1'
from user_tasks ut where is_ragetap = 1
)
select cte1.*,cte2.* from cte1
inner join user_tasks_metadata utm on utm.asi = cte1.asi
inner join cte2 on b.id = utm.user_task_id
I need like below output;
+----------+
| rage_tap |
+----------+
| 1812564 |
+----------+
It takes time to search so I choose cte, I have tried with subquery but it does not work and it takes around 30 sec - 1.14 min.
as per this, I have indexed the column but also takes time : slow performance of query and scanning many rows
is there any other way to optimize it?
This is your query with two CTEs. The aggregation takes place after the tables after the joins, just as in the original query.
with s as
(
select *
from ue_summary.summary_funnel_1066
where seq_no = 1
and created_at >= date '2022-09-27'
and created_at < date '2022-10-27'
)
, ut as
(
select *
from user_tasks
where is_ragetap = 1
)
select count(*) as rage_tap
from s
join user_tasks_metadata utm on utm.asi = s.asi
join ut on ut.id = utm.user_task_id;
As created_at is a datetime, you should not use BETWEEN, but >= and <. Please check if the date range that I put in my query matches your requirements. It excludes 2022-10-27. If you want to include it, change this to and created_at < date '2022-10-28'.
select id, count(*) 'rage_tap1'
from user_tasks ut where is_ragetap = 1
does not make sense. There is an aggregate (COUNT(*)) but no GROUP BY. Were you showing to get one row? If so, which row? GROUP BY id does not make since id is Unique.
DOUBLE(m,n) is worse than simply DOUBLE. In fact, (m,n) is going away in 8.0 for FLOAT and DOUBLE.
When you have INDEX(a,b), you don't need INDEX(a).
count(1) as rage_tap is done after the JOINs, so it may have an inflated value. Did you do a sanity check?
utm has two indexes on asi. Toss both and add INDEX(asi, user_task_id)
As for turning the Joins into Ctes, go back to when you envisioned the query. You probably said "I need this stuff from this table", then "that stuff from that table", and finally "put things together this way". If you can go back to that thought process, you have the CTEs. (I don't have any idea what the data means or what the goal is, so I cannot reproduce that thought process.)

Optimizing MySQL query with INNER JOINS, LEFT JOINS, GROUP BY and HAVING

I'm having trouble optimizing this really big query and I can't change the table structure except creating additional indexes and small adjustments.
SELECT
'Fattura Prodotti Postali' AS `type`,
SUM(dpd.qta) AS `products_count_quantity`,
COUNT(dpd.IDlavorazione_dett) AS `products_count`,
GROUP_CONCAT(DISTINCT dp.prod_totali - CAST(dp.opzione1 AS UNSIGNED) SEPARATOR ' |-| ') AS `process_products_count`,
GROUP_CONCAT(DISTINCT dp.IDdistinta) AS `product_code`,
GROUP_CONCAT(DISTINCT dp.data_distinta) AS `process_date`,
GROUP_CONCAT(DISTINCT dp.IDesito) AS `process_status_id`,
GROUP_CONCAT(DISTINCT dp.note) AS `process_note`,
GROUP_CONCAT(DISTINCT dp.IDlavorazione) AS `unique_id`,
SUM(dpd.tariffa) AS `products_total`,
SUM(IF(o.IDdoc IS NULL,1,0)*dpd.qta) AS `tobill_count_quantity`,
SUM(IF(NOT o.IDdoc IS NULL,1,0)*dpd.qta) AS `billed_count_quantity`,
SUM(IF(o.IDdoc IS NULL,1,0)) AS `tobill_count`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa,0)) AS `tobill_total`,
SUM(IF(NOT o.IDdoc IS NULL,1,0)) AS `billed_count`,
SUM(IF(NOT o.IDdoc IS NULL, dpd.tariffa,0)) AS `billed_total`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa,0)-IF(NOT o.IDdoc IS NULL, dpd.tariffa,0)) AS `bill_diff`,
COUNT(dpd.IDlavorazione_dett) AS `products_count`,
SUM(dpd.tariffa) AS `products_total`,
SUM(dpd.tariffa*(dpd.iva/100)) AS `products_vat`,
SUM(dpd.tariffa*(dpd.sconto/100.0)) AS `products_discount`,
SUM(dpd.tariffa*(1.0-dpd.sconto/100.0)*(dpd.iva/100)) AS `products_discount_vat`,
SUM(dpd.tariffa*(1+(dpd.iva/100.0))) AS `products_total_vat`,
SUM(ROUND(dpd.tariffa*(1.0-dpd.sconto/100.0),5)) AS `products_total_discount`,
SUM(dpd.tariffa*(1.0-dpd.sconto/100.0)*(1+(dpd.iva/100.0))) AS `products_total_discount_vat`,
SUM(dpd.qta) AS `products_quantity`,
SUM(IF(o.IDdoc IS NULL,1,0)) AS `tobill_count`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa,0)) AS `tobill_total`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa*(1+(dpd.iva/100.0)),0)) AS `tobill_total_vat`,
SUM(IF(NOT o.IDdoc IS NULL,1,0)) AS `billed_count`,
SUM(IF(NOT o.IDdoc IS NULL, dpd.tariffa,0)) AS `billed_total`,
SUM(IF(NOT o.IDdoc IS NULL, dpd.tariffa*(1+(dpd.iva/100.0)),0)) AS `billed_total_vat`
FROM doc_prodottipostali_dett dpd
INNER JOIN tracking t ON (dpd.IDlavorazione_dett=t.product_id)
LEFT JOIN prodotti_pp ppp ON (dpd.IDprodotto=ppp.IDprodotto)
LEFT JOIN categorie_pp cpp ON (ppp.categoria=cpp.IDcategoria)
LEFT JOIN categorie_pp cppp ON (CAST(dpd.IDcategoria AS UNSIGNED)=cppp.IDcategoria)
INNER JOIN doc_prodottipostali dp ON (dpd.IDlavorazione=dp.IDlavorazione)
LEFT JOIN ordini o ON (dpd.IDfattura=o.IDdoc)
WHERE
(
(dp.tipo = 'PT' AND t.date >= '2022-05-15 00:00:00' AND t.date <= '2022-07-26 23:59:59') AND
((IF(dpd.IDprodotto>0, cpp.codice, IF(CAST(dpd.IDcategoria AS UNSIGNED)>0, cppp.codice, 'NONE')) NOT IN ('LAW','AR','CAD','EMESSOCAD') OR IF(dpd.IDprodotto>0, cpp.codice, IF(CAST(dpd.IDcategoria AS UNSIGNED)>0, cppp.codice, 'NONE')) IS NULL)) AND
((t.last IN (-1,2,3,6,7,10,11,34,35,130,131,258,514,4098,4354,8194,8450))) AND
(((dp.opzione2 = 'PI' AND dp.data_distinta < '2022-08-25')) OR ((dp.data_distinta >= '2022-08-25')) OR ((o.IDdoc >= '1'))) AND
(((t.last IN (-1,2,3,6,7,10,11,34,35,130,131,258,514,4098,4354,8194,8450))))
) AND
(
((NOT t.last IN (-1,4,5,6,7,20,21,132,133,149,516,532,1028,1157,8197)))
)
GROUP BY dpd.IDlavorazione
HAVING (1=1 AND ((tobill_count > '0')))
ORDER BY dp.data_distinta ASC;
The create tables are as following:
CREATE TABLE `doc_prodottipostali_dett` (
`IDlavorazione_dett` int(11) NOT NULL AUTO_INCREMENT,
`IDlavorazione` int(11) NOT NULL,
`IDdistinta` varchar(45) NOT NULL,
`IDcategoria` varchar(45) NOT NULL,
`IDprodotto` int(11) NOT NULL,
`codiceabarre` varchar(255) NOT NULL,
`codiceavviso` varchar(255) NOT NULL DEFAULT '',
`IDvettore` int(11) NOT NULL,
`IDlistino` int(11) NOT NULL,
`rif` varchar(45) NOT NULL,
`IDmittente` int(11) NOT NULL,
`IDdestinatario` int(11) NOT NULL,
`ufficio_mittente` varchar(45) NOT NULL,
`nome_lavoro` varchar(255) NOT NULL,
`IDpostino` int(11) DEFAULT NULL,
`note` longtext NOT NULL,
`allegati` char(1) NOT NULL,
`utente` varchar(45) NOT NULL,
`peso` decimal(10,5) NOT NULL,
`tariffa` decimal(10,5) NOT NULL,
`sconto` float NOT NULL DEFAULT 0,
`iva` int(11) NOT NULL,
`qta` int(11) NOT NULL,
`am` char(1) NOT NULL,
`cp` char(1) NOT NULL,
`eu` char(1) NOT NULL,
`aa` char(10) NOT NULL,
`ee` char(10) NOT NULL,
`stato` int(11) NOT NULL,
`lavorato` int(1) NOT NULL,
`IDesito` int(11) NOT NULL,
`esito` varchar(45) NOT NULL,
`data_op` datetime NOT NULL,
`fatturato` int(11) NOT NULL,
`data_fatt` date NOT NULL,
`IDfattura` int(11) NOT NULL,
`data` datetime DEFAULT NULL,
`data_ar` datetime DEFAULT NULL,
`nome_ar` varchar(255) DEFAULT NULL,
`file_ar` varchar(255) DEFAULT NULL,
`IDflusso_dett` int(11) NOT NULL DEFAULT 0,
`type` varchar(2) NOT NULL DEFAULT '',
`typology` varchar(2) NOT NULL DEFAULT '',
`related_id` int(11) NOT NULL DEFAULT 0,
`related_ar_id` int(11) NOT NULL DEFAULT 0,
`unregistered` int(1) NOT NULL DEFAULT 0,
`total_attachments` int(11) NOT NULL DEFAULT 0,
`repeated_recipient` int(4) DEFAULT NULL,
`law_tomanage` tinyint(4) NOT NULL DEFAULT 0,
`law_towork` int(11) NOT NULL DEFAULT 0,
`law_toprint` int(11) NOT NULL DEFAULT 0,
`notlaw_tocomplete` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`IDlavorazione_dett`),
KEY `IDvettore` (`IDvettore`),
KEY `IDmittente` (`IDmittente`),
KEY `IDdestinatario` (`IDdestinatario`),
KEY `IDlavorazione` (`IDlavorazione`),
KEY `IDfattura` (`IDfattura`),
KEY `codiceabarre` (`codiceabarre`),
KEY `rif` (`rif`),
KEY `IDpostino` (`IDpostino`),
KEY `IDcategoria` (`IDcategoria`),
KEY `IDprodotto` (`IDprodotto`),
KEY `IDlistino` (`IDlistino`),
KEY `codiceavviso` (`codiceavviso`),
KEY `IDflusso_dett` (`IDflusso_dett`),
KEY `typology` (`typology`),
KEY `nome_ar` (`nome_ar`),
KEY `unregistered` (`unregistered`),
KEY `related_ar_id` (`related_ar_id`),
KEY `related_id` (`related_id`) USING BTREE,
KEY `type` (`type`),
KEY `stato` (`stato`),
KEY `lavorato` (`lavorato`),
KEY `law_tomanage` (`law_tomanage`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `tracking` (
`tracking_id` int(11) NOT NULL AUTO_INCREMENT,
`product_id` int(11) NOT NULL,
`status_id` int(11) NOT NULL,
`lat` double NOT NULL DEFAULT 0,
`lng` double NOT NULL DEFAULT 0,
`entity_id` int(11) NOT NULL,
`sub_entity_id` int(11) NOT NULL DEFAULT 0,
`date` datetime NOT NULL,
`note` text DEFAULT NULL,
`last` int(11) NOT NULL DEFAULT 0,
`date_last` datetime NOT NULL DEFAULT current_timestamp(),
`tracking_rel` int(11) NOT NULL DEFAULT 0,
`quantity_from` int(11) NOT NULL DEFAULT 1,
`quantity_to` int(11) NOT NULL DEFAULT 1,
`price` decimal(14,4) NOT NULL DEFAULT 0.0000,
`price_unit` decimal(14,4) NOT NULL DEFAULT 0.0000,
`vat` decimal(14,4) NOT NULL DEFAULT 0.0000,
`invoice_id` int(11) NOT NULL DEFAULT 0,
`management_status` int(11) NOT NULL DEFAULT 0,
`package_id` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`tracking_id`),
KEY `product_id` (`product_id`),
KEY `status_id` (`status_id`),
KEY `entity_id` (`entity_id`),
KEY `sub_entity_id` (`sub_entity_id`),
KEY `date` (`date`),
KEY `last` (`last`),
KEY `package_id` (`package_id`),
KEY `management_status` (`management_status`),
KEY `date_last` (`date_last`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `prodotti_pp` (
`IDprodotto` int(10) unsigned NOT NULL AUTO_INCREMENT,
`codice` varchar(255) NOT NULL DEFAULT '',
`codiceabarre` varchar(255) NOT NULL DEFAULT '',
`descrizione` longtext NOT NULL,
`prezzo` decimal(10,2) NOT NULL DEFAULT 0.00,
`prezzo_2` decimal(10,2) NOT NULL DEFAULT 0.00,
`prezzo_3` decimal(10,2) NOT NULL DEFAULT 0.00,
`UM` char(3) NOT NULL DEFAULT '',
`peso` decimal(10,2) NOT NULL DEFAULT 0.00,
`datapv` date NOT NULL DEFAULT '0000-00-00',
`iva` decimal(10,0) NOT NULL DEFAULT 0,
`fornitore` varchar(45) NOT NULL DEFAULT '',
`categoria` int(10) unsigned NOT NULL DEFAULT 0,
`trasp` decimal(10,2) NOT NULL DEFAULT 0.00,
`dettaglio` longtext NOT NULL,
`ico` varchar(255) NOT NULL DEFAULT '',
`foto` varchar(255) NOT NULL DEFAULT '',
`visibility_web` char(1) NOT NULL DEFAULT '',
`composito` char(1) NOT NULL DEFAULT '',
`sottocat` int(10) unsigned NOT NULL DEFAULT 0,
`tipologia` int(10) NOT NULL DEFAULT 0,
`marchio` int(10) unsigned NOT NULL DEFAULT 0,
`disp` char(1) NOT NULL DEFAULT '',
`vetrina1` char(1) NOT NULL DEFAULT '',
`vetrina2` char(1) NOT NULL DEFAULT '',
`click` decimal(10,0) NOT NULL DEFAULT 0,
`IDnote` int(11) NOT NULL DEFAULT 1,
PRIMARY KEY (`IDprodotto`),
KEY `categoria` (`categoria`),
KEY `codice` (`codice`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
CREATE TABLE `categorie_pp` (
`IDcategoria` int(10) unsigned NOT NULL AUTO_INCREMENT,
`IDmadre` int(11) NOT NULL DEFAULT 0,
`codice` varchar(255) NOT NULL,
`nome` varchar(45) NOT NULL DEFAULT '',
`visibility_web` char(1) NOT NULL DEFAULT '',
PRIMARY KEY (`IDcategoria`),
KEY `codice` (`codice`),
KEY `IDmadre` (`IDmadre`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
CREATE TABLE `doc_prodottipostali` (
`IDlavorazione` int(11) NOT NULL AUTO_INCREMENT,
`codice_lavorazione` varchar(255) NOT NULL,
`IDdistinta` varchar(45) NOT NULL,
`tipo` varchar(45) NOT NULL,
`data_distinta` date NOT NULL,
`data_lavorazione` date NOT NULL,
`IDcliente` int(11) DEFAULT 1,
`IDpagamento` int(11) NOT NULL,
`note` longtext NOT NULL,
`utente` varchar(45) NOT NULL,
`stato` int(11) NOT NULL,
`data_op` datetime NOT NULL,
`opzioni` varchar(45) NOT NULL,
`rif` varchar(45) NOT NULL,
`opzione1` varchar(45) NOT NULL,
`opzione2` varchar(45) NOT NULL,
`allegati` int(11) NOT NULL DEFAULT 0,
`IDesito` int(11) NOT NULL,
`esito` varchar(45) NOT NULL,
`prod_totali` int(11) NOT NULL,
`prod_accettati` int(11) NOT NULL,
`prod_fatturati` int(11) NOT NULL,
`prod_chiusi` int(11) NOT NULL,
`prod_end_shipping` int(11) NOT NULL,
`stato_fatt` char(1) NOT NULL,
`IDfattura` int(11) NOT NULL DEFAULT 0,
`IDrel` int(11) NOT NULL DEFAULT 0,
`IDflow` int(11) NOT NULL DEFAULT 0,
`laws_tomanage` tinyint(4) NOT NULL DEFAULT 0,
`idSender` int(11) NOT NULL,
PRIMARY KEY (`IDlavorazione`),
KEY `IDdistinta` (`IDdistinta`),
KEY `IDfattura` (`IDfattura`),
KEY `IDrel` (`IDrel`),
KEY `allegati` (`allegati`),
KEY `tipo` (`tipo`),
KEY `IDesito` (`IDesito`),
KEY `laws_tomanage` (`laws_tomanage`),
KEY `data_distinta` (`data_distinta`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
CREATE TABLE `ordini` (
`IDdoc` int(11) NOT NULL AUTO_INCREMENT,
`IDsoggetto` int(11) DEFAULT 1,
`numero` varchar(45) NOT NULL DEFAULT '0',
`numero_web` int(11) NOT NULL,
`datadoc` date NOT NULL DEFAULT '0000-00-00',
`tipodoc` varchar(255) NOT NULL,
`type_doc` varchar(4) NOT NULL DEFAULT '',
`modpag` int(10) unsigned NOT NULL DEFAULT 0,
`div_dest` longtext NOT NULL,
`IDagente` varchar(6) NOT NULL DEFAULT '0',
`IDvettore` int(10) unsigned NOT NULL DEFAULT 0,
`imballo` decimal(20,5) NOT NULL DEFAULT 0.00000,
`colli` decimal(10,0) NOT NULL DEFAULT 0,
`datascad` date NOT NULL DEFAULT '0000-00-00',
`stato` char(1) NOT NULL DEFAULT '',
`note` longtext NOT NULL,
`importo` decimal(20,5) NOT NULL DEFAULT 0.00000,
`tipo` varchar(4) NOT NULL DEFAULT '',
`causale` varchar(45) NOT NULL DEFAULT '',
`azione` char(1) NOT NULL DEFAULT '',
`utente` varchar(45) NOT NULL DEFAULT '',
`data_op` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`acconto` decimal(20,5) NOT NULL DEFAULT 0.00000,
`iva` decimal(20,5) NOT NULL DEFAULT 0.00000,
`sconto` decimal(20,5) NOT NULL DEFAULT 0.00000,
`n_doc_passive` varchar(45) NOT NULL DEFAULT '',
`porto` varchar(45) NOT NULL DEFAULT '',
`dataora_rit` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`ordine_web` char(1) NOT NULL DEFAULT '',
`clonato` char(1) NOT NULL DEFAULT '',
`trasp` decimal(20,5) NOT NULL DEFAULT 0.00000,
`varie` decimal(20,5) NOT NULL DEFAULT 0.00000,
`banca_appo` int(11) NOT NULL DEFAULT 0,
`IDmagazzino` int(11) NOT NULL DEFAULT 0,
`pv` int(11) NOT NULL DEFAULT 0,
`fornitore` int(11) NOT NULL DEFAULT 0,
`qta_colli` decimal(10,0) NOT NULL DEFAULT 0,
`bolli` decimal(20,5) NOT NULL DEFAULT 0.00000,
`agibilitamezzi` varchar(255) NOT NULL DEFAULT '',
`opzioneprezzo` varchar(45) NOT NULL DEFAULT '',
`emailcc` varchar(255) NOT NULL DEFAULT '',
`U_IBAN` varchar(45) NOT NULL,
`stato_fatt` char(1) NOT NULL,
`IDfattura` int(11) NOT NULL,
`data_fatt` date NOT NULL,
`digital_idinvoice` varchar(255) NOT NULL DEFAULT '0',
`digital_idupload` bigint(11) NOT NULL DEFAULT 0,
`digital_status` int(11) NOT NULL DEFAULT 0,
`digital_type` varchar(2) NOT NULL DEFAULT '',
`digital_office` int(11) NOT NULL,
`digital_sectional` char(2) NOT NULL DEFAULT '',
`digital_last_office` int(11) NOT NULL,
`digital_flag` varchar(4) NOT NULL,
`determines_code` varchar(255) NOT NULL,
`determines_date` date NOT NULL,
`determines_id` varchar(255) NOT NULL,
`flag_2` varchar(255) NOT NULL,
`flag` int(11) NOT NULL,
`flag_note` longtext NOT NULL,
`flag_3` varchar(255) NOT NULL,
`type_op` varchar(255) NOT NULL,
`IDpartenza_rel` int(11) NOT NULL,
`IDdestinazione_rel` int(11) NOT NULL,
`doc_insinuazionepassivo` varchar(255) NOT NULL,
`doc_insinuazionepassivo_data` date NOT NULL DEFAULT '0000-00-00',
PRIMARY KEY (`IDdoc`),
KEY `IDsoggetto` (`IDsoggetto`),
KEY `IDfattura` (`IDfattura`),
KEY `pv` (`pv`),
KEY `azione` (`azione`),
KEY `numero` (`numero`),
KEY `tipodoc` (`tipodoc`),
KEY `datadoc` (`datadoc`),
KEY `digital_sectional` (`digital_sectional`),
KEY `digital_type` (`digital_type`),
KEY `digital_office` (`digital_office`),
KEY `digital_status` (`digital_status`),
KEY `type_doc` (`type_doc`),
KEY `flag_2` (`flag_2`),
KEY `flag` (`flag`),
KEY `flag_3` (`flag_3`),
KEY `type_op` (`type_op`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
These are the number of rows for each table:
tracking:42231628
doc_prodottipostali_det:11316150
doc_prodottipostali:40556
ordini:40360
prodotti_pp:52
categorie_pp:30
I've tried making a subquery on the tracking table (the biggest), trying to filter out most of the rows but it didn't work.
This is the explain I get, it looks fine except for the "Using temporary" and "Using filesort".
The problem with the query is that with low numbers of elaborated rows it runs fine, the moment it tries to elaborate more than 1 million it begins to slow down.
Explain
id
select_type
table
type
possible_keys
key
key_len
ref
rows
Extra
1
SIMPLE
t
range
product_id,date,last
date
5
\N
69266
Using index condition; Using where; Using temporary; Using filesort
1
SIMPLE
dpd
eq_ref
PRIMARY,IDlavorazione
PRIMARY
4
db.t.product_id
1
1
SIMPLE
ppp
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDprodotto
1
Using where
1
SIMPLE
cpp
eq_ref
PRIMARY
PRIMARY
4
db.ppp.categoria
1
Using where
1
SIMPLE
cppp
eq_ref
PRIMARY
PRIMARY
4
func
1
Using where
1
SIMPLE
o
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDfattura
1
Using index
1
SIMPLE
dp
eq_ref
PRIMARY,tipo,data_distinta
PRIMARY
4
db.dpd.IDlavorazione
1
Using where
When you try to extend the time range on the tracking table t.date >= '2022-05-15 00:00:00' AND t.date <= '2022-07-26 23:59:59' the optimizer switches to a full table scan.
I've tried forcing the 'date' index on the tracking table but it still slows down with big ranges. The users are supposed to search without the time range too, so an index on the date isn't really the best option.
Update:
I've fixed the malformed query by aggregating the nonggregated fields. The GROUP_CONCAT fields returned the same values and the ONLY_FULL_GROUP_BY setting was disabled, that's why it was working normally.
I'll be more specific on the problems that I'm facing. When a user tries to search with a more extended range of dates, for example from '2022-03-15 00:00:00' to '2022-07-26 23:59:59' the query takes 5:24 minutes to complete with the following explain:
Explain with extended date range:
id
select_type
table
type
possible_keys
key
key_len
ref
rows
Extra
1
SIMPLE
dp
ref
PRIMARY,tipo,data_distinta
tipo
137
const
20400
Using index condition; Using where; Using temporary; Using filesort
1
SIMPLE
dpd
ref
PRIMARY,IDlavorazione
IDlavorazione
4
db.dp.IDlavorazione
115
1
SIMPLE
ppp
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDprodotto
1
Using where
1
SIMPLE
cpp
eq_ref
PRIMARY
PRIMARY
4
db.ppp.categoria
1
Using where
1
SIMPLE
cppp
eq_ref
PRIMARY
PRIMARY
4
func
1
Using where
1
SIMPLE
o
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDfattura
1
Using where; Using index
1
SIMPLE
t
ref
product_id,date,last
product_id
4
db.dpd.IDlavorazione_dett
1
Using where
Update 2:
Removing the ORDER BY clause removes the 'Using temporary' and 'Using filesort' from the explain, I guess it's an index issue, even if the fields are covered by indexes. After removing the ORDER BY clause the query time is still 5 minutes.
Your query appears to filter your tracking table on IN-lists on last and a date range on date. A multi-column index on those two columns, with the equality-matched column first and the range-matched column second may help you. And your query uses product_id from that table. So this covering index is worth a try.
ALTER TABLE tracking
ADD INDEX last_date (last, date, product_id);
You can look up covering indexes to learn more.
For starters, give these indexes a try:
dpd: INDEX(IDprodotto, IDcategoria, IDlavorazione_dett, IDlavorazione, IDfattura)
dpd: INDEX(IDlavorazione_dett, IDlavorazione, IDprodotto, IDcategoria, IDfattura)
dp: INDEX(tipo, opzione2, data_distinta, IDlavorazione)
dp: INDEX(IDlavorazione)
o: INDEX(IDdoc)
t: INDEX(date, last, product_id)
t: INDEX(product_id)
ppp: INDEX(IDprodotto, categoria)
categorie_pp: INDEX(IDcategoria)
categorie_pp: INDEX(codice, IDcategoria)
When adding a composite index, DROP any plain index(es) with the same leading columns. That is, when you have both INDEX(a) and INDEX(a,b), toss the former.
CAST(dpd.IDcategoria AS UNSIGNED) -- Perhaps you should change the datatype of IDcategoria so you can avoid the conversion here? (That might lead to better index usage.)
The main things slowing down the query (and making it difficult to optimize):
WHERE clause referencing multiple tables.
OR
NOT
GROUP BY after JOINing 7 tables
But I don't see how to improve on any of them.

Need to optimize mysql query of a PHP framework

I have a query which is generated via PHP framework core function. I don't have control to change the query. Therefore i will need to perform optimization on server side i.e. mysql to execute this query in efficient time. I have applied some indices but still it is taking around 4-5 seconds and ideally it should take 1-1.5 seconds. Following is the query:
(
SELECT rr.rt_bids_aos_quotes_relaos_quotes_idb AS so_id,
rr.sales_order_sequence sequence,
so.*
FROM rt_bids_aos_quotes_rel AS rr
INNER JOIN aos_quotes AS so ON so.id = rr.rt_bids_aos_quotes_relaos_quotes_idb
WHERE rr.deleted = 0
AND rr.rt_bids_aos_quotes_relrt_bids_ida='490395-403600-b'
)
UNION
(
SELECT ra.rt_bids_aos_quotes_altaos_quotes_idb AS so_id,
'' AS sequence,
so.*
FROM rt_bids_aos_quotes_alternate AS ra
INNER JOIN aos_quotes AS so ON so.id = ra.rt_bids_aos_quotes_altaos_quotes_idb
WHERE ra.deleted = 0
AND ra.rt_bids_aos_quotes_altrt_bids_ida='490395-403600-b'
)
Following is the Explain query result:Image_here
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY rt_bids_aos_quotes_rel const idx_rt_bids_aos_quotes_relrt_bids_ida_deleted,rt_bids_aos_quotes_rel_alt,idx_rt_bids_aos_quotes_rel_rt_bids_aos_quotes_relaos_quotes_idb idx_rt_bids_aos_quotes_relrt_bids_ida_deleted 113 const,const 1 NULL
1 PRIMARY so ALL NULL NULL NULL NULL 631950 Using where
2 UNION NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables
NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL Using temporary
Show create table results :
CREATE TABLE `rt_bids_aos_quotes_rel` (
`id` varchar(36) NOT NULL,
`date_modified` datetime DEFAULT NULL,
`deleted` tinyint(1) DEFAULT '0',
`rt_bids_aos_quotes_relrt_bids_ida` varchar(36) DEFAULT NULL,
`rt_bids_aos_quotes_relaos_quotes_idb` varchar(36) DEFAULT NULL,
`sales_order_sequence` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_rt_bids_aos_quotes_relrt_bids_ida_deleted` (`deleted`,`rt_bids_aos_quotes_relrt_bids_ida`),
KEY `rt_bids_aos_quotes_rel_alt` (`rt_bids_aos_quotes_relrt_bids_ida`,`rt_bids_aos_quotes_relaos_quotes_idb`),
KEY `idx_rt_bids_aos_quotes_rel_rt_bids_aos_quotes_relaos_quotes_idb` (`rt_bids_aos_quotes_relaos_quotes_idb`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `aos_quotes` (
`id` char(36) NOT NULL,
`name` varchar(255) DEFAULT NULL,
`date_entered` datetime DEFAULT NULL,
`date_modified` datetime DEFAULT NULL,
`modified_user_id` char(36) DEFAULT NULL,
`created_by` char(36) DEFAULT NULL,
`description` text,
`deleted` tinyint(1) DEFAULT '0',
`assigned_user_id` char(36) DEFAULT NULL,
`approval_issue` text,
`billing_account_id` char(36) DEFAULT NULL,
`billing_contact_id` char(36) DEFAULT NULL,
`billing_address_street` varchar(150) DEFAULT NULL,
`billing_address_city` varchar(100) DEFAULT NULL,
`billing_address_state` varchar(100) DEFAULT NULL,
`billing_address_postalcode` varchar(20) DEFAULT NULL,
`billing_address_country` varchar(255) DEFAULT NULL,
`shipping_address_street` varchar(150) DEFAULT NULL,
`shipping_address_city` varchar(100) DEFAULT NULL,
`shipping_address_state` varchar(100) DEFAULT NULL,
`shipping_address_postalcode` varchar(20) DEFAULT NULL,
`shipping_address_country` varchar(255) DEFAULT NULL,
`expiration` date DEFAULT NULL,
`number` int(11) NOT NULL,
`opportunity_id` char(36) DEFAULT NULL,
`template_ddown_c` text,
`total_amt` decimal(26,6) DEFAULT NULL,
`total_amt_usdollar` decimal(26,6) DEFAULT NULL,
`subtotal_amount` decimal(26,6) DEFAULT NULL,
`subtotal_amount_usdollar` decimal(26,6) DEFAULT NULL,
`discount_amount` decimal(26,6) DEFAULT NULL,
`discount_amount_usdollar` decimal(26,6) DEFAULT NULL,
`tax_amount` decimal(26,6) DEFAULT NULL,
`tax_amount_usdollar` decimal(26,6) DEFAULT NULL,
`shipping_amount` decimal(26,6) DEFAULT NULL,
`shipping_amount_usdollar` decimal(26,6) DEFAULT NULL,
`shipping_tax` varchar(100) DEFAULT NULL,
`shipping_tax_amt` decimal(26,6) DEFAULT NULL,
`shipping_tax_amt_usdollar` decimal(26,6) DEFAULT NULL,
`total_amount` decimal(26,6) DEFAULT NULL,
`total_amount_usdollar` decimal(26,6) DEFAULT NULL,
`currency_id` char(36) DEFAULT NULL,
`stage` varchar(100) DEFAULT 'Draft',
`term` varchar(100) DEFAULT NULL,
`terms_c` text,
`approval_status` varchar(100) DEFAULT NULL,
`invoice_status` varchar(100) DEFAULT 'Not Invoiced',
`subtotal_tax_amount` decimal(26,6) DEFAULT NULL,
`subtotal_tax_amount_usdollar` decimal(26,6) DEFAULT NULL,
`bid_id` char(36) DEFAULT NULL,
`alt` varchar(255) DEFAULT NULL,
`group_desc` longtext,
`hold` tinyint(1) DEFAULT '0',
`order_amount` decimal(26,2) DEFAULT NULL,
`order_description` longtext,
`status` varchar(100) DEFAULT NULL,
`type` varchar(255) DEFAULT NULL,
`contract_id` char(36) DEFAULT NULL,
`customer_discount` decimal(26,2) DEFAULT NULL,
`customer_markup` decimal(26,2) DEFAULT NULL,
`markup_inv_type` decimal(26,2) DEFAULT NULL,
`location_id` char(36) DEFAULT NULL,
`active` varchar(100) DEFAULT 'Active',
`city` varchar(255) DEFAULT NULL,
`rt_jobs_id` char(36) DEFAULT NULL,
`system_type` varchar(36) DEFAULT NULL,
`com_address` varchar(255) DEFAULT NULL,
`com_city` varchar(255) DEFAULT NULL,
`com_mapsco` varchar(255) DEFAULT NULL,
`com_state_zip` varchar(255) DEFAULT NULL,
`job_type` varchar(100) DEFAULT NULL,
`calc_labor` varchar(100) DEFAULT 'Item_Install_amt',
`comission` decimal(10,4) DEFAULT '0.0000',
`hourly_labor_rate` decimal(8,4) DEFAULT NULL,
`labor_percentage_of_price` decimal(12,2) DEFAULT NULL,
`overhead` decimal(12,4) DEFAULT '0.0000',
`profit` decimal(12,4) DEFAULT '0.0000',
`pricing_checkbox` tinyint(1) DEFAULT '0',
`pkg_package_id` char(36) DEFAULT NULL,
`dh_factor` decimal(6,2) DEFAULT '0.00',
`addl_builder_price` decimal(12,4) DEFAULT '0.0000',
`billing_notes` longtext,
`billing_notes_home` longtext,
`builder_percentage` decimal(12,4) DEFAULT '0.0000',
`discount` decimal(12,4) DEFAULT '0.0000',
`jobs_contact_homeowner_id` char(36) DEFAULT NULL,
`mortage` varchar(100) DEFAULT 'mortage',
`oh_percentage` decimal(12,2) DEFAULT '0.00',
`origin` varchar(255) DEFAULT NULL,
`package_items` decimal(10,2) DEFAULT NULL,
`package_items_unit_left` decimal(10,2) DEFAULT '0.00',
`selected_package_units` decimal(10,2) DEFAULT '0.00',
`jobs_account_superintendent_id` char(36) DEFAULT NULL,
`subdivision_selected_id` char(36) DEFAULT NULL,
`subdivision_selected_name` varchar(255) DEFAULT NULL,
`department` varchar(255) DEFAULT 'Dallas',
`locked` tinyint(1) DEFAULT '0',
`billed_amount` decimal(26,2) DEFAULT '0.00',
`billed_percentage` decimal(26,2) DEFAULT '0.00',
`retained_amount` decimal(26,2) DEFAULT '0.00',
`selected_package_amount` decimal(26,2) DEFAULT '0.00',
`builder_amount` decimal(26,2) DEFAULT '0.00',
`home_owner_amount` decimal(26,2) DEFAULT '0.00',
`builderContractAmount` decimal(26,2) DEFAULT '0.00',
`homeOwnerContractAmount` decimal(26,2) DEFAULT '0.00',
`ho_pct` decimal(26,6) DEFAULT '0.000000',
`builder_pct` decimal(26,6) DEFAULT '0.000000',
`labor_pct` decimal(26,6) DEFAULT '0.000000',
`mortgage` tinyint(1) DEFAULT '0',
`ho_order` tinyint(1) DEFAULT '0',
`home_owner_sale_order` tinyint(1) DEFAULT '0',
`directly_created_contract` tinyint(1) DEFAULT '0',
`crew_manager_id` char(36) DEFAULT NULL,
`estimate_date` date DEFAULT NULL,
`complete` tinyint(1) DEFAULT '0',
`complete_note` text,
`plan_estimate_date` date DEFAULT NULL,
`plan_manager_id` char(36) DEFAULT NULL,
`lock_bid_id` char(36) DEFAULT NULL,
`documents_id` text,
`no_item_change` tinyint(1) DEFAULT '0',
`tagged_at_yard` tinyint(1) DEFAULT '0',
`soap_created_so` tinyint(1) DEFAULT '0',
`soap_created_so_amount` decimal(10,2) DEFAULT '0.00',
`subcon_vendor` varchar(255) DEFAULT NULL,
`plan_complete` varchar(100) DEFAULT 'In Progress',
`plan_required` tinyint(1) DEFAULT '0',
`plan_important` varchar(255) DEFAULT '0',
`confirm` varchar(100) DEFAULT 'Unconfirm',
`designer_notes` text,
`plan_due_date` date DEFAULT NULL,
`plan_required_so` tinyint(1) DEFAULT '0',
`plan_request` tinyint(1) DEFAULT '0',
`material_hold_status` varchar(255) DEFAULT NULL,
`wh_warehouse_id` char(36) DEFAULT NULL,
`work_order_created` tinyint(1) DEFAULT '0',
`maintenance_status` varchar(100) DEFAULT NULL,
`classification_type` varchar(255) DEFAULT NULL,
`pricing_type` varchar(255) DEFAULT NULL,
`estimate_end_date` date DEFAULT NULL,
`install_date` date DEFAULT NULL,
`is_matched` tinyint(1) DEFAULT '0',
`builder_discount` decimal(10,2) DEFAULT '0.00',
`ho_discount` decimal(10,2) DEFAULT '0.00',
`locate_required` tinyint(1) DEFAULT '0',
`priority` varchar(100) DEFAULT NULL,
`is_no_charged` tinyint(1) DEFAULT '0',
`no_charge_reasons` varchar(255) DEFAULT NULL,
`no_charge_users` varchar(50) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_aos_quotes_type` (`type`),
KEY `idx_aos_quotes_rt_jobs_id` (`rt_jobs_id`),
KEY `idx_aos_quotes_id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE TABLE `rt_bids_aos_quotes_alternate` (
`id` varchar(36) NOT NULL,
`date_modified` datetime DEFAULT NULL,
`deleted` tinyint(1) DEFAULT '0',
`rt_bids_aos_quotes_altrt_bids_ida` varchar(36) DEFAULT NULL,
`rt_bids_aos_quotes_altaos_quotes_idb` varchar(36) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_rt_bids_aos_quotes_altrt_bids_ida_deleted` (`deleted`,`rt_bids_aos_quotes_altrt_bids_ida`),
KEY `rt_bids_aos_quotes_alt` (`rt_bids_aos_quotes_altrt_bids_ida`,`rt_bids_aos_quotes_altaos_quotes_idb`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
Please advise how can i improve it.
Your default charset for aos_quotes is latin1, but the default charset for the other tables is utf8. When you JOIN based on comparing two strings with different collations, they can't use the index, so they are forced to do a table-scan. I have no doubt that's what's slowing down your query.
When I use your tables as is, I get this EXPLAIN for the so tables:
*************************** 2. row ***************************
id: 1
select_type: PRIMARY
table: so
partitions: NULL
type: ALL
possible_keys: NULL
key: NULL
key_len: NULL
ref: NULL
rows: 1
filtered: 100.00
Extra: Using where
When I convert your aos_quotes table to utf8, I get this EXPLAIN for the so tables:
*************************** 2. row ***************************
id: 1
select_type: PRIMARY
table: so
partitions: NULL
type: const
possible_keys: PRIMARY,idx_aos_quotes_id
key: PRIMARY
key_len: 108
ref: const
rows: 1
filtered: 100.00
Extra: NULL
The "type: PRIMARY" is much better than "type: ALL".
So you need to convert your aos_quotes table to utf8.
See How do I change a MySQL table to UTF-8?
Other possible improvements:
A PRIMARY KEY is a UNIQUE key is a KEY. So KEY idx_aos_quotes_id (id) is redundant and could (should) be dropped.
I see VARCHAR(36); that usually smacks of random UUIDs. But your query shows something else. What is the situation. (The randomness of standard UUIDs is a performance problem in large tables.)
I see lots of TEXT columns and SELECT .. so.* fetching all of them. If you don't need them all, don't fetch them all. Each one is potentially an extra disk hit.
UNION defaults to UNION DISTINCT, which involves a de-dup pass. If you don't need that, then UNION ALL would be faster.
(Not a speed issue, just a readability issue.) Can you remove the clutter of the "rt_bids_aos_quotes_" prefix?
If rt_bids_aos_quotes_alternate and rt_bids_aos_quotes_rel are many:many mappings, see my blog for several performance tips.
decimal(26,6) takes 12 bytes. Do you need that much precision and range? Or would some other datatype do? Smaller -> more cacheable -> less I/O -> faster. (Rule of Thumb: DECIMAL(m,n) occupies about m/2 bytes. INT and FLOAT occupy exactly 4 bytes. TINYINT: 1 byte.)
In addition to fixing the CHARSET for id, fix the inconsistency between CHAR(36) and VARCHAR(36), especially if your ids are only 15 characters. (Again, space may lead to speed.)
I see UNIQUE(deleted, ida) -- Does this mean that there can be 1 or 2 rows with a given ida? If there can be only one (either deleted or not), then make (ida) the PK.

mysql query taking about 3 seconds - and all keys indexed

I have a simple join between 3 mysql tables with 1613, 369, and 500 rows respectively. I believe I have everything indexed but the query is still taking 3 seconds or more. My query is:
SELECT
fe.pid, fe.encounter, s.Templates_ID
FROM
(form_encounter fe
JOIN em__structure s ON fe.pc_catid=s.Templates_ID
JOIN em__assemblies a ON s.Massemblies_ID=a.Massemblies_ID)
When I do an EXPLAIN on this I get the following, sorry for the format:
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE a index Massemblies_ID Massemblies_ID 4 NULL 651 Using index; Using temporary; Using filesort
1 SIMPLE s ref Templates_ID,Massemblies_ID Massemblies_ID 4 oaks_1410.a.Massemblies_ID 1
1 SIMPLE fe ref pc_catid pc_catid,4,oaks_1410.s.Templates_ID 100 Using where
Here are my 3 tables, I am trying to put every available clue in place to figure this out:
CREATE TABLE `form_encounter` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`date` datetime DEFAULT NULL,
`reason` longtext,
`facility` longtext,
`facility_id` int(11) NOT NULL DEFAULT '0',
`pid` bigint(20) DEFAULT NULL,
`encounter` bigint(20) DEFAULT NULL,
`onset_date` datetime DEFAULT NULL,
`sensitivity` varchar(30) DEFAULT NULL,
`pc_catid` int(11) NOT NULL DEFAULT '5' COMMENT 'event category from openemr_postcalendar_categories',
`last_stmt_date` date DEFAULT NULL,
`stmt_count` int(11) NOT NULL DEFAULT '0',
`provider_id` int(11) DEFAULT '0' COMMENT 'default and main provider for this visit',
`supervisor_id` int(11) DEFAULT '0' COMMENT 'supervising provider, if any, for this visit',
`invoice_refno` varchar(31) NOT NULL DEFAULT '',
`referral_source` varchar(31) NOT NULL DEFAULT '',
`billing_facility` int(11) NOT NULL DEFAULT '0',
`billing_status` int(11) DEFAULT '10' COMMENT 'see form_encounter_status_codes',
`billing_due_date` date DEFAULT NULL COMMENT 'date an encounter becomes due',
`last_level_billed` int(11) NOT NULL DEFAULT '0' COMMENT '0=none, 1=ins1, 2=ins2, etc',
`last_level_closed` int(11) NOT NULL DEFAULT '0' COMMENT '0=none, 1=ins1, 2=ins2, etc',
`billing_note` text,
`workmans_comp` varchar(1) DEFAULT 'N',
`workmans_comp_claim_number` varchar(40) DEFAULT NULL COMMENT 'workmans comp claim number',
`prior_authorization_number` varchar(40) DEFAULT NULL COMMENT 'prior authorization number for workmans comp',
`auto_accident` varchar(1) DEFAULT 'N',
`auto_accident_state` varchar(2) DEFAULT NULL COMMENT 'auto accident state',
`other_accident` varchar(1) DEFAULT 'N',
`referral_provider_id` int(11) DEFAULT NULL,
`referral_provider_npi` varchar(10) DEFAULT NULL,
`signed_by` int(11) NOT NULL DEFAULT '0' COMMENT 'Provider who signed the encounter',
`pc_templateid` varchar(10) DEFAULT NULL COMMENT 'Physical exam template used for this encounter',
`pc_poptemplateid` int(10) unsigned DEFAULT NULL COMMENT 'Encounter template including data population',
`pc_poptemplatestatus` tinyint(1) unsigned DEFAULT NULL COMMENT 'Whether ecounter has been materially modified from template',
`deleted` tinyint(4) NOT NULL DEFAULT '0' COMMENT 'flag indicates form has been deleted',
PRIMARY KEY (`id`),
UNIQUE KEY `pid_encounter_unique` (`pid`,`encounter`) USING BTREE,
KEY `pc_catid` (`pc_catid`)
) ENGINE=InnoDB AUTO_INCREMENT=1614 DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC
Second table:
CREATE TABLE `em__structure` (
`ID` int(11) unsigned NOT NULL AUTO_INCREMENT,
`Templates_ID` int(11) unsigned DEFAULT NULL,
`Sections_ID` int(11) unsigned DEFAULT NULL,
`Idx` mediumint(4) unsigned DEFAULT '1',
`Massemblies_ID` int(5) unsigned NOT NULL DEFAULT '0',
`EditDate` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `Templates_ID` (`Templates_ID`),
KEY `Sections_ID` (`Sections_ID`),
KEY `Massemblies_ID` (`Massemblies_ID`)
) ENGINE=InnoDB
Third table:
CREATE TABLE `em__assemblies` (
`ID` int(11) unsigned NOT NULL AUTO_INCREMENT,
`Massemblies_ID` int(5) unsigned NOT NULL DEFAULT '0',
`Groupings_ID` int(11) unsigned DEFAULT NULL,
`Components_ID` int(11) unsigned DEFAULT NULL,
`Idx` mediumint(4) unsigned DEFAULT '1',
`Sex` char(1) DEFAULT NULL,
`Responsetypes_ID` int(11) DEFAULT NULL,
PRIMARY KEY (`ID`),
KEY `Groupings_ID` (`Groupings_ID`),
KEY `Components_ID` (`Components_ID`),
KEY `Responsetypes_ID` (`Responsetypes_ID`),
KEY `Massemblies_ID` (`Massemblies_ID`)
) ENGINE=InnoDB AUTO_INCREMENT=505 DEFAULT CHARSET=latin1
The joins are all on indexed integer keys.

Avoiding filesort and query optimization

I have written a query and optimized it to extent i can. But still query takes 3 minutes to give output.
Please find the below query.
SELECT ps_product.id_product,ps_product_lang.name,pa.id_product_attribute,
pa.`ean13` AS `a_ean13`,MAX(IF((agl.name LIKE "Color"),al.name,'')) AS Color,
MAX(IF((agl.name LIKE "Size"),al.name,'')) AS Size,
ps_product.price,
ps_product_lang.`description`,`ps_product_lang`.`description_short`,`ps_product_lang`.`meta_description`,
`ps_product_lang`.`meta_keywords`,`ps_product_lang`.`meta_title`,pa.`default_on`,
ps_product.`additional_shipping_cost`,
pa.`reference`,
pa.`price`
FROM
`ps_product`
JOIN ps_product_lang ON ps_product_lang.id_product = ps_product.id_product
join ps_product_attribute pa on pa.`id_product` = ps_product.id_product
LEFT JOIN ps_product_attribute_combination pac ON pac.id_product_attribute = pa.id_product_attribute
LEFT JOIN ps_attribute a ON a.id_attribute = pac.id_attribute
LEFT JOIN ps_attribute_group ag ON ag.id_attribute_group = a.id_attribute_group
LEFT JOIN ps_attribute_lang al ON (a.id_attribute = al.id_attribute AND al.id_lang = 1)
LEFT JOIN ps_attribute_group_lang agl ON (ag.id_attribute_group = agl.id_attribute_group AND agl.id_lang = 1)
GROUP BY pa.id_product_attribute ORDER BY ps_product.id_product;
Here is the explain command output
id select_type table type possible_keys key key_len ref rows Extra
------ ----------- --------------- ------ --------------------------------------------------------------------------- ------------------------------------------ ------- ------------------------------- ------ ---------------------------------
1 SIMPLE ps_product ALL PRIMARY (NULL) (NULL) (NULL) 11378 Using temporary; Using filesort
1 SIMPLE pa ref product_attribute_product,product_default product_attribute_product 4 catalog.ps_product.id_product 2
1 SIMPLE pac ref id_product_attribute id_product_attribute 4 catalog.pa.id_product_attribute 1 Using index
1 SIMPLE a eq_ref PRIMARY PRIMARY 4 catalog.pac.id_attribute 1
1 SIMPLE ag eq_ref PRIMARY PRIMARY 4 catalog.a.id_attribute_group 1 Using index
1 SIMPLE al ref id_lang,ps_attribute_lang_lang_idx,ps_attribute_lang_attribute_idx ps_attribute_lang_attribute_idx 4 catalog.a.id_attribute 1
1 SIMPLE agl ref ps_attribute_group_lang_attributegroup_idx,ps_attribute_group_lang_lang_idx ps_attribute_group_lang_attributegroup_idx 4 catalog.ag.id_attribute_group 1
1 SIMPLE ps_product_lang ref ps_product_lang_product_lang_shop,ps_product_lang_product_idx ps_product_lang_product_lang_shop 4 catalog.ps_product.id_product 3
Is there any way i can improve performance.
Here are the create statements.
CREATE TABLE `ps_product` (
`id_product` int(10) unsigned NOT NULL AUTO_INCREMENT,
`id_supplier` int(10) unsigned DEFAULT NULL,
`id_manufacturer` int(10) unsigned DEFAULT NULL,
`id_category_default` int(10) unsigned DEFAULT NULL,
`id_shop_default` int(10) unsigned NOT NULL DEFAULT '1',
`id_tax_rules_group` int(11) unsigned NOT NULL,
`on_sale` tinyint(1) unsigned NOT NULL DEFAULT '0',
`online_only` tinyint(1) unsigned NOT NULL DEFAULT '0',
`ean13` varchar(13) DEFAULT NULL,
`upc` varchar(12) DEFAULT NULL,
`ecotax` decimal(17,6) NOT NULL DEFAULT '0.000000',
`quantity` int(10) NOT NULL DEFAULT '0',
`minimal_quantity` int(10) unsigned NOT NULL DEFAULT '1',
`price` decimal(20,6) NOT NULL DEFAULT '0.000000',
`wholesale_price` decimal(20,6) NOT NULL DEFAULT '0.000000',
`unity` varchar(255) DEFAULT NULL,
`unit_price_ratio` decimal(20,6) NOT NULL DEFAULT '0.000000',
`additional_shipping_cost` decimal(20,2) NOT NULL DEFAULT '0.00',
`reference` varchar(32) DEFAULT NULL,
`supplier_reference` varchar(32) DEFAULT NULL,
`location` varchar(64) DEFAULT NULL,
`width` decimal(20,6) NOT NULL DEFAULT '0.000000',
`height` decimal(20,6) NOT NULL DEFAULT '0.000000',
`depth` decimal(20,6) NOT NULL DEFAULT '0.000000',
`weight` decimal(20,6) NOT NULL DEFAULT '0.000000',
`out_of_stock` int(10) unsigned NOT NULL DEFAULT '2',
`quantity_discount` tinyint(1) DEFAULT '0',
`customizable` tinyint(2) NOT NULL DEFAULT '0',
`uploadable_files` tinyint(4) NOT NULL DEFAULT '0',
`text_fields` tinyint(4) NOT NULL DEFAULT '0',
`active` tinyint(1) unsigned NOT NULL DEFAULT '0',
`available_for_order` tinyint(1) NOT NULL DEFAULT '1',
`available_date` date NOT NULL,
`condition` enum('new','used','refurbished') NOT NULL DEFAULT 'new',
`show_price` tinyint(1) NOT NULL DEFAULT '1',
`indexed` tinyint(1) NOT NULL DEFAULT '0',
`visibility` enum('both','catalog','search','none') NOT NULL DEFAULT 'both',
`cache_is_pack` tinyint(1) NOT NULL DEFAULT '0',
`cache_has_attachments` tinyint(1) NOT NULL DEFAULT '0',
`is_virtual` tinyint(1) NOT NULL DEFAULT '0',
`cache_default_attribute` int(10) unsigned DEFAULT NULL,
`date_add` datetime NOT NULL,
`date_upd` datetime NOT NULL,
`advanced_stock_management` tinyint(1) NOT NULL DEFAULT '0',
`style_guide_url` varchar(100) DEFAULT NULL,
`sequence` int(10) DEFAULT '0',
`a_createddate` datetime DEFAULT NULL,
`a_createdby` bigint(20) DEFAULT NULL,
`a_modifieddate` datetime DEFAULT NULL,
`a_modifiedby` bigint(20) DEFAULT NULL,
PRIMARY KEY (`id_product`),
KEY `product_supplier` (`id_supplier`),
KEY `product_manufacturer` (`id_manufacturer`),
KEY `id_category_default` (`id_category_default`),
KEY `indexed` (`indexed`),
KEY `date_add` (`date_add`),
KEY `ps_product.active_ind` (`active`)
)
and
CREATE TABLE `ps_product_lang` (
`a_productlangid` int(11) NOT NULL AUTO_INCREMENT,
`id_product` int(10) unsigned NOT NULL,
`id_shop` int(11) unsigned NOT NULL DEFAULT '1',
`id_lang` int(10) unsigned NOT NULL,
`description` text,
`description_short` text,
`link_rewrite` varchar(128) NOT NULL,
`meta_description` varchar(255) DEFAULT NULL,
`meta_keywords` varchar(255) DEFAULT NULL,
`meta_title` varchar(128) DEFAULT NULL,
`name` varchar(128) NOT NULL,
`available_now` varchar(255) DEFAULT NULL,
`available_later` varchar(255) DEFAULT NULL,
`a_createddate` datetime DEFAULT NULL,
`a_createdby` bigint(20) DEFAULT NULL,
`a_modifieddate` datetime DEFAULT NULL,
`a_modifiedby` bigint(20) DEFAULT NULL,
PRIMARY KEY (`a_productlangid`,`id_product`,`id_shop`,`id_lang`),
UNIQUE KEY `ps_product_lang_product_lang_shop` (`id_product`,`id_shop`,`id_lang`),
KEY `id_lang` (`id_lang`),
KEY `name` (`name`),
KEY `ps_product_lang_product_idx` (`id_product`),
KEY `ps_product_lang_shop_idx` (`id_shop`),
CONSTRAINT `ps_product_lang_lang` FOREIGN KEY (`id_lang`) REFERENCES `ps_lang` (`id_lang`) ON DELETE NO ACTION ON UPDATE NO ACTION,
CONSTRAINT `ps_product_lang_product` FOREIGN KEY (`id_product`) REFERENCES `ps_product` (`id_product`) ON DELETE NO ACTION ON UPDATE NO ACTION,
CONSTRAINT `ps_product_lang_shop` FOREIGN KEY (`id_shop`) REFERENCES `ps_shop` (`id_shop`) ON DELETE NO ACTION ON UPDATE NO ACTION
)