Related
I have big table 'placement' with 12 558 392 records;
When I try to get data using this table I face the performance problem(load time arount 5 seconds).
When I explain this query all looks good but query time too long.
My query example:
SELECT SQL_NO_CACHE om.*
FROM order_materials om
INNER JOIN material m ON om.material_id = m.id AND om.deleted = FALSE
INNER JOIN placement p ON m.id = p.material_id AND m.deleted = FALSE AND p.deleted = FALSE
INNER JOIN block b ON p.block_id = b.id AND b.deleted = FALSE
INNER JOIN orders o ON om.order_id = o.id AND o.deleted = FALSE
INNER JOIN product pr ON pr.mediaPlan_id = p.mediaplan_id
WHERE
b.advTable_id = 139
AND p.date >= '2018-03-01 00:00:00' AND p.date <= '2018-04-01 00:00:00'
GROUP BY om.material_id;
Explain:
+----+-------------+-------+------------+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+---------+---------------------------+------+----------+----------------------------------------------+
| id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+-------------+-------+------------+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+---------+---------------------------+------+----------+----------------------------------------------+
| 1 | SIMPLE | b | NULL | ref | PRIMARY,FK597C48D47B04A3 | FK597C48D47B04A3 | 5 | const | 455 | 50.00 | Using where; Using temporary; Using filesort |
| 1 | SIMPLE | p | NULL | ref | FK6ADE12E521DC3251,FK6ADE12E59B1CA2F1,FK6ADE12E5AFA9B543,date_ind,placement_b,placement_material_id_mediaplan_id_index,placement_material_id,placement_material_id_mediaplan_id_order_id_block_id_index,block_id_date_ind | FK6ADE12E521DC3251 | 5 | openmarket.b.id | 135 | 0.82 | Using where |
| 1 | SIMPLE | pr | NULL | ref | FKED8DCCEF9B1CA2F1 | FKED8DCCEF9B1CA2F1 | 5 | openmarket.p.mediaplan_id | 1 | 100.00 | Using index |
| 1 | SIMPLE | m | NULL | eq_ref | PRIMARY | PRIMARY | 4 | openmarket.p.material_id | 1 | 50.00 | Using where |
| 1 | SIMPLE | om | NULL | ref | FK_order_materials_1,FK_order_materials_2 | FK_order_materials_2 | 4 | openmarket.p.material_id | 2 | 50.00 | Using where |
| 1 | SIMPLE | o | NULL | eq_ref | PRIMARY | PRIMARY | 4 | openmarket.om.order_id | 1 | 50.00 | Using where |
+----+-------------+-------+------------+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+---------+---------------------------+------+----------+----------------------------------------------+
SHOW CREATE TABLE order_materials:
CREATE TABLE `order_materials` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`order_id` int(11) NOT NULL,
`material_id` int(11) NOT NULL,
`deleted` bit(1) NOT NULL DEFAULT b'0',
`created_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`last_mod_user_id` int(11) DEFAULT NULL,
`placements_count` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `FK_order_materials_1` (`order_id`),
KEY `FK_order_materials_2` (`material_id`),
CONSTRAINT `FK_order_materials_1` FOREIGN KEY (`order_id`) REFERENCES `orders` (`id`),
CONSTRAINT `FK_order_materials_2` FOREIGN KEY (`material_id`) REFERENCES `material` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=251369 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
SHOW CREATE TABLE placement;
CREATE TABLE `placement` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`changeDate` datetime DEFAULT NULL,
`date` datetime NOT NULL,
`plannedPosition` tinyint(4) DEFAULT NULL,
`realPosition` tinyint(4) DEFAULT NULL,
`positionWithPolitics` tinyint(4) DEFAULT NULL,
`material_id` int(11) DEFAULT NULL,
`mediaplan_id` int(11) NOT NULL,
`block_id` int(11) DEFAULT NULL,
`visible` bit(1) NOT NULL,
`blockStartTime` datetime DEFAULT NULL,
`price` float DEFAULT NULL,
`pricedPrice` float DEFAULT NULL,
`actualStartTime` datetime DEFAULT NULL,
`playedPosition` tinyint(4) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`played_material_id` int(11) DEFAULT NULL,
`deleted` bit(1) NOT NULL DEFAULT b'0',
`created_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`last_mod_user_id` int(11) DEFAULT NULL,
`conflict_status_lid` int(11) DEFAULT NULL COMMENT 'lookup category placement_conflict',
`conflict_by_type_in_block` bit(1) NOT NULL DEFAULT b'0',
`conflict_by_type_near` bit(1) NOT NULL DEFAULT b'0',
`conflict_by_time_overflow` bit(1) NOT NULL DEFAULT b'0',
`conflict_by_position` bit(1) NOT NULL DEFAULT b'0',
`order_id` int(11) DEFAULT NULL,
`order_status_lid` int(11) DEFAULT NULL,
`play_type_lid` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `FK6ADE12E521DC3251` (`block_id`),
KEY `FK6ADE12E59B1CA2F1` (`mediaplan_id`),
KEY `FK6ADE12E5AFA9B543` (`material_id`),
KEY `blockstarttime` (`blockStartTime`),
KEY `date_ind` (`date`),
KEY `bst_rp_ind` (`blockStartTime`,`realPosition`),
KEY `status_ind` (`status`),
KEY `FK_played_material` (`played_material_id`),
KEY `FK_placement_1` (`conflict_status_lid`),
KEY `FK_placement_2` (`order_id`),
KEY `FK_placement_3` (`order_status_lid`),
KEY `FK_placement_4` (`play_type_lid`),
KEY `placement_b` (`date`,`blockStartTime`,`block_id`,`plannedPosition`),
KEY `placement_material_id_mediaplan_id_index` (`material_id`,`mediaplan_id`),
KEY `placement_material_id` (`material_id`,`mediaplan_id`,`order_id`),
KEY `placement_material_id_mediaplan_id_order_id_block_id_index` (`material_id`,`mediaplan_id`,`order_id`,`block_id`),
KEY `block_id_date_ind` (`block_id`,`date`),
CONSTRAINT `FK6ADE12E521DC3251` FOREIGN KEY (`block_id`) REFERENCES `block` (`id`),
CONSTRAINT `FK6ADE12E59B1CA2F1` FOREIGN KEY (`mediaplan_id`) REFERENCES `mediaplan` (`id`),
CONSTRAINT `FK6ADE12E5AFA9B543` FOREIGN KEY (`material_id`) REFERENCES `material` (`id`),
CONSTRAINT `FK_placement_1` FOREIGN KEY (`conflict_status_lid`) REFERENCES `lookups` (`id`),
CONSTRAINT `FK_placement_2` FOREIGN KEY (`order_id`) REFERENCES `orders` (`id`),
CONSTRAINT `FK_placement_3` FOREIGN KEY (`order_status_lid`) REFERENCES `lookups` (`id`),
CONSTRAINT `FK_placement_4` FOREIGN KEY (`play_type_lid`) REFERENCES `lookups` (`id`),
CONSTRAINT `FK_played_material` FOREIGN KEY (`played_material_id`) REFERENCES `played_material` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=12578822 DEFAULT CHARSET=utf8 COMMENT='Розміщення рекламного матеріалу')
) ENGINE=InnoDB AUTO_INCREMENT=251369 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
After rewrite query with exist i have next explain
+----+--------------------+-------+------------+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+---------+---------------------------+--------+----------+-------------+
| id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+--------------------+-------+------------+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+---------+---------------------------+--------+----------+-------------+
| 1 | PRIMARY | om | NULL | ALL | NULL | NULL | NULL | NULL | 243300 | 50.00 | Using where |
| 6 | DEPENDENT SUBQUERY | o | NULL | eq_ref | PRIMARY | PRIMARY | 4 | openmarket.om.order_id | 1 | 50.00 | Using where |
| 2 | DEPENDENT SUBQUERY | m | NULL | eq_ref | PRIMARY | PRIMARY | 4 | openmarket.om.material_id | 1 | 100.00 | Using where |
| 3 | DEPENDENT SUBQUERY | p | NULL | ref | FK6ADE12E5AFA9B543,date_ind,placement_b,placement_material_id_mediaplan_id_index,placement_material_id,placement_material_id_mediaplan_id_order_id_block_id_index | placement_material_id | 5 | openmarket.m.id | 101 | 0.82 | Using where |
| 5 | DEPENDENT SUBQUERY | pr | NULL | ref | FKED8DCCEF9B1CA2F1 | FKED8DCCEF9B1CA2F1 | 5 | openmarket.p.mediaplan_id | 1 | 100.00 | Using index |
| 4 | DEPENDENT SUBQUERY | b | NULL | eq_ref | PRIMARY,FK597C48D47B04A3 | PRIMARY | 4 | openmarket.p.block_id | 1 | 5.00 | Using where |
+----+--------------------+-------+------------+--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+---------+---------------------------+--------+----------+-------------+
As far as I understand from your schema, you are looking for a distinct list of "order_materials" to be filtered according to the other listed tables.
In my experience, you can't filter/group/distinct results with joins without suffering the "using temporary - using filesort"
In your case, anyway, since it looks that you don't need the values from the other tables, I believe it would be possible to rewrite your query removing all the joins and just using the EXISTS clause.
I would try something like this:
SELECT SQL_NO_CACHE om.*
FROM order_materials om
where
om.deleted = false
and
exists (
select 1
from material m
where exists
(select 1 from placement p
where m.id = p.material_id
AND m.deleted = FALSE
AND p.deleted = FALSE
and p.date >= '2018-03-01 00:00:00' AND p.date <= '2018-04-01 00:00:00'
and exists (select 1 from block b
where p.block_id = b.id
AND b.deleted = FALSE
and b.advTable_id = 139
)
and exists (select 1 from product pr
where pr.mediaPlan_id = p.mediaplan_id)
)
and m.id=om.material_id
)
and exists (select 1 from orders o
where om.order_id = o.id
AND o.deleted = FALSE)
The explain plan of such a query is:
+------+--------------------+-------------+--------+-----------------------------------+--------------+---------+--------------------+------+-------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+------+--------------------+-------------+--------+-----------------------------------+--------------+---------+--------------------+------+-------------+
| 1 | PRIMARY | om | ALL | order_id,material_id | NULL | NULL | NULL | 1 | Using where |
| 1 | PRIMARY | m | eq_ref | PRIMARY | PRIMARY | 4 | abc.om.material_id | 1 | Using where |
| 1 | PRIMARY | o | eq_ref | PRIMARY | PRIMARY | 4 | abc.om.order_id | 1 | Using where |
| 3 | DEPENDENT SUBQUERY | p | ALL | material_id,block_id,mediaplan_id | NULL | NULL | NULL | 1 | Using where |
| 3 | DEPENDENT SUBQUERY | <subquery5> | eq_ref | distinct_key | distinct_key | 4 | func | 1 | |
| 3 | DEPENDENT SUBQUERY | b | eq_ref | PRIMARY,advTable_id | PRIMARY | 4 | abc.p.block_id | 1 | Using where |
| 5 | MATERIALIZED | pr | index | mediaPlan_id | mediaPlan_id | 5 | NULL | 1 | Using index |
+------+--------------------+-------------+--------+-----------------------------------+--------------+---------+--------------------+------+-------------+
As you can see, no temporary, no filesort. Therefore, since exists is a boolean operator, you won't receive duplicates as it happens when you join.
Finally, please, be extremely careful: I wrote this answer on the fly, you need to check that the nested exists are consistent with your expected results, this iis not the final solution but just a shared hint on what I've learned so far.
order_materials seems to be a many:many mapping table. The schema is inefficient. Get rid of id and make the other changes suggested in here .
block needs INDEX(advTable_id, deleted, id)
placement needs INDEX(material_id, deleted, date)
There are several redundant indexes. Follow this rule: If you have INDEX(a, b), you don't need INDEX(a).
key_len = 5 usually refers to INT NULL -- check to see if they should be INT NOT NULL.
I have this query:
EXPLAIN EXTENDED
SELECT DISTINCT
PMS_STAGIONI.DINIZVAL,
PMS_STAGIONI.DFINEVAL,
PMS_DISPO.DDATA
FROM
PMS_DISPO JOIN PMS_STAGIONI
HAVING
PMS_DISPO.DDATA BETWEEN PMS_STAGIONI.DINIZVAL AND PMS_STAGIONI.DFINEVAL
The output of explain is:
+----+-------------+--------------+-------+---------------+------------------------------+---------+------+------+----------+--------------------------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+-------------+--------------+-------+---------------+------------------------------+---------+------+------+----------+--------------------------------+
| 1 | SIMPLE | PMS_STAGIONI | index | NULL | IDX_INIZFINEVAL_PMS_STAGIONI | 6 | NULL | 3 | 100.00 | Using index; Using temporary |
| 1 | SIMPLE | PMS_DISPO | index | NULL | IDX_DDATA_PMS_DISPO | 3 | NULL | 1199 | 100.00 | Using index; Using join buffer |
+----+-------------+--------------+-------+---------------+------------------------------+---------+------+------+----------+--------------------------------+
My question is how to calculate the product of the join using explain. For example, in this case are performed 3597 (1199x3) scans or only 1199?
1)If I add "ORDER BY DDATA" lines scanned in the table "PMS_DISPO" become 1130.
2)If I use the "WHERE" clause instead of "HAVING" clause scan no longer uses the indexes. How is it possible?
3)If i want show PMS_STAGIONI.CSTAGIONI (primary key) explain show me that:
+----+-------------+--------------+-------+---------------+---------------------+---------+------+------+----------+--------------------------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+-------------+--------------+-------+---------------+---------------------+---------+------+------+----------+--------------------------------+
| 1 | SIMPLE | PMS_STAGIONI | ALL | NULL | NULL | NULL | NULL | 3 | 100.00 | Using temporary |
| 1 | SIMPLE | PMS_DISPO | index | NULL | IDX_DDATA_PMS_DISPO | 3 | NULL | 1130 | 100.00 | Using index; Using join buffer |
+----+-------------+--------------+-------+---------------+---------------------+---------+------+------+----------+--------------------------------+
How can I force the use of the other index?
Thanks in advance.
Edit:
The structure of "PMS_DISPO" is:
CREATE TABLE IF NOT EXISTS `PMS_DISPO` (
`ID` int(11) NOT NULL AUTO_INCREMENT ,
`CPRENOTA` int(11) NOT NULL,
`DDATA` date NOT NULL,
`CCATRIS` int(4) NOT NULL,
`NQUANT` int(4) NOT NULL,
`CAZIENDA` int(4) NOT NULL,
`CAFFILIATO` int(4) NOT NULL,
PRIMARY KEY (`ID`),
KEY `IDX_DDATA_PMS_DISPO` (`DDATA`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1084 ;
And "PMS_STAGIONI" is:
CREATE TABLE IF NOT EXISTS `PMS_STAGIONI` (
`CSTAGIONE` int(11) NOT NULL,
`NVALIDI` tinyint(2) NOT NULL,
`BECCEZIONE` tinyint(1) NOT NULL,
`AGGSET` varchar(7) DEFAULT NULL,
`DINIZVAL` date NOT NULL,
`DFINEVAL` date NOT NULL,
`CAZIENDA` int(4) NOT NULL,
`ID` int(11) NOT NULL AUTO_INCREMENT,
PRIMARY KEY (`ID`),
KEY `CSTAGIONE` (`CSTAGIONE`),
KEY `IDX_INIZFINEVAL_PMS_STAGIONI` (`DINIZVAL`,`DFINEVAL`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=4 ;
A query of this sort would normally be written as follows, with indexes just as you have them...
SELECT DISTINCT s.dinizval
, s.dfineval
, d.ddata
FROM pms_dispo d
JOIN pms_stagioni s
ON d.ddata BETWEEN s.dinizval AND s.dfineval
This is the shortversion of my query:
SELECT product.* FROM product_list product
LEFT JOIN language_item language ON (product.title=language.languageVariable)
WHERE language.languageID = 1
ORDER BY language.languageValue ASC
When I use it, the query has 3 seconds. When I remove the order by the query has 0.3 seconds. Can you recommend a change to accelerate it?
product.title and language.languageVariable is a language variable like global.product.title1, and languageValue is the title like car, doll or something else.
CREATE TABLE `language_item` (
`languageItemID` int(10) UNSIGNED NOT NULL,
`languageID` int(10) UNSIGNED NOT NULL DEFAULT '0',
`languageVariable` varchar(255) NOT NULL DEFAULT '',
`languageValue` mediumtext NOT NULL,
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
ALTER TABLE `language_item`
ADD PRIMARY KEY (`languageItemID`),
ADD UNIQUE KEY `languageVariable` (`languageVariable`,`languageID`),
ADD KEY `languageValue` (`languageValue`(300));
id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra
1 | SIMPLE | product | NULL | ALL | PRIMARY,inactive,archive,productCategoryID | NULL | NULL | NULL | 1475 | 88.27 | Using where; Using temporary; Using filesort
1 | SIMPLE | language | NULL | ref | languageVariable | languageVariable | 767 | db.product.title | 136 | 1.00 | Using index condition
Here is the structur from language_item with the index:
CREATE TABLE `language_item` (
`languageItemID` int(10) UNSIGNED NOT NULL,
`languageID` int(10) UNSIGNED NOT NULL DEFAULT '0',
`languageVariable` varchar(255) NOT NULL DEFAULT '',
`languageValue` mediumtext NOT NULL,
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
ALTER TABLE `language_item`
ADD PRIMARY KEY (`languageItemID`),
ADD UNIQUE KEY `languageVariable` (`languageVariable`,`languageID`),
ADD KEY `languageValue` (`languageValue`(300));
The Explain:
id | select_type | table | partitions | type | possible_keys | key |
key_len | ref | rows | filtered | Extra 1 | SIMPLE | product | NULL |
ALL | PRIMARY,inactive,archive,productCategoryID | NULL | NULL | NULL
| 1475 | 88.27 | Using where; Using temporary; Using filesort 1 |
SIMPLE | language | NULL | ref | languageVariable | languageVariable |
767 | db.product.title | 136 | 1.00 | Using index condition
TRy this:
SELECT d.* from (
SELECT product.*, language.languageValue AS lv
FROM product_list product
JOIN language_item language ON (product.title=language.languageVariable)
WHERE language.languageID = 1
) as d
ORDER BY d.lv ASC
I have a table with 18,310,298 records right now.
And next query
SELECT COUNT(obj_id) AS cnt
FROM
`common`.`logs`
WHERE
`event` = '11' AND
`obj_type` = '2' AND
`region` = 'us' AND
DATE(`date`) = DATE('20120213010502');
With next structure
CREATE TABLE `logs` (
`log_id` int(11) NOT NULL AUTO_INCREMENT,
`event` tinyint(4) NOT NULL,
`obj_type` tinyint(1) NOT NULL DEFAULT '0',
`obj_id` int(11) unsigned NOT NULL DEFAULT '0',
`region` varchar(3) NOT NULL DEFAULT '',
`date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`log_id`),
KEY `event` (`event`),
KEY `obj_type` (`obj_type`),
KEY `region` (`region`),
KEY `for_stat` (`event`,`obj_type`,`obj_id`,`region`,`date`)
) ENGINE=InnoDB AUTO_INCREMENT=83126347 DEFAULT CHARSET=utf8 COMMENT='Logs table' |
and MySQL explain show the next
+----+-------------+-------+------+--------------------------------+----------+---------+-------------+--------+----------+--------------------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+-------------+-------+------+--------------------------------+----------+---------+-------------+--------+----------+--------------------------+
| 1 | SIMPLE | logs | ref | event,obj_type,region,for_stat | for_stat | 2 | const,const | 837216 | 100.00 | Using where; Using index |
+----+-------------+-------+------+--------------------------------+----------+---------+-------------+--------+----------+--------------------------+
1 row in set, 1 warning (0.00 sec)
Running such query in daily peak usage time take about 5 seconds.
What can I do to make it faster ?
UPDATED: Regarding all comments I modified INDEX and take off DATE function in WHERE clause
+-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+
| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment |
+-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+
| logs | 0 | PRIMARY | 1 | log_id | A | 15379109 | NULL | NULL | | BTREE | |
| logs | 1 | event | 1 | event | A | 14 | NULL | NULL | | BTREE | |
| logs | 1 | obj_type | 1 | obj_type | A | 14 | NULL | NULL | | BTREE | |
| logs | 1 | region | 1 | region | A | 14 | NULL | NULL | | BTREE | |
| logs | 1 | for_stat | 1 | event | A | 157 | NULL | NULL | | BTREE | |
| logs | 1 | for_stat | 2 | obj_type | A | 157 | NULL | NULL | | BTREE | |
| logs | 1 | for_stat | 3 | region | A | 157 | NULL | NULL | | BTREE | |
| logs | 1 | for_stat | 4 | date | A | 157 | NULL | NULL | | BTREE | |
+-------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+
mysql> explain extended SELECT COUNT(obj_id) as cnt
-> FROM `common`.`logs`
-> WHERE `event`= '11' AND
-> `obj_type` = '2' AND
-> `region`= 'est' AND
-> date between '2012-11-25 00:00:00' and '2012-11-25 23:59:59';
+----+-------------+-------+-------+--------------------------------+----------+---------+------+------+----------+-------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+-------------+-------+-------+--------------------------------+----------+---------+------+------+----------+-------------+
| 1 | SIMPLE | logs | range | event,obj_type,region,for_stat | for_stat | 21 | NULL | 9674 | 75.01 | Using where |
+----+-------------+-------+-------+--------------------------------+----------+---------+------+------+----------+-------------+
It seems it's running faster. Thanks everyone.
The EXPLAIN output shows that the query is using only the first two columns of the for_stat index.
This is because the query doesn't use obj_id in the WHERE clause. If you create a new key without obj_id (or modify the existing key to reorder the columns), more of the key can be used and you may see better performance:
KEY `for_stat2` (`event`,`obj_type`,`region`,`date`)
If it's still too slow, changing the last condition, where you use DATE(), as said by Salman and Sashi, might improve things.
#Joni already explained what is wrong with your index. For query, I assume that your example query selects all records for 2012-02-13 regardless of time. You can change the where clause to use >= and < instead of DATE cast:
SELECT COUNT(obj_id) AS cnt
FROM
`common`.`logs`
WHERE
`event` = 11 AND
`obj_type` = 2 AND
`region` = 'us' AND
`date` >= DATE('20120213010502') AND
`date` < DATE('20120213010502') + INTERVAL 1 DAY
The date function on the date column is making the full table scan.
Try this ::
SELECT COUNT(obj_id) as cnt
FROM
`common`.`logs`
WHERE
`event` = 11
AND
`obj_type` = 2
AND
`region` = 'us'
AND
`date` = DATE('20120213010502')
As logging (inserts) needs to be fast too, use as less indices as possible.
Evaluation may take long as that is admin, not necessarily needing indices.
CREATE TABLE `logs` (
`log_id` int(11) NOT NULL AUTO_INCREMENT,
`event` tinyint(4) NOT NULL,
`obj_type` tinyint(1) NOT NULL DEFAULT '0',
`obj_id` int(11) unsigned NOT NULL DEFAULT '0',
`region` varchar(3) NOT NULL DEFAULT '',
`date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`log_id`),
KEY `for_stat` (`event`,`obj_type`,`region`,`date`)
) ENGINE=InnoDB AUTO_INCREMENT=83126347 DEFAULT CHARSET=utf8 COMMENT='Logs table' |
And about the date search #SashiKant and #SalmanA already answered.
Is Mysql you should place index columns by collation count; less possible values in table - placed closer to the left.
Also you can try to change column region to enum() and try to search date with BETWEEN clause.
Mysql is not using third column in the index because it's usage takes more efforts then just filtering (it's a common thing in Mysql).
Goal: Display ethnicity data by district for each race and year as a percent of total.
Problem: Query takes over 30 minutes to run. I would like to get this down to under 10 seconds. Is there another strategy to solve this problem that I am not thinking of?
Query:
SELECT field_data_field_district_id.field_district_id_value as district_id, year, race, ROUND(SUM(count)/(
SELECT SUM(count)
FROM school_data_race_ethnicity_raw as school_data_race_ethnicity_raw_inner
INNER JOIN field_data_field_school_id as field_data_field_school_id_inner ON field_data_field_school_id_inner.field_school_id_value = school_data_race_ethnicity_raw_inner.school_id
INNER JOIN field_data_field_district as field_data_field_district_inner ON field_data_field_district_inner.entity_id = field_data_field_school_id_inner.entity_id
INNER JOIN field_data_field_district_id as field_data_field_district_id_inner ON field_data_field_district_inner.field_district_nid = field_data_field_district_id_inner.entity_id
WHERE field_data_field_district_id_inner.field_district_id_value = field_data_field_district_id.field_district_id_value and
school_data_race_ethnicity_raw_inner.year = school_data_race_ethnicity_raw.year
) * 100 ,2) as percent
FROM school_data_race_ethnicity_raw
INNER JOIN field_data_field_school_id ON field_data_field_school_id.field_school_id_value = school_data_race_ethnicity_raw.school_id
INNER JOIN field_data_field_district ON field_data_field_district.entity_id = field_data_field_school_id.entity_id
INNER JOIN field_data_field_district_id ON field_data_field_district.field_district_nid = field_data_field_district_id.entity_id
GROUP BY district_id, year, race
Explain:
+----+--------------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+--------------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
| 1 | PRIMARY | field_data_field_district_id | ALL | entity_id | NULL | NULL | NULL | 685 | Using temporary; Using filesort |
| 1 | PRIMARY | field_data_field_district | ref | entity_id,field_district_nid | field_district_nid | 5 | rocdocs_main_drupal_7.field_data_field_district_id.entity_id | 3 | Using where; Using index |
| 1 | PRIMARY | field_data_field_school_id | ref | entity_id | entity_id | 4 | rocdocs_main_drupal_7.field_data_field_district.entity_id | 1 | |
| 1 | PRIMARY | school_data_race_ethnicity_raw | ALL | NULL | NULL | NULL | NULL | 97098 | Using where; Using join buffer |
| 2 | DEPENDENT SUBQUERY | field_data_field_district_id_inner | ALL | entity_id | NULL | NULL | NULL | 685 | Using where |
| 2 | DEPENDENT SUBQUERY | field_data_field_district_inner | ref | entity_id,field_district_nid | field_district_nid | 5 | rocdocs_main_drupal_7.field_data_field_district_id_inner.entity_id | 3 | Using where; Using index |
| 2 | DEPENDENT SUBQUERY | field_data_field_school_id_inner | ref | entity_id | entity_id | 4 | rocdocs_main_drupal_7.field_data_field_district_inner.entity_id | 1 | |
| 2 | DEPENDENT SUBQUERY | school_data_race_ethnicity_raw_inner | ref | year | year | 4 | func | 32366 | Using where |
+----+--------------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
Tables:
school_data_race_ethnicity_raw - (Approx. 100,000 rows)
mysql> show create table school_data_race_ethnicity_raw;
+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Table | Create Table |
+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| school_data_race_ethnicity_raw | CREATE TABLE `school_data_race_ethnicity_raw` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`school_id` varchar(255) NOT NULL,
`year` int(11) NOT NULL,
`race` varchar(255) NOT NULL,
`count` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `school_id` (`school_id`,`year`),
KEY `year` (`year`,`race`),
KEY `school_id_2` (`school_id`)
) ENGINE=MyISAM AUTO_INCREMENT=97099 DEFAULT CHARSET=latin1 |
+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)
field_data_field_school_id - Approx. 5000 rows
mysql> show create table field_data_field_school_id;
+----------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Table | Create Table |
+----------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| field_data_field_school_id | CREATE TABLE `field_data_field_school_id` (
`entity_type` varchar(128) NOT NULL DEFAULT '' COMMENT 'The entity type this data is attached to',
`bundle` varchar(128) NOT NULL DEFAULT '' COMMENT 'The field instance bundle to which this row belongs, used when deleting a field instance',
`deleted` tinyint(4) NOT NULL DEFAULT '0' COMMENT 'A boolean indicating whether this data item has been deleted',
`entity_id` int(10) unsigned NOT NULL COMMENT 'The entity id this data is attached to',
`revision_id` int(10) unsigned DEFAULT NULL COMMENT 'The entity revision id this data is attached to, or NULL if the entity type is not versioned',
`language` varchar(32) NOT NULL DEFAULT '' COMMENT 'The language for this data item.',
`delta` int(10) unsigned NOT NULL COMMENT 'The sequence number for this data item, used for multi-value fields',
`field_school_id_value` varchar(255) DEFAULT NULL,
`field_school_id_format` varchar(255) DEFAULT NULL,
PRIMARY KEY (`entity_type`,`entity_id`,`deleted`,`delta`,`language`),
KEY `entity_type` (`entity_type`),
KEY `bundle` (`bundle`),
KEY `deleted` (`deleted`),
KEY `entity_id` (`entity_id`),
KEY `revision_id` (`revision_id`),
KEY `language` (`language`),
KEY `field_school_id_format` (`field_school_id_format`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Data storage for field 234 (field_school_id)' |
+----------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)
field_data_field_district - Approx. 5000 rows
mysql> show create table field_data_field_district;
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Table | Create Table |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| field_data_field_district | CREATE TABLE `field_data_field_district` (
`entity_type` varchar(128) NOT NULL DEFAULT '' COMMENT 'The entity type this data is attached to',
`bundle` varchar(128) NOT NULL DEFAULT '' COMMENT 'The field instance bundle to which this row belongs, used when deleting a field instance',
`deleted` tinyint(4) NOT NULL DEFAULT '0' COMMENT 'A boolean indicating whether this data item has been deleted',
`entity_id` int(10) unsigned NOT NULL COMMENT 'The entity id this data is attached to',
`revision_id` int(10) unsigned DEFAULT NULL COMMENT 'The entity revision id this data is attached to, or NULL if the entity type is not versioned',
`language` varchar(32) NOT NULL DEFAULT '' COMMENT 'The language for this data item.',
`delta` int(10) unsigned NOT NULL COMMENT 'The sequence number for this data item, used for multi-value fields',
`field_district_nid` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`entity_type`,`entity_id`,`deleted`,`delta`,`language`),
KEY `entity_type` (`entity_type`),
KEY `bundle` (`bundle`),
KEY `deleted` (`deleted`),
KEY `entity_id` (`entity_id`),
KEY `revision_id` (`revision_id`),
KEY `language` (`language`),
KEY `field_district_nid` (`field_district_nid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Data storage for field 221 (field_district)' |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)
field_data_field_district_id - Approx: 1000 rows
mysql> show create table field_data_field_district_id;
+------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Table | Create Table |
+------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| field_data_field_district_id | CREATE TABLE `field_data_field_district_id` (
`entity_type` varchar(128) NOT NULL DEFAULT '' COMMENT 'The entity type this data is attached to',
`bundle` varchar(128) NOT NULL DEFAULT '' COMMENT 'The field instance bundle to which this row belongs, used when deleting a field instance',
`deleted` tinyint(4) NOT NULL DEFAULT '0' COMMENT 'A boolean indicating whether this data item has been deleted',
`entity_id` int(10) unsigned NOT NULL COMMENT 'The entity id this data is attached to',
`revision_id` int(10) unsigned DEFAULT NULL COMMENT 'The entity revision id this data is attached to, or NULL if the entity type is not versioned',
`language` varchar(32) NOT NULL DEFAULT '' COMMENT 'The language for this data item.',
`delta` int(10) unsigned NOT NULL COMMENT 'The sequence number for this data item, used for multi-value fields',
`field_district_id_value` varchar(255) DEFAULT NULL,
`field_district_id_format` varchar(255) DEFAULT NULL,
PRIMARY KEY (`entity_type`,`entity_id`,`deleted`,`delta`,`language`),
KEY `entity_type` (`entity_type`),
KEY `bundle` (`bundle`),
KEY `deleted` (`deleted`),
KEY `entity_id` (`entity_id`),
KEY `revision_id` (`revision_id`),
KEY `language` (`language`),
KEY `field_district_id_format` (`field_district_id_format`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Data storage for field 218 (field_district_id)' |
+------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
NOTE: The only table I can change is school_data_race_ethnicity_raw as the others are generated by drupal.
mysql> explain SELECT field_data_field_district_id_inner.field_district_id_value,
-> school_data_race_ethnicity_raw_inner.year,
-> SUM(count) as total
-> FROM school_data_race_ethnicity_raw as school_data_race_ethnicity_raw_inner
-> INNER JOIN field_data_field_school_id as field_data_field_school_id_inner
-> ON field_data_field_school_id_inner.field_school_id_value
-> = school_data_race_ethnicity_raw_inner.school_id
-> INNER JOIN field_data_field_district as field_data_field_district_inner
-> ON field_data_field_district_inner.entity_id
-> = field_data_field_school_id_inner.entity_id
-> INNER JOIN field_data_field_district_id as field_data_field_district_id_inner
-> ON field_data_field_district_inner.field_district_nid
-> = field_data_field_district_id_inner.entity_id
-> GROUP BY field_district_id_value, year
-> ;
+----+-------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+-------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
| 1 | SIMPLE | field_data_field_district_id_inner | ALL | entity_id | NULL | NULL | NULL | 773 | Using temporary; Using filesort |
| 1 | SIMPLE | field_data_field_district_inner | ref | entity_id,field_district_nid | field_district_nid | 5 | rocdocs_main_drupal_7.field_data_field_district_id_inner.entity_id | 3 | Using where; Using index |
| 1 | SIMPLE | field_data_field_school_id_inner | ref | entity_id | entity_id | 4 | rocdocs_main_drupal_7.field_data_field_district_inner.entity_id | 1 | |
| 1 | SIMPLE | school_data_race_ethnicity_raw_inner | ALL | NULL | NULL | NULL | NULL | 97098 | Using where; Using join buffer |
+----+-------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
4 rows in set (0.00 sec)
How about this:
SELECT field_data_field_district_id.field_district_id_value as district_id,
x.year,
race,
ROUND(SUM(count)/(x.total) * 100 , 2) as percent
FROM school_data_race_ethnicity_raw
INNER JOIN field_data_field_school_id
ON field_data_field_school_id.field_school_id_value
= school_data_race_ethnicity_raw.school_id
INNER JOIN field_data_field_district
ON field_data_field_district.entity_id
= field_data_field_school_id.entity_id
INNER JOIN field_data_field_district_id
ON field_data_field_district.field_district_nid
= field_data_field_district_id.entity_id
INNER JOIN (
SELECT field_data_field_district_id_inner.field_district_id_value,
school_data_race_ethnicity_raw_inner.year,
SUM(count) as total
FROM school_data_race_ethnicity_raw as school_data_race_ethnicity_raw_inner
INNER JOIN field_data_field_school_id as field_data_field_school_id_inner
ON field_data_field_school_id_inner.field_school_id_value
= school_data_race_ethnicity_raw_inner.school_id
INNER JOIN field_data_field_district as field_data_field_district_inner
ON field_data_field_district_inner.entity_id
= field_data_field_school_id_inner.entity_id
INNER JOIN field_data_field_district_id as field_data_field_district_id_inner
ON field_data_field_district_inner.field_district_nid
= field_data_field_district_id_inner.entity_id
GROUP BY field_district_id_value, year
) x ON x.field_district_id_value = field_data_field_district_id.field_district_id_value
AND x.year = school_data_race_ethnicity_raw.year
GROUP BY district_id, x.year, race
I basically moved your query that calculates the total count for each district/year out of the SELECT and into another JOIN.
Explain:
+----+-------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+-------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
| 1 | PRIMARY | field_data_field_district_id | ALL | entity_id | NULL | NULL | NULL | 773 | Using temporary; Using filesort |
| 1 | PRIMARY | field_data_field_district | ref | entity_id,field_district_nid | field_district_nid | 5 | rocdocs_main_drupal_7.field_data_field_district_id.entity_id | 3 | Using where; Using index |
| 1 | PRIMARY | field_data_field_school_id | ref | entity_id | entity_id | 4 | rocdocs_main_drupal_7.field_data_field_district.entity_id | 1 | |
| 1 | PRIMARY | <derived2> | ALL | NULL | NULL | NULL | NULL | 1902 | Using where; Using join buffer |
| 1 | PRIMARY | school_data_race_ethnicity_raw | ref | year | year | 4 | x.year | 32366 | Using where |
| 2 | DERIVED | field_data_field_district_id_inner | ALL | entity_id | NULL | NULL | NULL | 773 | Using temporary; Using filesort |
| 2 | DERIVED | field_data_field_district_inner | ref | entity_id,field_district_nid | field_district_nid | 5 | rocdocs_main_drupal_7.field_data_field_district_id_inner.entity_id | 3 | Using where; Using index |
| 2 | DERIVED | field_data_field_school_id_inner | ref | entity_id | entity_id | 4 | rocdocs_main_drupal_7.field_data_field_district_inner.entity_id | 1 | |
| 2 | DERIVED | school_data_race_ethnicity_raw_inner | ALL | NULL | NULL | NULL | NULL | 97098 | Using where; Using join buffer |
+----+-------------+--------------------------------------+------+------------------------------+--------------------+---------+--------------------------------------------------------------------+-------+---------------------------------+
9 rows in set (4 min 0.59 sec)