Query is taking too much time .and impacting performance - mysql

The below query is taking around 10mins to 1 hr to execute. And due to this we are getting load spikes frequently. Can anyone please help rewrite the query in a more optimised way.
Query:
select count(*)
from t_tab6 tab1
left join t_tab6_element_tab2 tab2
on tab2.tab6_id = tab1.id
left join t_tab3 tab3
on tab1.create_tab3_id = tab3.id
left join t_tab4 tab4
on tab2.element_id = tab4.id and tab2.element_type_id = 1
left join t_tab7 tab5
on tab2.element_id = tab5.id and tab2.element_type_id = 2
where tab1.domain_id = 522
Explain plan:
+----+-------------+-----------+--------+-------------------------+-------------------------+---------+----------------------------+--------+-------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+-------------+-----------+--------+-------------------------+-------------------------+---------+----------------------------+--------+-------------+
| 1 | SIMPLE | tab1 | ref | domain_id_tab6_type_cd | domain_id_tab6_type_cd | 4 | const | 243430 | |
| 1 | SIMPLE | tab2 | ref | FK_tab6 | FK_tab6 | 4 | comp1.tab1.id | 2 | |
| 1 | SIMPLE | tab3 | eq_ref | PRIMARY | PRIMARY | 4 | comp1.tab1.create_tab3_id | 1 | Using index |
| 1 | SIMPLE | tab4 | ref | id | id | 4 | comp1.tab2.element_id | 1 | Using index |
| 1 | SIMPLE | tab5 | ref | id | id | 4 | comp1.tab2.element_id | 1 | Using index |
+----+-------------+-----------+--------+-------------------------+-------------------------+---------+----------------------------+--------+-------------+
5 rows in set (0.45 sec)
Tables structure:
mysql> show create table t_tab6_element_tab2\G
*************************** 1. row ***************************
Table: t_tab6_element_tab2
Create Table: CREATE TABLE `t_tab6_element_tab2` (
`id` int(255) NOT NULL AUTO_INCREMENT,
`element_type_id` int(11) NOT NULL,
`tab6_id` int(11) NOT NULL,
`element_id` int(11) NOT NULL,
`element_desc` varchar(500) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `FK_tab6` (`tab6_id`),
KEY `element_type_id_element_id` (`element_type_id`,`element_id`)
) ENGINE=InnoDB AUTO_INCREMENT=45901159 DEFAULT CHARSET=utf8
1 row in set (0.00 sec)
mysql> show create table t_tab3\G
*************************** 1. row ***************************
Table: t_tab3
Create Table: CREATE TABLE `t_tab3` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL,
`password` varchar(255) DEFAULT NULL,
`email` varchar(255) DEFAULT NULL,
`is_admin` int(11) DEFAULT NULL,
`phone` varchar(50) DEFAULT NULL,
`company` varchar(255) DEFAULT NULL,
`last_login` datetime DEFAULT NULL,
`state` int(11) DEFAULT NULL,
`validate_code` varchar(50) DEFAULT NULL,
`has_login` smallint(6) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1304 DEFAULT CHARSET=utf8
1 row in set (0.00 sec)
mysql> show create table t_tab6\G
*************************** 1. row ***************************
Table: t_tab6
Create Table: CREATE TABLE `t_tab6` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`create_tab3_id` int(11) DEFAULT NULL,
`tab6_create_date` date DEFAULT NULL,
`tab6_type_cd` int(11) NOT NULL,
`tab6_desc` varchar(512) NOT NULL,
`IsGlobaltab6` int(2) DEFAULT NULL,
`tab6_start_date` datetime NOT NULL,
`tab6_end_date` datetime NOT NULL,
`job_id` int(11) DEFAULT NULL,
`domain_id` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `tab6_tab3_FK` (`create_tab3_id`),
KEY `domain_id_tab6_type_cd` (`domain_id`,`tab6_type_cd`)
) ENGINE=InnoDB AUTO_INCREMENT=8586007 DEFAULT CHARSET=utf8
1 row in set (0.00 sec)
mysql> show create table t_tab4\G
*************************** 1. row ***************************
Table: t_tab4
Create Table: CREATE TABLE `t_tab4` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`tab4_name` varchar(255) DEFAULT NULL,
`tab4_value` varchar(255) DEFAULT NULL,
`type` int(11) DEFAULT NULL,
`description` varchar(2000) DEFAULT NULL,
`own_domain_id` int(11) DEFAULT NULL,
`rank_check` int(11) DEFAULT NULL,
`rank1` int(11) DEFAULT NULL,
`rank2` int(11) DEFAULT NULL,
`rank3` int(11) DEFAULT NULL,
`yesterday_entrances` int(11) DEFAULT NULL,
`week_entrances` int(11) DEFAULT NULL,
`current_ctr` float(16,4) DEFAULT NULL,
`monthly_search_volume` int(11) DEFAULT NULL,
`avg_monthly_search_volume` int(11) DEFAULT NULL,
`traffic_increase` int(11) DEFAULT NULL,
`rank_improvement` int(11) DEFAULT NULL,
`rank_update_date` date DEFAULT NULL,
`top_rank_tab5_id` int(11) DEFAULT NULL,
`frequency` int(10) DEFAULT '1',
`score` float DEFAULT NULL,
`create_date` datetime DEFAULT NULL,
`bing_rank1` int(10) DEFAULT NULL,
`bing_rank2` int(10) DEFAULT NULL,
`yesterday_bing_entrances` int(11) DEFAULT NULL,
`bing_rank_improvement` int(11) DEFAULT NULL,
KEY `id` (`id`),
KEY `tab4_name` (`tab4_name`),
KEY `own_domain_id` (`own_domain_id`,`rank_check`),
KEY `rank_check` (`rank_check`)
) ENGINE=InnoDB AUTO_INCREMENT=670267018 DEFAULT CHARSET=utf8
/*!50100 PARTITION BY RANGE (`rank_check`)
(PARTITION p0 VALUES LESS THAN (0) ENGINE = InnoDB,
PARTITION p1 VALUES LESS THAN (1) ENGINE = InnoDB,
PARTITION p2 VALUES LESS THAN (2) ENGINE = InnoDB,
PARTITION pEOW VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */
1 row in set (0.00 sec)
mysql> show create table t_tab7\G
*************************** 1. row ***************************
Table: t_tab7
Create Table: CREATE TABLE `t_tab7` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`own_domain_id` int(11) DEFAULT NULL,
`url` varchar(2000) NOT NULL,
`create_date` datetime DEFAULT NULL,
`friendly_name` varchar(255) DEFAULT NULL,
`section_name_id` int(11) DEFAULT NULL,
`type` int(11) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`week_entrances` int(11) DEFAULT NULL,
`week_bounces` int(11) DEFAULT NULL,
`canonical_url_id` int(11) DEFAULT NULL,
KEY `id` (`id`),
KEY `urlindex` (`url`(255)),
KEY `own_domain_id_type_status` (`own_domain_id`,`type`,`status`),
KEY `canonical_url_id` (`canonical_url_id`),
KEY `type` (`type`,`status`)
) ENGINE=InnoDB AUTO_INCREMENT=237034388 DEFAULT CHARSET=utf8
/*!50100 PARTITION BY RANGE (`type`)
(PARTITION p0 VALUES LESS THAN (0) ENGINE = InnoDB,
PARTITION p1 VALUES LESS THAN (1) ENGINE = InnoDB,
PARTITION p2 VALUES LESS THAN (2) ENGINE = InnoDB,
PARTITION pEOW VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */
1 row in set (0.01 sec)

Try below
select count(*)
from (Select id,create_tab3_id from tab6 where domain_id = 522) as tab1
left join tab6_element_tab2 tab2
on tab2.tab1nt_id = tab1.id
left join t_tab3 tab3
on tab1.create_tab3_id = tab3.id
left join t_tab4 tab4
on tab2.element_id = tab4.id and tab2.element_type_id = 1
left join t_tab7 tab5
on tab2.element_id = tab5.id and tab2.element_type_id = 2
Remove left join t_tab3 tab3 if you are using for count only and if possible add domain type check in where clause

Define an index on the column used in your condition:
create index t_tab6_domain_id_index on t_tab6(domain_id);
The query itself seems fine.

Related

How to optimize MySQL query with large data on left join?

The query below returns a set of User and for each row a number of relations from the user perpective who is searching (id = 4)
SELECT `users`.`firstname` AS firstname,
`users`.`lastname` AS lastname,
COUNT(`trusted_users`.`id`) AS number_of_friend_in_common,
CASE ... AS friend,
CASE ... AS facebook_invitable,
CASE ... AS address_book_invitable,
CASE ... AS virtual_user,
FROM `users`
LEFT OUTER JOIN `trusted_users`
ON `trusted_users`.`user_id` = 4 AND `trusted_users`.`trust_user_id` = `users`.`id`
LEFT OUTER JOIN `facebook_friends`
ON (`facebook_friends`.`user_id` = 4 AND `facebook_friends`.`friend_user_id` = `users`.`id`
OR `facebook_friends`.`user_id` = `users`.`id` AND `facebook_friends`.`friend_user_id` = 4)
LEFT OUTER JOIN `address_book_contacts`
ON `address_book_contacts`.`owner_id` = 4 AND `address_book_contacts`.`email_digest` = `users`.`email_digest`
LEFT OUTER JOIN `friends`
ON (`friends`.`me_id` = `users`.`id` AND `friends`.`him_id` = 4
OR `friends`.`me_id` = 4 AND `friends`.`him_id` = `users`.`id`)
WHERE `users`.`id` NOT IN
(SELECT CASE
WHEN `friends`.`me_id` = 4 THEN `friends`.`him_id`
ELSE `friends`.`me_id`
END
FROM `friends`
WHERE (`friends`.`status` = 0
AND `friends`.`him_id` = 4
AND `friends`.`him_status` = 7
OR `friends`.`status` = 0
AND `friends`.`me_id` = 4
AND `friends`.`me_status` = 7))
AND (`users`.`firstname` LIKE '%a%' OR `users`.`lastname` LIKE '%a%')
GROUP BY `users`.`id`
ORDER BY friend DESC,
facebook_invitable DESC,
address_book_invitable DESC,
number_of_friend_in_common DESC,
virtual_user DESC,
firstname,
lastname LIMIT 0, 20
Number of row for each table:
trusted_users: 255k
facebook_friends: 1k
address_book_contacts: 1.5M
friends: 70k
users: 32k
All fields of join are indexed. The query takes 1.1s which is not acceptable for the amount of data we have.
What am I doing wrong? Should I split in multiple query and paginate my self?
Edit 1: EXPLAIN result
+----+-------------+-----------------------+-------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------+---------+----------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+-------------+-----------------------+-------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------+---------+----------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------+
| 1 | PRIMARY | users | ALL | PRIMARY,index_users_on_chat_id,index_users_login_facebook_id,index_users_on_login,index_users_on_parent_id,index_users_account_type,index_users_email_digest,index_users_id | NULL | NULL | NULL | 31847 | Using where; Using temporary; Using filesort |
| 1 | PRIMARY | trusted_users | ref | index_trusted_users_user,index_trusted_users_trust_user | index_trusted_users_trust_user | 5 | messenger_dev.users.id | 6 | Using where |
| 1 | PRIMARY | facebook_friends | index_merge | index_facebook_friends_user,index_facebook_friends_friend | index_facebook_friends_user,index_facebook_friends_friend | 5,5 | NULL | 2 | Using union(index_facebook_friends_user,index_facebook_friends_friend); Using where; Using join buffer (Block Nested Loop) |
| 1 | PRIMARY | address_book_contacts | ref | index_address_book_contacts_owner_id,index_address_book_contacts_email | index_address_book_contacts_email | 767 | messenger_dev.users.email_digest | 1 | Using where |
| 1 | PRIMARY | friends | index_merge | index_friends_me_him,index_friends_me,index_friends_him | index_friends_him,index_friends_me | 5,5 | NULL | 18 | Using union(index_friends_him,index_friends_me); Using where; Using join buffer (Block Nested Loop) |
| 2 | SUBQUERY | friends | index_merge | index_friends_me_him,index_friends_me,index_friends_him | index_friends_him,index_friends_me | 5,5 | NULL | 18 | Using union(index_friends_him,index_friends_me); Using where |
+----+-------------+-----------------------+-------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------+---------+----------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------+
6 rows in set (0,00 sec)
Edit 2: Table structure
CREATE TABLE `address_book_contacts` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`email_digest` varchar(191) DEFAULT NULL,
`code` varchar(191) DEFAULT NULL,
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
`owner_id` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `index_address_book_contacts_owner_id` (`owner_id`),
KEY `index_address_book_contacts_email` (`email_digest`)
) ENGINE=InnoDB AUTO_INCREMENT=1598109 DEFAULT CHARSET=utf8mb4;
CREATE TABLE `trusted_users` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`friend_id` int(11) DEFAULT NULL,
`user_id` int(11) DEFAULT NULL,
`trust_user_id` int(11) DEFAULT NULL,
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
PRIMARY KEY (`id`),
KEY `index_trusted_users_on_friend_id` (`friend_id`),
KEY `index_trusted_users_user` (`user_id`),
KEY `index_trusted_users_trust_user` (`trust_user_id`),
CONSTRAINT `fk_rails_007c31c802` FOREIGN KEY (`trust_user_id`) REFERENCES `users` (`id`),
CONSTRAINT `fk_rails_ca24cb4e23` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=275576 DEFAULT CHARSET=utf8mb4;
CREATE TABLE `facebook_friends` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`user_id` int(11) DEFAULT NULL,
`friend_user_id` int(11) DEFAULT NULL,
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
`code` varchar(191) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `index_facebook_friends_user` (`user_id`),
KEY `index_facebook_friends_friend` (`friend_user_id`),
KEY `index_facebook_friends_code` (`code`(5)),
CONSTRAINT `fk_rails_78285a074e` FOREIGN KEY (`friend_user_id`) REFERENCES `users` (`id`),
CONSTRAINT `fk_rails_aa3ac53a81` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1149 DEFAULT CHARSET=utf8mb4;
CREATE TABLE `friends` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`me_id` int(11) DEFAULT NULL,
`him_id` int(11) DEFAULT NULL,
`owner_id` int(11) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
`me_status` int(11) DEFAULT '0',
`him_status` int(11) DEFAULT '0',
PRIMARY KEY (`id`),
UNIQUE KEY `index_friends_me_him` (`me_id`,`him_id`),
KEY `index_friends_me` (`me_id`),
KEY `index_friends_him` (`him_id`),
KEY `index_friends_owner` (`owner_id`),
CONSTRAINT `fk_rails_9fa3474d31` FOREIGN KEY (`owner_id`) REFERENCES `users` (`id`),
CONSTRAINT `fk_rails_d3ebb6657f` FOREIGN KEY (`him_id`) REFERENCES `users` (`id`),
CONSTRAINT `fk_rails_fccfd1b821` FOREIGN KEY (`me_id`) REFERENCES `users` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=95724 DEFAULT CHARSET=utf8mb4;
CREATE TABLE `users` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`account_type` varchar(191) NOT NULL,
`firstname` varchar(191) DEFAULT NULL,
`lastname` varchar(191) DEFAULT NULL,
`login` varchar(191) DEFAULT NULL,
`avatar` varchar(191) DEFAULT NULL,
`gender` varchar(1) DEFAULT NULL,
`locale` varchar(191) DEFAULT NULL,
`birthdate` date DEFAULT NULL,
`password_digest` varchar(191) DEFAULT NULL,
`email_digest` varchar(191) DEFAULT NULL,
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
PRIMARY KEY (`id`),
KEY `index_users_on_login` (`login`),
KEY `index_users_account_type` (`account_type`),
KEY `index_users_email_digest` (`email_digest`),
KEY `index_uses_firstname` (`firstname`),
KEY `index_users_lastname` (`lastname`)
) ENGINE=InnoDB AUTO_INCREMENT=32516 DEFAULT CHARSET=utf8mb4;
It looks like MySQL is not choosing any of the available indicies for the users table.
First, run ANALYZE TABLE users;, then re-run the EXPLAIN command. Is the value of the first rows cell now substantially lower than 31847? If so, your problem should be solved!
If not, run OPTIMIZE TABLE users;, then re-run the EXPLAIN command. Is the value of the first rows cell now substantially lower than 31847? If so, your problem should be solved!
If neither of those steps help, try adding USE INDEX (PRIMARY) or USE INDEX (users_id) immediately after the FROM users portion of your query.
Hope this helps!

Generating auto incrementing numbers with two columns

I want to create a table so that I have an ID number that is based on the date, AND an ID as a unique, primary key.
Ie:
2015-2-1-1
2015-2-1-2
but, if I create:
2015-2-2-1
the counter should restart at 1.
I've tried using the following:
CREATE TABLE `invoices` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`date` date NOT NULL,
`po_id` int(11) unsigned DEFAULT NULL,
`description` varchar(256) NOT NULL,
`client_id` int(11) unsigned DEFAULT NULL,
`status` enum('unpaid','paid','partial') DEFAULT 'unpaid',
PRIMARY KEY (`id`,`date`),
UNIQUE KEY `id_UNIQUE` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=latin1;
But it doesn't work like I want.
If you want a two column primary key (date, id) and id that should auto increment starting from 1 each date. Just set date, for your primary column key, and id in second position in your key definition.
CREATE TABLE `invoices` (
`date` date NOT NULL,
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`po_id` int(11) unsigned DEFAULT NULL,
`description` varchar(256) NOT NULL,
`client_id` int(11) unsigned DEFAULT NULL,
`status` enum('unpaid','paid','partial') DEFAULT 'unpaid',
PRIMARY KEY (`date`, `id`)
) ENGINE=MyIsam DEFAULT CHARSET=latin1;
The internal key will be set as your example, but you will see two columns:
+----+------------+
| id | date |
+----+------------+
| 1 | 2012-01-01 |
| 2 | 2012-01-01 |
| 1 | 2012-01-02 |
| 2 | 2012-01-02 |
+----+------------+
EDIT: You have to use MyIsam format

MySQL query performance slow

I am still new to SQL and I am trying to improve the performance of my query. I have been searching around and have come to the conclusion that using JOINS instead of so many WHERE INS would help improve my performance, but I am unsure of how I would convert my statement. This is my current statement.
SELECT stop_id, stop_name FROM stops WHERE stop_id IN (
SELECT DISTINCT stop_id FROM stop_times WHERE trip_id IN (
SELECT trip_id from trips WHERE route_id = <routeid> ));
It takes anywhere from 5-25 seconds to return the results which is unacceptable. I was hoping to get it below 1 second. If anyone was wondering the data is from a GTFS feed. The stops and trips tables have about ~10,000 rows each, while the stop_times table has ~900,000. I have created indexes at each of the columns I am using. Here is the output of EXPLAIN, and also what was used to create each table.
Thanks for any help and if you need any more info let me know!
+----+--------------------+------------+-----------------+------------------+---------+---------+------+------+-------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+--------------------+------------+-----------------+------------------+---------+---------+------+------+-------------+
| 1 | PRIMARY | stops | ALL | NULL | NULL | NULL | NULL | 6481 | Using where |
| 2 | DEPENDENT SUBQUERY | stop_times | index_subquery | stop_id | stop_id | 63 | func | 63 | Using where |
| 3 | DEPENDENT SUBQUERY | trips | unique_subquery | PRIMARY,route_id | PRIMARY | 62 | func | 1 | Using where |
+----+--------------------+------------+-----------------+------------------+---------+---------+------+------+-------------+
| stops | CREATE TABLE `stops` (
`stop_id` varchar(20) NOT NULL,
`stop_code` varchar(50) DEFAULT NULL,
`stop_name` varchar(255) DEFAULT NULL,
`stop_desc` varchar(255) DEFAULT NULL,
`stop_lat` decimal(8,6) DEFAULT NULL,
`stop_lon` decimal(8,6) DEFAULT NULL,
`zone_id` int(11) DEFAULT NULL,
`stop_url` varchar(255) DEFAULT NULL,
`location_type` int(2) DEFAULT NULL,
`parent_station` int(11) DEFAULT NULL,
`wheelchair_boarding` int(2) DEFAULT NULL,
PRIMARY KEY (`stop_id`),
KEY `zone_id` (`zone_id`),
KEY `stop_lat` (`stop_lat`),
KEY `stop_lon` (`stop_lon`),
KEY `location_type` (`location_type`),
KEY `parent_station` (`parent_station`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 |
| stop_times | CREATE TABLE `stop_times` (
`trip_id` varchar(20) DEFAULT NULL,
`arrival_time` varchar(8) DEFAULT NULL,
`arrival_time_seconds` int(11) DEFAULT NULL,
`departure_time` varchar(8) DEFAULT NULL,
`departure_time_seconds` int(11) DEFAULT NULL,
`stop_id` varchar(20) DEFAULT NULL,
`stop_sequence` int(11) DEFAULT NULL,
`stop_headsign` varchar(50) DEFAULT NULL,
`pickup_type` int(2) DEFAULT NULL,
`drop_off_type` int(2) DEFAULT NULL,
`shape_dist_traveled` varchar(50) DEFAULT NULL,
KEY `trip_id` (`trip_id`),
KEY `arrival_time_seconds` (`arrival_time_seconds`),
KEY `departure_time_seconds` (`departure_time_seconds`),
KEY `stop_id` (`stop_id`),
KEY `stop_sequence` (`stop_sequence`),
KEY `pickup_type` (`pickup_type`),
KEY `drop_off_type` (`drop_off_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 |
| trips | CREATE TABLE `trips` (
`route_id` varchar(20) DEFAULT NULL,
`service_id` varchar(20) DEFAULT NULL,
`trip_id` varchar(20) NOT NULL,
`trip_headsign` varchar(255) DEFAULT NULL,
`trip_short_name` varchar(255) DEFAULT NULL,
`direction_id` tinyint(1) DEFAULT NULL,
`block_id` int(11) DEFAULT NULL,
`shape_id` varchar(50) DEFAULT NULL,
PRIMARY KEY (`trip_id`),
KEY `route_id` (`route_id`),
KEY `service_id` (`service_id`),
KEY `direction_id` (`direction_id`),
KEY `block_id` (`block_id`),
KEY `shape_id` (`shape_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 |
You're right in thinking that JOINS are usually faster than WHERE IN subqueries.
Try this:
SELECT T3.stop_id, T3.stop_name
FROM trips AS T1
JOIN
stop_times AS T2
ON T1.trip_id=T2.trip_id AND route_id = <routeid>
JOIN stops AS T3
ON T2.stop_id=T3.stop_id
GROUP BY T3.stop_id, T3.stop_name

Why is this simple query taking a long time to execute and performing full table scan?

simple query is taking long time around 630 sec to execute
performing full table scan.
please help me to rewrite the query and also suggest me if any indexes need to add.
Query:
mysql> explain SELECT count(DISTINCT(tab1.idnum)) as totalresults FROM (`tab1`) LEFT JOIN `tab2` ON tab2.idnum = tab1.col1id WHERE tab1.userid = '165258' AND `result` = 'correct' AND tab2.department = 'DEPT1' AND tab2.book = 2096 AND `quarantined` = 0;
Explain Plan:
+----+-------------+--------------+--------+------------------------------------+---------+---------+---------------------------+-------+-------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+-------------+--------------+--------+------------------------------------+---------+---------+---------------------------+-------+-------------+
| 1 | SIMPLE | tab1 | ref | userid,col1id,result,userid_status | userid | 4 | const | 14720 | Using where |
| 1 | SIMPLE | tab2 | eq_ref | PRIMARY | PRIMARY | 4 | comp1.tab1.col1id | 1 | Using where |
+----+-------------+--------------+--------+------------------------------------+---------+---------+---------------------------+-------+-------------+
2 rows in set (0.00 sec)
Table structutre:
mysql> show create table tab1\G
*************************** 1. row ***************************
Table: tab1
Create Table: CREATE TABLE `tab1` (
`idnum` int(11) NOT NULL AUTO_INCREMENT,
`questid` int(11) NOT NULL DEFAULT '0',
`userid` int(11) NOT NULL DEFAULT '0',
`col1id` int(11) NOT NULL DEFAULT '0',
`result` enum('correct','incorrect') NOT NULL DEFAULT 'def1',
`answergiven` varchar(35) NOT NULL DEFAULT '0',
`timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`status` enum('calibrating','normal') NOT NULL DEFAULT 'def2',
`quarantined` tinyint(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`idnum`),
KEY `questid` (`questid`),
KEY `userid` (`userid`),
KEY `col1id` (`col1id`),
KEY `result` (`result`),
KEY `userid_status` (`userid`,`status`),
KEY `questid_status` (`questid`,`status`),
KEY `timestamp` (`timestamp`)
) ENGINE=InnoDB AUTO_INCREMENT=143018786 DEFAULT CHARSET=latin1
1 row in set (0.00 sec)
mysql> show create table tab2\G
*************************** 1. row ***************************
Table: tab2
Create Table: CREATE TABLE `tab2` (
`idnum` int(11) NOT NULL AUTO_INCREMENT,
`userid` int(11) NOT NULL DEFAULT '0',
`timestarted` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`timefinished` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`questionlist` mediumtext NOT NULL,
`topics` mediumtext NOT NULL,
`totalnum` int(11) NOT NULL DEFAULT '0',
`completednum` int(11) NOT NULL DEFAULT '0',
`assignment` int(11) NOT NULL DEFAULT '0',
`department` varchar(255) NOT NULL DEFAULT '',
`book` int(11) NOT NULL DEFAULT '0',
`cqs` mediumtext NOT NULL,
`metatype` varchar(25) DEFAULT 'topic',
PRIMARY KEY (`idnum`),
KEY `userid` (`userid`),
KEY `assignment` (`assignment`)
) ENGINE=InnoDB AUTO_INCREMENT=13547403 DEFAULT CHARSET=latin1
1 row in set (0.00 sec)
You need a multi-column index, ie. one that spans userid AND col1id in tab1 at the same time. Try:
create index idx_usr_col1 on tab1(userid,col1id)
Try adding multi-column indices for MySQL to use:
ALTER TABLE `tab2` ADD INDEX idx_col1id_userid_result_quarantined (`col1id`, `userid`, `result`, `quarantined`);
ALTER TABLE `tab1` ADD INDEX idx_department_book (`department`, `book`);
Try this:
SELECT count(distinct result.idnum)) as totalresults FROM `tab1` as result
inner join (select distinct idnum from `tab2` where department = 'DEPT1' and book = 2096 ) as col11 ON col11.idnum = result.col1id
WHERE result.userid = '165258' AND `result` = 'correct' AND `quarantined` = 0;

too much time to execute also impact on db performance

1.query is taking around 10mins to 1 hr to execute.
2.due to this we are getting load spikes frequently.
3.please help me to rewrite the query.
4.also help me to improve the performance of query.
Query with explain plan
explain select count(*)
from t_event eve
left join t_event_element_rel rel on rel.event_id = eve.id
left join t_object object on eve.create_object_id = object.id
left join t_keyword keyword on rel.element_id = keyword.id and rel.element_type_id = 1
left join t_target_url targeturl on rel.element_id = targeturl.id and rel.element_type_id = 2
where eve.domain_id = 522
+----+-------------+-----------+--------+-------------------------+-------------------------+---------+----------------------------+--------+-------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+-------------+-----------+--------+-------------------------+-------------------------+---------+----------------------------+--------+-------------+
| 1 | SIMPLE | eve | ref | domain_id_event_type_cd | domain_id_event_type_cd | 4 | const | 243430 | |
| 1 | SIMPLE | rel | ref | FK_event | FK_event | 4 | company.eve.id | 2 | |
| 1 | SIMPLE | user | eq_ref | PRIMARY | PRIMARY | 4 | company.eve.create_user_id | 1 | Using index |
| 1 | SIMPLE | keyword | ref | id | id | 4 | company.rel.element_id | 1 | Using index |
| 1 | SIMPLE | targeturl | ref | id | id | 4 | company.rel.element_id | 1 | Using index |
+----+-------------+-----------+--------+-------------------------+-------------------------+---------+----------------------------+--------+-------------+
5 rows in set (0.45 sec)
Table structure:
mysql> show create table t_event_element_rel\G
*************************** 1. row ***************************
Table: t_event_element_rel
Create Table: CREATE TABLE `t_event_element_rel` (
`id` int(255) NOT NULL AUTO_INCREMENT,
`element_type_id` int(11) NOT NULL,
`event_id` int(11) NOT NULL,
`element_id` int(11) NOT NULL,
`element_desc` varchar(500) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `FK_event` (`event_id`),
KEY `element_type_id_element_id` (`element_type_id`,`element_id`)
) ENGINE=InnoDB AUTO_INCREMENT=45901159 DEFAULT CHARSET=utf8
1 row in set (0.00 sec)
mysql> show create table t_object\G
*************************** 1. row ***************************
Table: t_object
Create Table: CREATE TABLE `t_object` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL,
`password` varchar(255) DEFAULT NULL,
`email` varchar(255) DEFAULT NULL,
`is_admin` int(11) DEFAULT NULL,
`phone` varchar(50) DEFAULT NULL,
`company` varchar(255) DEFAULT NULL,
`last_login` datetime DEFAULT NULL,
`state` int(11) DEFAULT NULL,
`validate_code` varchar(50) DEFAULT NULL,
`has_login` smallint(6) DEFAULT NULL COMMENT 'if null or 0 then hasn''t login, first time login',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1304 DEFAULT CHARSET=utf8
1 row in set (0.00 sec)
mysql> show create table t_event\G
*************************** 1. row ***************************
Table: t_event
Create Table: CREATE TABLE `t_event` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`create_user_id` int(11) DEFAULT NULL,
`event_create_date` date DEFAULT NULL,
`event_type_cd` int(11) NOT NULL,
`event_desc` varchar(512) NOT NULL,
`IsGlobalEvent` int(2) DEFAULT NULL,
`event_start_date` datetime NOT NULL,
`event_end_date` datetime NOT NULL,
`job_id` int(11) DEFAULT NULL,
`domain_id` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `event_user_FK` (`create_user_id`),
KEY `domain_id_event_type_cd` (`domain_id`,`event_type_cd`)
) ENGINE=InnoDB AUTO_INCREMENT=8586007 DEFAULT CHARSET=utf8
1 row in set (0.00 sec)
mysql> show create table t_keyword\G
*************************** 1. row ***************************
Table: t_keyword
Create Table: CREATE TABLE `t_keyword` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`keyword_name` varchar(255) DEFAULT NULL,
`keyword_value` varchar(255) DEFAULT NULL,
`type` int(11) DEFAULT NULL,
`description` varchar(2000) DEFAULT NULL,
`own_domain_id` int(11) DEFAULT NULL,
`rank_check` int(11) DEFAULT NULL,
`rank1` int(11) DEFAULT NULL COMMENT 'yesterday rank value',
`rank2` int(11) DEFAULT NULL COMMENT 'the day before yesterday rank value',
`rank3` int(11) DEFAULT NULL COMMENT 'special date rank for overstock.com',
`yesterday_entrances` int(11) DEFAULT NULL COMMENT 'yesterday entrances',
`week_entrances` int(11) DEFAULT NULL COMMENT '7 days entrances',
`current_ctr` float(16,4) DEFAULT NULL COMMENT 'Current CTR',
`monthly_search_volume` int(11) DEFAULT NULL COMMENT 'Most Recent Month search volume',
`avg_monthly_search_volume` int(11) DEFAULT NULL COMMENT 'avg_monthly_search_volume',
`traffic_increase` int(11) DEFAULT NULL COMMENT 'Traffic Increase',
`rank_improvement` int(11) DEFAULT NULL COMMENT 'Rank Improvement',
`rank_update_date` date DEFAULT NULL COMMENT 'rank be updated for Special Date',
`top_rank_targeturl_id` int(11) DEFAULT NULL,
`frequency` int(10) DEFAULT '1' COMMENT '1: daily, 2: weekly, 3: monthly',
`score` float DEFAULT NULL,
`create_date` datetime DEFAULT NULL,
`bing_rank1` int(10) DEFAULT NULL,
`bing_rank2` int(10) DEFAULT NULL,
`yesterday_bing_entrances` int(11) DEFAULT NULL,
`bing_rank_improvement` int(11) DEFAULT NULL,
KEY `id` (`id`),
KEY `keyword_name` (`keyword_name`),
KEY `own_domain_id` (`own_domain_id`,`rank_check`),
KEY `rank_check` (`rank_check`)
) ENGINE=InnoDB AUTO_INCREMENT=670267018 DEFAULT CHARSET=utf8
/*!50100 PARTITION BY RANGE (`rank_check`)
(PARTITION p0 VALUES LESS THAN (0) ENGINE = InnoDB,
PARTITION p1 VALUES LESS THAN (1) ENGINE = InnoDB,
PARTITION p2 VALUES LESS THAN (2) ENGINE = InnoDB,
PARTITION pEOW VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */
1 row in set (0.00 sec)
mysql> show create table t_target_url\G
*************************** 1. row ***************************
Table: t_target_url
Create Table: CREATE TABLE `t_target_url` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`own_domain_id` int(11) DEFAULT NULL,
`url` varchar(2000) NOT NULL,
`create_date` datetime DEFAULT NULL,
`friendly_name` varchar(255) DEFAULT NULL,
`section_name_id` int(11) DEFAULT NULL,
`type` int(11) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`week_entrances` int(11) DEFAULT NULL COMMENT 'last 7 days entrances',
`week_bounces` int(11) DEFAULT NULL COMMENT 'last 7 days bounce',
`canonical_url_id` int(11) DEFAULT NULL COMMENT 'the primary URL ID, NOT allow canonical of canonical',
KEY `id` (`id`),
KEY `urlindex` (`url`(255)),
KEY `own_domain_id_type_status` (`own_domain_id`,`type`,`status`),
KEY `canonical_url_id` (`canonical_url_id`),
KEY `type` (`type`,`status`)
) ENGINE=InnoDB AUTO_INCREMENT=237034388 DEFAULT CHARSET=utf8
/*!50100 PARTITION BY RANGE (`type`)
(PARTITION p0 VALUES LESS THAN (0) ENGINE = InnoDB,
PARTITION p1 VALUES LESS THAN (1) ENGINE = InnoDB,
PARTITION p2 VALUES LESS THAN (2) ENGINE = InnoDB,
PARTITION pEOW VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */
1 row in set (0.01 sec)
The first line of your EXPLAIN shows a lot of rows for domain_id_event_type_cd, which is a compound index, yet in your query you only search on a part of the index (eve.domain_id = xxx). So unless domain 522 really has 243K entries, you can cut down on this by adding an index solely for domain_id.
Also, it seems you are trying to do an OR search by using rel.element_type_id as a "switch" for referencing different types of data. Since you are only interested in the count, I suggest you split the query in two parts, then add the results. This will cut down on the LEFT JOINS. You can even add the results within MySQL:
SELECT (SELECT COUNT (*) FROM ....) + (SELECT COUNT (*) FROM ...)
(You do know you can abuse SELECT to do math, don't you?)