I have some existing Mysql query and just wondering how to IMPROVE it. Because it's take sometimes up to 20s to execute.
Well in fact it's take up to 0.3690s to find right records but then when need to get 40k record is take up to 20s .
So my question is how can I improve my settings or my sql code to get records faster? Or it's depend now only on my machine (such SAS hard drive) ?
First some necessary info:
my application use MySQL server 5.6 and InnoDB Engine
my custom settings:
innodb_buffer_pool_size = 7G
innodb_log_buffer_size = 64M
innodb_log_file_size = 2G
innodb_flush_log_at_trx_commit = 0
innodb_write_io_threads = 32
join_buffer_size = 32M
tmp_table_size = 128M
max_heap_table_size = 128M
sort_buffer_size = 128M
table_open_cache = 4000
bulk_insert_buffer_size = 256M
Table definitions:
CREATE TABLE `tblusers` (
`user_id` int(11) NOT NULL AUTO_INCREMENT,
`email` varchar(100) NOT NULL,
`password` varchar(50) NOT NULL,
`user_name` varchar(45) NOT NULL,
`phone` varchar(12) NOT NULL,
`machine_id` int(11) NOT NULL DEFAULT '1',
`lang_id` int(11) NOT NULL DEFAULT '14',
`user_type` tinyint(4) NOT NULL DEFAULT '1' ,
`created_on` datetime NOT NULL,
`active_open` tinyint(4) NOT NULL DEFAULT '0' ,
`email_hash` varchar(50) NOT NULL DEFAULT '1',
`profile_approved` tinyint(4) NOT NULL DEFAULT '0',
`menage_data` tinyint(4) NOT NULL DEFAULT '0' ,
`mailing_agree` tinyint(4) NOT NULL DEFAULT '0' ,
`edited` tinyint(4) NOT NULL DEFAULT '0',
`deleted` tinyint(4) NOT NULL DEFAULT '0',
`warnings` tinyint(4) NOT NULL DEFAULT '0',
PRIMARY KEY (`user_id`),
UNIQUE KEY `email_UNIQUE` (`email`),
UNIQUE KEY `user_name_UNIQUEE` (`user_name`),
KEY `fk_tblUsers_hlpLangs1_idx` (`lang_id`),
KEY `email_hash` (`email_hash`),
KEY `trio` (`user_type`,`profile_approved`,`deleted`,`email_hash`),
CONSTRAINT `tblusers_ibfk_1` FOREIGN KEY (`lang_id`) REFERENCES `hlplangs` (`lang_id`) ON DELETE NO ACTION ON UPDATE NO ACTION
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `tblhostess` (
`user_id` int(11) NOT NULL,
`first_name` varchar(45) NOT NULL,
`sure_name` varchar(45) DEFAULT NULL,
`gender` tinyint(4) NOT NULL DEFAULT '0' ,
`dob` datetime NOT NULL,
`driver_license` tinyint(4) NOT NULL DEFAULT '0',
`sanepid` tinyint(4) NOT NULL DEFAULT '0',
`city_id` int(11) NOT NULL,
`province_id` int(11) NOT NULL,
`picture_id` int(11) NOT NULL DEFAULT '1',
`hair_id` int(11) NOT NULL DEFAULT '1',
`hair_color_id` int(11) NOT NULL DEFAULT '1',
`number_of_view` int(11) NOT NULL DEFAULT '0',
`who_can_see` tinyint(4) NOT NULL DEFAULT '0' ,
`complete_register` tinyint(4) NOT NULL DEFAULT '0',
`skin_id` int(11) NOT NULL DEFAULT '1',
`bra_id` int(11) NOT NULL DEFAULT '1',
`wear_id` int(11) NOT NULL DEFAULT '1',
`shoe_id` int(11) NOT NULL DEFAULT '1',
`desc` text,
`height` int(11) NOT NULL DEFAULT '0',
`weight` int(11) NOT NULL DEFAULT '0',
`bust` int(11) NOT NULL DEFAULT '0',
`waist` int(11) NOT NULL DEFAULT '0',
`hip` int(11) NOT NULL DEFAULT '0',
`redirect_url` varchar(255) DEFAULT NULL,
`friend_url` varchar(90) DEFAULT NULL,
`premium` tinyint(4) NOT NULL DEFAULT '0',
`premium_until` datetime DEFAULT NULL,
`work_as_model` tinyint(4) NOT NULL DEFAULT '0',
`work_as_hostess` tinyint(4) NOT NULL DEFAULT '1',
`work_as_fotomodel` tinyint(4) NOT NULL DEFAULT '0',
`work_in_club` tinyint(4) NOT NULL DEFAULT '0',
`work_in_party` tinyint(4) NOT NULL DEFAULT '0',
`work_in_promo` tinyint(4) NOT NULL DEFAULT '0',
`work_in_trade` tinyint(4) NOT NULL DEFAULT '0',
`work_in_event` tinyint(4) NOT NULL DEFAULT '0',
`work_in_gala` tinyint(4) NOT NULL DEFAULT '0',
`phone_ver` tinyint(4) NOT NULL DEFAULT '0',
`cert` tinyint(4) NOT NULL DEFAULT '0',
`fb_premium` tinyint(4) NOT NULL DEFAULT '0' ,
PRIMARY KEY (`user_id`),
KEY `fk_tblHostess_tblCities1_idx` (`city_id`),
KEY `fk_tblHostess_hlpProvinces1_idx` (`province_id`),
KEY `fk_tblHostess_hlpHairColor1_idx` (`hair_color_id`),
KEY `fk_tblHostess_hlpHair1_idx` (`hair_id`),
KEY `fk_tblHostess_hlpShoes1_idx` (`shoe_id`),
KEY `fk_tblHostess_hlpBra1_idx` (`bra_id`),
KEY `fk_tblHostess_hlpWear1_idx` (`wear_id`),
KEY `fk_tblHostess_hlpSkinColor1_idx` (`skin_id`),
KEY `premium` (`premium`),
KEY `num_of_views` (`number_of_view`),
KEY `views_premium` (`number_of_view`,`premium`),
CONSTRAINT `tblhostess_ibfk_1` FOREIGN KEY (`user_id`) REFERENCES `tblusers` (`user_id`) ON DELETE NO ACTION ON UPDATE NO ACTION,
CONSTRAINT `tblhostess_ibfk_2` FOREIGN KEY (`city_id`) REFERENCES `hlpcities` (`city_id`) ON DELETE NO ACTION ON UPDATE NO ACTION,
CONSTRAINT `tblhostess_ibfk_3` FOREIGN KEY (`province_id`) REFERENCES `hlpprovinces` (`province_id`) ON DELETE NO ACTION ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `tblhostessmailings` (
`hostess_id` int(11) NOT NULL,
`new_job_offers` int(11) NOT NULL DEFAULT '2' ,
`comments` int(11) NOT NULL DEFAULT '1' ,
`job_offer_accept` int(11) NOT NULL DEFAULT '1',
`private_message` int(11) NOT NULL DEFAULT '1' ,
`job_offer_sms` int(11) NOT NULL DEFAULT '0' ,
`job_offer_private` int(11) NOT NULL DEFAULT '0' ,
PRIMARY KEY (`hostess_id`),
CONSTRAINT `tblhostessmailings_ibfk_1` FOREIGN KEY (`hostess_id`) REFERENCES `tblhostess` (`user_id`) ON DELETE NO ACTION ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `tbljoboffers` (
`offer_id` int(11) NOT NULL AUTO_INCREMENT,
`employer_id` int(11) NOT NULL,
`subject` varchar(250) NOT NULL,
`content` text NOT NULL,
`content_html` text,
`date_added` datetime NOT NULL,
`active` tinyint(3) unsigned NOT NULL DEFAULT '1' ,
`approved` tinyint(3) unsigned NOT NULL DEFAULT '0',
`edited` tinyint(3) unsigned NOT NULL DEFAULT '0' ,
`email` varchar(100) DEFAULT NULL,
`freqence_id` int(11) NOT NULL DEFAULT '1' ,
`premium` tinyint(3) unsigned NOT NULL DEFAULT '0' ,
`start_date` datetime DEFAULT NULL ,
`end_date` datetime DEFAULT NULL ,
`premium_old_user` tinyint(3) unsigned NOT NULL DEFAULT '0',
`sending` tinyint(3) unsigned NOT NULL DEFAULT '0' ,
`external_sent` tinyint(3) unsigned NOT NULL DEFAULT '0' ,
`internal_sent` tinyint(3) unsigned NOT NULL DEFAULT '0',
`archiwal` tinyint(3) unsigned NOT NULL DEFAULT '0',
`friend_url` varchar(250) DEFAULT NULL,
`deleted` tinyint(3) unsigned NOT NULL DEFAULT '0',
`to_export` tinyint(3) unsigned NOT NULL DEFAULT '0',
`exported` tinyint(3) unsigned NOT NULL DEFAULT '0',
`sms_sent` tinyint(3) unsigned NOT NULL DEFAULT '0',
`sms_sending` tinyint(3) unsigned NOT NULL DEFAULT '0' ,
`private` tinyint(4) NOT NULL DEFAULT '0' ,
`private_paid` tinyint(4) NOT NULL DEFAULT '0' ,
PRIMARY KEY (`offer_id`,`freqence_id`),
KEY `fk_tblJoboffers_tblEmployers1_idx` (`employer_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `tbljobofferslocations` (
`location_id` int(11) NOT NULL AUTO_INCREMENT,
`offer_id` int(11) NOT NULL,
`city_id` int(11) NOT NULL,
`province_id` int(11) NOT NULL,
`ref_code` varchar(100) DEFAULT NULL,
`display_times` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`location_id`),
UNIQUE KEY `offer_id` (`offer_id`,`city_id`,`province_id`),
KEY `fk_tblJobOffersLocations_hlpProvinces1_idx` (`province_id`),
KEY `fk_tblJobOffersLocations_tblCities1_idx` (`city_id`),
KEY `fk_tblJobOffersLocations_tblJobOffers1_idx` (`offer_id`),
CONSTRAINT `tbljobofferslocations_ibfk_1` FOREIGN KEY (`offer_id`) REFERENCES `tbljoboffers` (`offer_id`) ON DELETE NO ACTION ON UPDATE NO ACTION,
CONSTRAINT `tbljobofferslocations_ibfk_2` FOREIGN KEY (`city_id`) REFERENCES `hlpcities` (`city_id`) ON DELETE NO ACTION ON UPDATE NO ACTION,
CONSTRAINT `tbljobofferslocations_ibfk_3` FOREIGN KEY (`province_id`) REFERENCES `hlpprovinces` (`province_id`) ON DELETE NO ACTION ON UPDATE NO ACTION
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
records count approx in db
users 100k
job offers around 20k (all)
and locations around 40k
And finally my sql example:
select email, first_name, sure_name
from tblusers u join tblhostess h on h.user_id = u.user_id
join tblhostessmailings m on m.hostess_id = h.user_id
join
(select province_id from tbljoboffers o join tbljobofferslocations l on l.offer_id = o.offer_id
where o.deleted = 0 and o.start_date > date_add(current_timestamp, INTERVAL -1 DAY) and o.approved = 1 and o.active = 1 and o.internal_sent = 1 and o.private = 0
group by l.province_id) z on z.province_id = h.province_id
where u.deleted = 0 and u.email_hash = '1' and email != '' and user_type = 1 and (m.new_job_offers = 2)
What I wonna to get from here is a list of all users within 1 day who gets at least one offer on their inside mail box
All jobs are divided into province so this
(select province_id from tbljoboffers o join tbljobofferslocations l on l.offer_id = o.offer_id
where o.deleted = 0 and o.start_date > date_add(current_timestamp, INTERVAL -1 DAY) and o.approved = 1 and o.active = 1 and o.internal_sent = 1 and o.private = 0
group by l.province_id)
gets the full list of region and rest get information about users
Sometimes happens i will need retrive 40k records with this query so I really need to improve this.
Thanks in advance.
I would look at the subquery you are joining by itself.
select province_id from tbljoboffers o join tbljobofferslocations l on l.offer_id = o.offer_id
where o.deleted = 0 and o.start_date > date_add(current_timestamp, INTERVAL -1 DAY) and o.approved = 1 and o.active = 1 and o.internal_sent = 1 and o.private = 0
group by l.province_id
The date comparison in the WHERE clause is likely part of the slow down, how long does that subquery take to run on its own? A greater than comparison on a date_add function will likely be slow. But, without changing the query itself you might gain some performance by simply making that subquery into a view and then you can join the view back to the main query.
CREATE VIEW vprovince_ids AS
select province_id from tbljoboffers o join tbljobofferslocations l on l.offer_id = o.offer_id
where o.deleted = 0 and o.start_date > date_add(current_timestamp, INTERVAL -1 DAY) and o.approved = 1 and o.active = 1 and o.internal_sent = 1 and o.private = 0
group by l.province_id
Then your main query would become:
select email, first_name, sure_name
from tblusers u join tblhostess h on h.user_id = u.user_id
join tblhostessmailings m on m.hostess_id = h.user_id
join vprovince_ids z on z.province_id = h.province_id
where u.deleted = 0 and u.email_hash = '1' and email != '' and user_type = 1 and (m.new_job_offers = 2)
Hopefully that helps!
I've slightly restructured the query for readability and to follow the explanation of how I would think / apply indexes to the respective tables.
select
u.email,
u.first_name,
u.sure_name
from
tblusers u
join tblhostess h
on u.user_id = h.user_id
join tblhostessmailings m
on h.user_id = m.hostess_id
and m.new_job_offers = 2
join (
select DISTINCT
l.province_id
from
tbljoboffers o
join tbljobofferslocations l
on o.offer_id = l.offer_id
where
o.deleted = 0
and o.approved = 1
and o.active = 1
and o.internal_sent = 1
and o.private = 0
and o.start_date > date_add( current_timestamp, INTERVAL -1 DAY)
) z
on h.province_id = z.province_id
where
u.deleted = 0
and u.email_hash = '1'
and u.user_type = 1
and u.email != ''
First, your queries would be best to have all columns represent the proper table (or alias) they are coming from so others in the future, or offering help don't have to guess where columns are coming from which tables (yet you did provide the table structures).
To better handle your queries, having your table with a bunch of individual indexes is not the best way to support your queries. Instead, you need to have indexes that match the type of criteria you are searching on. In the case of the inner query of job offers, you are explicitly looking for 6 criteria, AND applying a group by. Having an index on the criteria components will significantly improve. Also, due to the other criteria, I have moved the date basis to the end, both visually, and as part of the index. Since you are not doing any aggregates by province, I removed the group by, and just did DISTINCT.
table create index on...
tblJobOffers ( deleted, approved, active, internal_sent, private, start_date )
Think of the indexes like this. If you had individual indexes on things like deleted or approved or active... Only one of them would be used as basis to help optimize the query. But since multiple fields are used in your query, it would be faster to use those indexes that matched multiple criteria you are running with.
Try to think of it like this... Each index is a box with the content ordered first by the first field, then sub-sorted by the field under it and so forth.
If you index on just "deleted", you have two boxes... one with all deleted stuff, one with NOT deleted. Similar if only an "active" index. But by having a composite index as I have, the engine can jump quickly to the records in question... Seeing your column names, they appear to be "flags" of either on = 1, or off = 0
Deleted = 0
Approved = 0
(ignored, you don't want approved = 0)
Approved = 1
Active = 0
(ignored, you don't want active = 0)
Active = 1
Internal_Sent = 0
(ignored, you don't want internal sent = 0)
Internal_Sent = 1
Private = 0
Start_Date
2014 Dec...
2015 Jan...
2015 Feb...
2015 Feb 4
2015 Feb 5
2015 Feb 6
Private = 1
(ignored, you don't want private = 1)
Deleted = 1
(ignored since you are not even looking for deleted = 1 records)
So, following the tree down, the index can jump directly to the 2015 Feb transactions, get those and ignore everything else. Use that to join to the province locations and it's done.
For your OUTER query, your join TO the "tblhostessmailings" is based on the hostess ID, but also you are only looking for specific flag of new job offers = 2. Have that as a compound index too. Hostess table is joined by the user ID, so that should have an index. And tblUsers has multiple criteria too, so that (in similar context as job offers above) should have a compound index
table index
tblhostessmailings ( hostess_id, new_job_offers )
tblhostess ( user_id )
tblusers ( deleted, user_type, email_hash, email )
See what these sugggestions and revised query do for your performance, and let us know the performance improvement as a result, good or bad.
Indexed columns on performing the search, not on selecting
Use unique indexes
Use short indices
Do not abuse indexes
Related
I have experienced a strange behavior in MySQL 5.7
SELECT orderId
FROM `order`
WHERE vendor = 11 AND status >= 0 AND city = 1
AND created_date >= '2020-12-01' AND created_date <= '2020-12-31'
SELECT userId
FROM `order`
WHERE vendor = 11 AND status >= 0 AND city = 1
AND created_date >= '2020-12-01' AND created_date <= '2020-12-31'
I have a composite index on all the 4 columns
INDEX is vendor_city_date (vendor, city, created_date, status).
But for some columns I get a filter of 50 for others I get a filter of 5.55.
This is the structure of the table it has 50 other columns I have removed those.
CREATE TABLE `order` (
`orderId` int(11) NOT NULL AUTO_INCREMENT,
`vendor` int(11) NOT NULL DEFAULT '1',
`userId` int(11) NOT NULL,
`offer` int(11) NOT NULL DEFAULT '0',
`city` int(11) NOT NULL,
`offset` int(11) NOT NULL DEFAULT '',
`status` tinyint(4) NOT NULL DEFAULT '0',
`payment` tinyint(4) NOT NULL DEFAULT '0',
`created_date` date NOT NULL,
PRIMARY KEY (`orderId`),
KEY `city_id` (`city`),
KEY `user_id` (`userId`),
KEY `status` (`status`),
KEY `vendor` (`vendor`),
KEY `user_status` (`userId`,`status`),
KEY `vendor_city_date` (`vendor`,`city`,`created_date`,`status`) USING BTREE
);
I have an sql query that collects the year to date totals for all products in the inventory. This query runs quickly, returning around 5000 results in a little over a second.
The query used is
SELECT `ITINCODE` as Code, `IT_product_id` as ID,
SUM(IF(YEAR(`ITDATE`) = YEAR(CURDATE() ), IF(ITTYPE = "O",`ITVAL`, -`ITVAL` ), 0) ) as 'YTD_VAL'
FROM `FITEMS`
WHERE (ITTYPE = 'O' OR `ITTYPE` = 'R') AND ITBACKORDER = 'N' AND ITAMNT > 0 AND YEAR(`ITDATE`) >= YEAR(CURDATE() ) -1
GROUP BY `ITINCODE`
ORDER BY YTD_VAL DESC
I want to take the values in YTD_VAL and store them in the actual products table so that they can be used as an ORDER BY on other queries. I added a new field called ytd_val to the products table and then ran
UPDATE products p SET p.ytd_val =
(SELECT SUM(IF(YEAR(`ITDATE`) = YEAR(CURDATE() ), IF(ITTYPE = 'O',`ITVAL`, -`ITVAL` ), 0) ) as 'YTD_VAL'
FROM `FITEMS`
WHERE ITINCODE = p.products_model AND (ITTYPE = 'O' OR `ITTYPE` = 'R') AND ITBACKORDER = 'N' AND ITAMNT > 0 AND YEAR(`ITDATE`) >= YEAR(CURDATE() ) -1
GROUP BY `ITINCODE`
ORDER BY YTD_VAL DESC)
The idea was to run this by cron job each night so that the values were updated to reflect the previous days sales.
However, running this query has taken 10 plus minutes and it still hasn't completed.
ITINCODE in FITEMS table is the same as products_model in the products table.
IT_product_id in the FITEMS table is the same as products_id in the products table.
What can I do to speed up the query? I thought that as the original results query returns quickly enough that simply updating the values on another table would take seconds longer!
Table structure is as follows:
show create table fitems\G;
Create Table: CREATE TABLE `fitems` (
`ITUNIQUEREF` int(11) unsigned NOT NULL,
`ITAMNT` int(11) NOT NULL,
`ITREF` int(11) unsigned NOT NULL,
`ITTYPE` char(1) NOT NULL,
`ITVAL` decimal(10,4) NOT NULL,
`ITVAT` decimal(10,4) NOT NULL,
`ITPRICE` decimal(10,4) NOT NULL,
`ITDATE` date NOT NULL,
`ITBACKORDER` char(1) NOT NULL,
`ITDISC` decimal(10,2) NOT NULL,
`ITORDERREF` int(11) NOT NULL,
`ITTREF` int(11) unsigned NOT NULL,
`ITDATEDLY` date NOT NULL,
`ITINCODE` char(20) NOT NULL,
`IT_product_id` int(11) unsigned NOT NULL,
`ITBUILT` int(11) NOT NULL,
PRIMARY KEY (`ITUNIQUEREF`),
KEY `ITREF` (`ITREF`,`ITTYPE`,`ITDATE`,`ITBACKORDER`,`ITORDERREF`,`ITDATEDLY`,`ITINCODE`,`IT_product_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
.
show create table products\G;
CREATE TABLE `products` (
`products_id` int(11) NOT NULL,
`products_type` int(11) NOT NULL DEFAULT '1',
`products_quantity` float NOT NULL DEFAULT '0',
`products_model` varchar(32) CHARACTER SET utf8 DEFAULT NULL,
`products_image` varchar(64) CHARACTER SET utf8 DEFAULT NULL,
`products_price` decimal(15,4) NOT NULL DEFAULT '0.0000',
`products_group_a_price` decimal(15,4) NOT NULL,
`products_group_b_price` decimal(15,4) NOT NULL,
`products_group_c_price` decimal(15,4) NOT NULL,
`products_group_d_price` decimal(15,4) NOT NULL,
`products_group_e_price` decimal(15,4) NOT NULL,
`products_group_f_price` decimal(15,4) NOT NULL,
`products_group_g_price` decimal(15,4) NOT NULL,
`products_virtual` tinyint(1) NOT NULL DEFAULT '0',
`products_date_added` datetime NOT NULL DEFAULT '0001-01-01 00:00:00',
`products_last_modified` datetime DEFAULT NULL,
`products_date_available` datetime DEFAULT NULL,
`products_weight` float NOT NULL DEFAULT '0',
`products_status` tinyint(1) NOT NULL DEFAULT '0',
`products_tax_class_id` int(11) NOT NULL DEFAULT '0',
`manufacturers_id` int(11) DEFAULT NULL,
`products_ordered` float NOT NULL DEFAULT '0',
`products_quantity_order_min` float NOT NULL DEFAULT '1',
`products_quantity_order_units` float NOT NULL DEFAULT '1',
`products_priced_by_attribute` tinyint(1) NOT NULL DEFAULT '0',
`product_is_free` tinyint(1) NOT NULL DEFAULT '0',
`product_is_call` tinyint(1) NOT NULL DEFAULT '0',
`products_quantity_mixed` tinyint(1) NOT NULL DEFAULT '0',
`product_is_always_free_shipping` tinyint(1) NOT NULL DEFAULT '0',
`products_qty_box_status` tinyint(1) NOT NULL DEFAULT '1',
`products_quantity_order_max` float NOT NULL DEFAULT '0',
`products_sort_order` int(11) NOT NULL DEFAULT '0',
`products_canonical` text COLLATE utf8_unicode_ci NOT NULL,
`products_discount_type` tinyint(1) NOT NULL DEFAULT '0',
`products_discount_type_from` tinyint(1) NOT NULL DEFAULT '0',
`products_price_sorter` decimal(15,4) NOT NULL DEFAULT '0.0000',
`master_categories_id` int(11) NOT NULL DEFAULT '0',
`products_mixed_discount_quantity` tinyint(1) NOT NULL DEFAULT '1',
`metatags_title_status` tinyint(1) NOT NULL DEFAULT '0',
`metatags_products_name_status` tinyint(1) NOT NULL DEFAULT '0',
`metatags_model_status` tinyint(1) NOT NULL DEFAULT '0',
`metatags_price_status` tinyint(1) NOT NULL DEFAULT '0',
`metatags_title_tagline_status` tinyint(1) NOT NULL DEFAULT '0',
`pricing_group` varchar(16) COLLATE utf8_unicode_ci DEFAULT NULL,
`ytd_val` int(20) NOT NULL,
PRIMARY KEY (`products_id`),
KEY `idx_products_date_added_zen` (`products_date_added`),
KEY `idx_products_status_zen` (`products_status`),
KEY `idx_products_date_available_zen` (`products_date_available`),
KEY `idx_products_ordered_zen` (`products_ordered`),
KEY `idx_products_model_zen` (`products_model`),
KEY `idx_products_price_sorter_zen` (`products_price_sorter`),
KEY `idx_master_categories_id_zen` (`master_categories_id`),
KEY `idx_products_sort_order_zen` (`products_sort_order`),
KEY `idx_manufacturers_id_zen` (`manufacturers_id`),
KEY `products_price` (`products_price`),
KEY `products_status_products_price` (`products_status`,`products_price`),
FULLTEXT KEY `idx_enhanced_products_model` (`products_model`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
SELECT is always way faster then UPDATE.
To speed up an update:
Look at the indexes on the table you are updating: Are they all needed? If not, remove unneeded ones (I would at least remove idx_products_status_zen since that is also covered by products_status_products_price)
Look at the data model: Can you partition the table you are updating? If so, that will speed up the update since the indexes you are updating will be smaller and thus quicker to update;
Use InnoDB. It is faster;
Do you need to be ACID compliant? If not, then change the settings of InnoDB to speed up the system.
If you are really using MySQL: Switch to MariaDB: It is about 8% faster;
Install monitoring to see where your bottleneck is: IO or CPU: If it is IO on read, then try compression.
If you are satisfied with first query performance.
You can just transform your second query to use INNER JOIN like:
http://sqlfiddle.com/#!9/1028a/1
UPDATE products p
INNER JOIN (
SELECT SUM(IF(YEAR(`ITDATE`) = YEAR(CURDATE() ), IF(ITTYPE = 'O',`ITVAL`, -`ITVAL` ), 0) ) as 'YTD_VAL',
ITINCODE
FROM `FITEMS`
WHERE (ITTYPE = 'O' OR `ITTYPE` = 'R')
AND ITBACKORDER = 'N' AND ITAMNT > 0 AND YEAR(`ITDATE`) >= YEAR(CURDATE() ) -1
GROUP BY `ITINCODE`
) t
ON t.ITINCODE = p.products_model
SET p.ytd_val = t.YTD_VAL
UPDATE And by the way you don't need that ORDER BY YTD_VAL DESC it has no sense in this particular case I guess.
UPDATE 2
UPDATE products p
INNER JOIN (
SELECT SUM(IF(YEAR(`ITDATE`) = YEAR(CURDATE() ), IF(ITTYPE = 'O',`ITVAL`, -`ITVAL` ), 0) ) as 'YTD_VAL',
IT_product_id
FROM `FITEMS`
WHERE (ITTYPE = 'O' OR `ITTYPE` = 'R')
AND ITBACKORDER = 'N' AND ITAMNT > 0 AND YEAR(`ITDATE`) >= YEAR(CURDATE() ) -1
GROUP BY `IT_product_id`
) t
ON t.IT_product_id = p.products_id
SET p.ytd_val = t.YTD_VAL
I am not one to say do it, but consider the case for Covering Indexes. One in particular on table fitems. Those columns are relatively slim for that query. I will spook up one to try on particular columns. We have seen cases where terribly long queries can be completed in snappy time. But no promises. Ah, I see you have some. Was looking at the first edit with the alter tables below it. I will keep looking.
Covering Index
A covering index is one in which the query can be resolved via a quick stroll through the index b-tree, without requiring a lookup into the actual table data page. These are the ultimate nirvana for speed. Percona quick article on it.
Explain
And run the query (the update one) thru Explain and examine the output. See also the article Using Explain to Write Better Mysql Queries.
Note, some of these comments are for those that follows, not necessarily this op. He seems to have his house in order pretty well.
In your subquery, neither the order by nor the group by are necessary, so the update can be written as:
UPDATE products p
SET p.ytd_val = (SELECT SUM(IF(YEAR(`ITDATE`) = YEAR(CURDATE() ), IF(ITTYPE = 'O',`ITVAL`, -`ITVAL` ), 0) )
FROM `FITEMS`
WHERE FITEMS.ITINCODE = p.products_model AND
ITTYPE IN ('O', 'R') AND
ITBACKORDER = 'N' AND
ITAMNT > 0 AND
YEAR(`ITDATE`) >= YEAR(CURDATE() ) - 1
);
For this, you want an index on FITEMS(ITINCODE, ITBACKORDER, ITTYPE, ITAMNT, ITVAL). This might significantly speed your query.
I have been left with a report which returns rows at an awfully slow rate. I feel I need to redo without so many or any sub-queries. But I am having a complete brain freeze to how to attempt this.
I have looked at indexes and the keys are not unique enough, which forces a full table scan time. Is there any way I can pull certain information from the other tables using a separate query, adding that as a variable and use it in the main query. As really the result of this query is only a few lines.
Are there any tips or tricks, that I could use to optimise or correct this SQL statement to speed it up.
(EDIT) I have added some create code for the tables.
SELECT
case when (select count(ag.`PKEY`) - count(ag.`ANSWERTIME`) from acdcallinformation ag
where (ag.`COMPLETED`) = 1 and answertime is null and time(ag.INSTIME) and DATE_FORMAT(DATEOFCALL,'%Y-%m-%d') >= date(now()) and ag.skillid = acdcallinformation.skillid) is null
then 0
else
(select count(ag.`PKEY`) - count(ag.`ANSWERTIME`) from acdcallinformation ag where (ag.`COMPLETED`) = 1
and answertime is null and DATE_FORMAT(DATEOFCALL,'%Y-%m-%d') >= date(now()) and ag.skillid = acdcallinformation.skillid)
end as LostCalls,
case when count(acdcallinformation.idleonqueue) is null then 0 else count(acdcallinformation.idleonqueue) end as CountCallsACD,
case when count(acdcallinformation.`ANSWERTIME`) is null then 0 else count(acdcallinformation.`ANSWERTIME`) end AS acdcallinformation_ANSWERED,
(select skillinfo.skillname from skillinfo where skillinfo.pkey = acdcallinformation.skillid) AS acdcallinformation_SKILLIDTEXT,
(select count(pkey) from acdcallinformation age
where DATE_FORMAT(DATEOFCALL,'%Y-%m-%d') >= date(now()) and age.skillid = acdcallinformation.skillid and (age.`COMPLETED`) = 0 and answertime is null
and SKILLID in (select SKILLID
from
callcenterinformation
where time > (now() - INTERVAL 5 SECOND) and callswaiting > 0)) as Waiting,
-- count(acdcallinformation.`PKEY`) as CallsWaiting,
acdcallinformation.`DATEOFCALL` AS acdcallinformation_DATEOFCALL,
acdcallinformation.`FIRSTRINGONQUEUE` AS acdcallinformation_FIRSTRINGONQUEUE,
case when acdcallinformation.`CONNECTTIME` is null then time('00:00:00') else acdcallinformation.`CONNECTTIME` end AS acdcallinformation_CONNECTTIME,
acdcallinformation.`CALLSTATEBEFOREIDLE` AS acdcallinformation_CALLSTATEBEFOREIDLE,
case when acdcallinformation.`AGENTRINGTIME` is null then time('00:00:00') else acdcallinformation.`AGENTRINGTIME` end AS acdcallinformation_AGENTRINGTIME,
acdcallinformation.`IDLEONQUEUE` AS acdcallinformation_IDLEONQUEUE,
acdcallinformation.`DDI` AS acdcallinformation_DDI,
acdcallinformation.`CLIP` AS acdcallinformation_CLIP,
acdcallinformation.`SKILLID` AS acdcallinformation_SKILLID,
acdcallinformation.`ACTIONTYPE` AS acdcallinformation_ACTIONTYPE,
acdcallinformation.`ACTIONDESTINATION` AS acdcallinformation_ACTIONDESTINATION,
acdcallinformation.`COMPLETED` AS acdcallinformation_COMPLETED,
acdcallinformation.`HANDLED` AS acdcallinformation_HANDLED,
acdcallinformation.`CONFIRMED` AS acdcallinformation_CONFIRMED,
(
SELECT
cal.`AGENTSREADY` AS callcenterinformation_AGENTSREADY
FROM
`callcenterinformation` cal
WHERE cal.skillid <> 1 and acdcallinformation.skillid = skillid order by pkey desc limit 1,1) as agentsready
FROM
`acdcallinformation` acdcallinformation
where DATE_FORMAT(DATEOFCALL,'%Y-%m-%d') >= date(now()- interval 1 day )
group by (select skillinfo.skillname from skillinfo where skillinfo.pkey = acdcallinformation.skillid);
CREATE TABLE `callcenterinformation` (
`INSTIME` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`PKEY` INT(11) NOT NULL,
`SKILLID` INT(11) NULL DEFAULT '0',
`DATE` DATE NULL DEFAULT NULL,
`TIME` TIME NULL DEFAULT NULL,
`AGENTSLOGGEDIN` INT(11) NULL DEFAULT '0',
`AGENTSREADY` INT(11) NULL DEFAULT '0',
`AGENTSRINGING` INT(11) NULL DEFAULT '0',
`AGENTSCONNECTED` INT(11) NULL DEFAULT '0',
`AGENTSINPAUSE` INT(11) NULL DEFAULT '0',
`AGENTSINWRAPUP` INT(11) NULL DEFAULT '0',
`CALLSWAITING` INT(11) NULL DEFAULT '0',
`COMPLETED` TINYINT(1) NULL DEFAULT '0',
`HANDLED` TINYINT(1) NULL DEFAULT '0',
`CONFIRMED` TINYINT(1) NULL DEFAULT '0',
PRIMARY KEY (`PKEY`),
INDEX `DATE` (`DATE`),
INDEX `TIME` (`TIME`),
INDEX `SKILLID` (`SKILLID`)
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB;
CREATE TABLE `acdcallinformation` (
`INSTIME` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`PKEY` INT(11) NOT NULL,
`DATEOFCALL` DATE NULL DEFAULT NULL,
`FIRSTRINGONQUEUE` TIME NULL DEFAULT NULL,
`CONNECTTIME` TIME NULL DEFAULT NULL,
`CALLSTATEBEFOREIDLE` INT(11) NULL DEFAULT '0',
`AGENTRINGTIME` TIME NULL DEFAULT NULL,
`ANSWERTIME` TIME NULL DEFAULT NULL,
`IDLEONQUEUE` TIME NULL DEFAULT NULL,
`DDI` TEXT NULL,
`CLIP` TEXT NULL,
`SKILLID` INT(11) NULL DEFAULT '0',
`ACTIONTYPE` INT(11) NULL DEFAULT '0',
`ACTIONDESTINATION` TEXT NULL,
`COMPLETED` TINYINT(1) NULL DEFAULT '0',
`HANDLED` TINYINT(1) NULL DEFAULT '0',
`CONFIRMED` TINYINT(1) NULL DEFAULT '0',
PRIMARY KEY (`PKEY`),
INDEX `DATEOFCALL` (`DATEOFCALL`),
INDEX `IDLEONQUEUE_HANDLED` (`IDLEONQUEUE`, `HANDLED`),
INDEX `SKILLID` (`SKILLID`)
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB;
CREATE TABLE `skillinfo` (
`INSTIME` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`PKEY` INT(11) NOT NULL,
`SKILLNAME` TEXT NULL,
`CLIP` TEXT NULL,
`WRAPUPTIMELENGTH` INT(11) NULL DEFAULT '0',
`MAXRINGTIMELENGTH` INT(11) NULL DEFAULT '0',
`FORCEDTICKET` TINYINT(1) NULL DEFAULT '0',
`STATEAFTERWRAPUP` INT(11) NULL DEFAULT '0',
`STATEAFTERUNANSWEREDCALL` INT(11) NULL DEFAULT '0',
`ACTIONTYPE` INT(11) NULL DEFAULT '0',
`ACTIONDESTINATION` TEXT NULL,
`DEFLECTAFTERCOURTESY` TINYINT(1) NULL DEFAULT '0',
`MAXOVERALLRINGTIMELENGTH` INT(11) NULL DEFAULT '0',
`AUTOCLIP` TINYINT(1) NULL DEFAULT '0',
`OUTGOINGSETTINGSACTIVE` TINYINT(1) NULL DEFAULT '0',
`NUMPLANIDENTIFIER` INT(11) NULL DEFAULT '0',
`TYPEOFNUMBER` INT(11) NULL DEFAULT '0',
`CLIR` INT(11) NULL DEFAULT '0',
`OUTGOINGROUTEID` INT(11) NULL DEFAULT '0',
`USELASTAGENT` TINYINT(1) NULL DEFAULT '0',
`CLIPROUTINGACTIVE` TINYINT(1) NULL DEFAULT '0',
`USETHRESHOLD` TINYINT(1) NULL DEFAULT '0',
`NORMALLOADTHRESHOLD` INT(11) NULL DEFAULT '0',
`OVERLOADTHRESHOLD` INT(11) NULL DEFAULT '0',
`STATEAFTERFORWARD` INT(11) NULL DEFAULT '0',
`CALLDISTTYPE` INT(11) NULL DEFAULT '0',
`USERGROUPID` INT(11) NULL DEFAULT '0',
`EXTERNALCONTROL` TINYINT(1) NULL DEFAULT '0',
`LASTAGENTLIMIT` INT(11) NULL DEFAULT '0',
PRIMARY KEY (`PKEY`)
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB;
Too be honest, there is so much 'wrong' with this query it just isn't fun anymore =/
Some thoughts:
IFNULL() is much more readable than CASE WHEN <field> IS NULL THEN constant ELSE <field> END, especially if turns out to be a sub-query.
AFAIK COUNT(*) will always return 0, even if nothing is found. Thus, there is no need to write an IFNULL() around it
COUNT(field) only counts the non-NULL records for this field, but again, if nothing is found it will return 0, so no need for an IFNULL() around it
You should teach yourself how to JOIN tables as it's (much) better practice than using correlated sub-queries all over the place.
I don't know much about mysql, but it seems to me that you're killing your performance by putting casts and functions around the fields that otherwise seem to have a useful index. I'm pretty sure that due to these constructions the engine simply is not able to use said indexes causing performance to go down the drain. eg. I would try to rewrite
AND DATE_FORMAT(DATEOFCALL,'%Y-%m-%d') >= date(now()) into something like AND DATEOFCALL >= CUR_DATE(), after all, both sides are dates (= numbers)
DATE_FORMAT(DATEOFCALL,'%Y-%m-%d') >= date(now()- interval 1 day) into DATEOFCALL >= date(now()- interval 1 day) for the very same reason
I'm also not sure what time(ag.INSTIME) should do ?!?! Is it true whenever the time is different from 00:00:00 ?
I'm VERY surprised this query actually compiles at all as you seem to GROUP BY on just the skillname, but also fetch quite a lot of other fields from the table (e.g. idleonqueue). From an MSSQL background that should not work.. I guess mysql is different although I do wonder what the result will be like.
Anyway, trying to apply some of the above to your query I end up with below. I doubt it will be 'much faster'; it might be just a bit, but I'd consider it a step forward in your mission to clean it up further...
Good luck!
SELECT (SELECT COUNT(ag.`PKEY`) - COUNT(ag.`ANSWERTIME`)
FROM acdcallinformation ag
WHERE ag.`COMPLETED` = 1
AND answertime is null
AND time(ag.INSTIME)
AND ag.DATEOFCALL >= CURDATE()
AND ag.skillid = info.skillid) AS LostCalls,
COUNT(info.idleonqueue) AS CountCallsACD,
COUNT(info.`ANSWERTIME`) AS acdcallinformation_ANSWERED,
skillinfo.skillname AS acdcallinformation_SKILLIDTEXT,
(SELECT COUNT(pkey)
FROM acdcallinformation age
WHERE age.DATEOFCALL >= CURDATE()
AND age.skillid = info.skillid
AND age.`COMPLETED` = 0
AND age.answertime is null
AND age.SKILLID IN (SELECT SKILLID
FROM callcenterinformation cci
WHERE cci.time > (now() - INTERVAL 5 SECOND)
AND cci.callswaiting > 0)) AS Waiting,
-- count(info.`PKEY`) AS CallsWaiting,
info.`DATEOFCALL` AS acdcallinformation_DATEOFCALL,
info.`FIRSTRINGONQUEUE` AS acdcallinformation_FIRSTRINGONQUEUE,
IFNULL(info.`CONNECTTIME`, time('00:00:00')) AS acdcallinformation_CONNECTTIME,
info.`CALLSTATEBEFOREIDLE` AS acdcallinformation_CALLSTATEBEFOREIDLE,
IFNULL(info.`AGENTRINGTIME`, time('00:00:00')) AS acdcallinformation_AGENTRINGTIME,
info.`IDLEONQUEUE` AS acdcallinformation_IDLEONQUEUE,
info.`DDI` AS acdcallinformation_DDI,
info.`CLIP` AS acdcallinformation_CLIP,
info.`SKILLID` AS acdcallinformation_SKILLID,
info.`ACTIONTYPE` AS acdcallinformation_ACTIONTYPE,
info.`ACTIONDESTINATION` AS acdcallinformation_ACTIONDESTINATION,
info.`COMPLETED` AS acdcallinformation_COMPLETED,
info.`HANDLED` AS acdcallinformation_HANDLED,
info.`CONFIRMED` AS acdcallinformation_CONFIRMED,
(SELECT cal.`AGENTSREADY` AS callcenterinformation_AGENTSREADY
FROM `callcenterinformation` cal
WHERE cal.skillid <> 1
AND cal.skillid = info.skillid
ORDER BY pkey DESC LIMIT 1,1) AS agentsready
FROM `acdcallinformation` info
JOIN `skillinfo`
ON skillinfo.pkey = info.skillid
WHERE info.DATEOFCALL >= (date(now()- interval 1 day ))
GROUP BY skillinfo.skillname ;
I have three tables in a database:
Product table - +100000 entries
Attribute table (list of possible attributes of a product)
Product attribtue table (which contains the value of the attribute of a product)
I am looking for 8 random products and one of their attributes (attribute_id = 2), but if a product hasn't this attribute it should appear at the return of the query. I have been trying some sql queries without any succesful result because my return only shows the products that have the attribute and hide the others.
My three tables are like this:
CREATE TABLE `product` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`sku` varchar(20) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`name` varchar(90) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`provider_id` int(11) unsigned DEFAULT NULL,
`url` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`active` int(1) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
UNIQUE KEY `sku_UNIQUE` (`sku`)
) ENGINE=InnoDB AUTO_INCREMENT=123965 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE `attribute` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL DEFAULT '',
`data_type` varchar(50) DEFAULT '',
PRIMARY KEY (`id`),
) ENGINE=InnoDB AUTO_INCREMENT=30 DEFAULT CHARSET=latin1;
CREATE TABLE `product_attribute` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`product_id` int(11) unsigned NOT NULL,
`attribute_id` int(11) unsigned NOT NULL DEFAULT '6',
`value` longtext NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `product_id` (`product_id`,`attribute_id`)
) ENGINE=InnoDB AUTO_INCREMENT=869437 DEFAULT CHARSET=latin1;
And this is one of the queries I tried, I thought it was correct but it have the same problem as the others...
SELECT product.id, product.sku, product.name,provider.name as provider_name,
product_attribute.value as author
FROM (`product`)
LEFT JOIN `provider` ON `product`.`provider_id` = `provider`.`id`
LEFT JOIN `product_attribute` ON `product`.`id` = `product_attribute`.`product_id`
WHERE `product`.`active` = '1' AND `product`.`url` IS NOT NULL
AND (`product_attribute`.`attribute_id` = 8 OR `product_attribute`.`attribute_id` IS NULL)
AND `product`.`provider_id` = '7' ORDER BY RAND() LIMIT 8
I was trying with left, inner and right join and nothing works.
You should put the condition for the left-joined table in the join, not the where clause
...
from product
left join provider ON product.provider_id = provider.id
left join product_attribute on product.id = product_attribute.product_id
and product_attribute.attribute_id = 8
where `product`.`active` = '1'
and `product`.`url` IS NOT NULL
and `product`.`provider_id` = '7'
...
I've this query, which works fine except it takes a long while (7 seconds, with 40k records in the jobs table, and 700k in the wq table).
I tried an EXPLAIN and it says its looking at all the records in the job table, and not using any of the indexes.
I don't know how to tell MySQL that it should use the jobs.status field to filter the the records before looking up the wq table.
The objective of this, is to get all the records from jobs that have a status != 331, and also any other job which has a wq status of (101, 111, 151).
Query:
SELECT jobs.*
FROM jobs
LEFT OUTER JOIN wq ON (wq.job = jobs.id AND jobs.status IN (341, 331) AND wq.status IN (101, 111, 151))
WHERE ((wq.info is not NULL) or (jobs.status != 331 and ack = 0))
EXPLAIN output:
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE jobs ALL ack,status,status_ack NULL NULL NULL 38111 Using filesort
1 SIMPLE wq ref PRIMARY,job,status PRIMARY 4 cts.jobs.id 20 Using where
Table definitions:
CREATE TABLE jobs ( id int(10) NOT NULL AUTO_INCREMENT,
comment varchar(100) NOT NULL DEFAULT '',
profile varchar(60) NOT NULL DEFAULT '',
start_at int(10) NOT NULL DEFAULT '0',
data text NOT NULL,
status int(10) NOT NULL DEFAULT '0',
info varchar(200) NOT NULL DEFAULT '',
finish int(10) NOT NULL DEFAULT '0',
priority int(5) NOT NULL DEFAULT '0',
ack tinyint(4) NOT NULL DEFAULT '0',
PRIMARY KEY (id),
KEY start_at (start_at),
KEY status (status),
KEY status_ack (status,
ack) ) ENGINE=MyISAM AUTO_INCREMENT=2037530 DEFAULT CHARSET=latin1;
CREATE TABLE wq ( job int(10) NOT NULL DEFAULT '0',
process varchar(60) NOT NULL DEFAULT '',
step varchar(60) NOT NULL DEFAULT '',
status int(10) NOT NULL DEFAULT '0',
run_at int(10) NOT NULL DEFAULT '0',
original_run_at int(10) NOT NULL DEFAULT '0',
info varchar(200) NOT NULL DEFAULT '',
pos int(10) NOT NULL DEFAULT '0',
changed_at int(10) NOT NULL DEFAULT '0',
file varchar(60) NOT NULL DEFAULT '',
PRIMARY KEY (job,
process,
step,
file),
KEY job (job),
KEY status (status) ) ENGINE=MyISAM DEFAULT CHARSET=latin1
Unfortunately mysql (and perhaps any dbms) cannot optimize expressions like jobs.status != 331 and ack = 0 because B-Tree is not a structure that allows to find fast anything that is-not-equal-to-a-constant-value. Thus you'll always get a fullscan.
If there were some better condition like jobs.status = 331 and ack = 0 (note on the fact that i've changed != to =) then it would be an advice to speed up this query:
split the query into 2, joined by UNION ALL
replace in one query LEFT JOIN to INNER JOIN (in the one that implies that wq.info is not NULL)