Related
I have 3 large table around 1 table have 1,000,000 rows other 2 have 500,000 rows.
When I use ORDER BY in query it takes 15 seconds (without ORDER BY 0.001s).
I already try add index on invoice.created_at and invoice.invoice_id
But still take a lot of time.
SELECT
`i`.*,
GROUP_CONCAT(DISTINCT ii.item_id SEPARATOR ',') AS item_id,
GROUP_CONCAT(DISTINCT it.transaction_id SEPARATOR ',') AS transactions,
SUM(DISTINCT IF(ii.status = 1, ii.subtotal, 0)) AS subtotal,
SUM(DISTINCT it.subtotal) AS total_paid_amount,
SUM(DISTINCT it.additional_fee) AS total_additional_fee_amount,
SUM(ii.quantity) AS total_quantity,
(total_amount - SUM(DISTINCT COALESCE(it.amount, 0))) AS total_balance
FROM
`invoices` `i`
LEFT JOIN
`invoices_items` `ii` ON `ii`.`invoice_id` = `i`.`invoice_id`
LEFT JOIN
`invoices_transactions` `it` ON `it`.`invoice_id` = `i`.`invoice_id`
AND `it`.`process_status` = 1
AND `it`.`status` = 1
WHERE
`i`.`status` = '1'
GROUP BY `i`.`invoice_id`
ORDER BY `i`.`created_at` DESC
LIMIT 50
Query with EXPLAIN:
+----+-------------+-------+------------+-------+--------------------------------------------------+------------+---------+------------------------------+--------+----------+----------------------------------------------+
| id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
+----+-------------+-------+------------+-------+--------------------------------------------------+------------+---------+------------------------------+--------+----------+----------------------------------------------+
| 1 | SIMPLE | i | NULL | index | PRIMARY,customer_id,code,invoice_id_2,created_at | PRIMARY | 4 | NULL | 473309 | 10.00 | Using where; Using temporary; Using filesort |
| 1 | SIMPLE | ii | NULL | ref | invoice_id | invoice_id | 5 | test.i.invoice_id | 2 | 100.00 | NULL |
| 1 | SIMPLE | it | NULL | ref | invoice_id,status | invoice_id | 5 | test.i.invoice_id | 1 | 100.00 | Using where |
+----+-------------+-------+------------+-------+--------------------------------------------------+------------+---------+------------------------------+--------+----------+----------------------------------------------+
I try to remove SUM ,GROUP_CONCAT and WHERE CASE,
But still need 10 sec
SELECT
`i`.*
FROM
`invoices` `i`
LEFT JOIN
`invoices_items` `ii` ON `ii`.`invoice_id` = `i`.`invoice_id`
LEFT JOIN
`invoices_transactions` `it` ON `it`.`invoice_id` = `i`.`invoice_id`
GROUP BY `i`.`invoice_id`
ORDER BY `i`.`created_at` DESC
LIMIT 50
Create Table:
CREATE TABLE `invoices` (
`invoice_id` int NOT NULL AUTO_INCREMENT,
`warehouse_id` tinyint DEFAULT NULL,
`invoice_type` varchar(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i NOT NULL DEFAULT 'C',
`invoice_date` date DEFAULT NULL,
`customer_id` int DEFAULT NULL,
`contact_name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`contact_no` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`contact_email` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`address` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`code` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`total_amount` deimal(10,2) DEFAULT '0.00',
`total_cost` deimal(20,2) DEFAULT NULL,
`delivery_type` tinyint DEFAULT '0',
`remark` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i,
`payment_status` tinyint(1) NOT NULL DEFAULT '3',
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` datetime DEFAULT NULL,
`updated_by` int NOT NULL,
`confirmed_at` datetime DEFAULT NULL,
`status` tinyint(1) NOT NULL DEFAULT '2'
PRIMARY KEY (`invoice_id`),
KEY `customer_id` (`customer_id`),
KEY `code` (`code`),
KEY `invoice_id_2` (`invoice_id`,`created_at`),
KEY `created_at` (`created_at`)
) ENGINE=InnoDB AUTO_INCREMENT=513697 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_i;
CREATE TABLE `invoices_items` (
`item_id` int NOT NULL AUTO_INCREMENT,
`invoice_id` int NOT NULL,
`warehouse_id` int DEFAULT NULL,
`product_id` int DEFAULT NULL,
`variant_id` int DEFAULT NULL,
`name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`quantity` int DEFAULT '1',
`unit_price` deimal(10,2) DEFAULT NULL,
`cost` deimal(10,2) DEFAULT NULL,
`subtotal` deimal(10,2) DEFAULT NULL,
`serial_no` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`StockoutDate` datetime DEFAULT NULL,
`ReturnDate` datetime DEFAULT NULL,
`Return_StockLocationID` tinyint DEFAULT NULL,
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` datetime DEFAULT NULL,
`status` tinyint(1) NOT NULL DEFAULT '1',
`priority` int NOT NULL DEFAULT '0',
PRIMARY KEY (`item_id`),
KEY `invoice_id` (`invoice_id`),
KEY `variant_id` (`variant_id`),
KEY `quantity` (`quantity`),
KEY `unit_price` (`unit_price`),
KEY `subtotal` (`subtotal`),
KEY `created_at` (`created_at`),
KEY `updated_at` (`updated_at`),
KEY `status` (`status`),
KEY `priority` (`priority`),
KEY `variant_id_2` (`variant_id`,`quantity`),
KEY `variant_id_3` (`variant_id`,`name`,`quantity`),
KEY `product_id` (`product_id`),
KEY `StockoutDate` (`StockoutDate`)
) ENGINE=InnoDB AUTO_INCREMENT=1200951 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_i;
CREATE TABLE `invoices_transactions` (
`transaction_id` int NOT NULL AUTO_INCREMENT,
`invoice_id` int DEFAULT NULL,
`warehouse_id` int DEFAULT NULL,
`invoice_date` date DEFAULT NULL,
`payment_id` int DEFAULT NULL,
`balance` deimal(10,2) DEFAULT NULL,
`amount` deimal(10,2) DEFAULT NULL,
`additional_rate` deimal(10,2) DEFAULT NULL,
`additional_fee` deimal(10,2) DEFAULT NULL,
`subtotal` deimal(10,2) DEFAULT NULL,
`process_status` int DEFAULT '1',
`remark` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`payment_status` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_i DEFAULT NULL,
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` datetime DEFAULT NULL,
`status` tinyint NOT NULL DEFAULT '1',
`CreatedByID` int DEFAULT NULL,
`ModifiedByID` int DEFAULT NULL,
`priority` int DEFAULT '0',
PRIMARY KEY (`transaction_id`),
KEY `invoice_id` (`invoice_id`),
KEY `payment_status` (`payment_status`),
KEY `created_at` (`created_at`),
KEY `updated_at` (`updated_at`),
KEY `status` (`status`),
KEY `amount` (`amount`),
KEY `additional_fee` (`additional_fee`),
KEY `subtotal` (`subtotal`),
KEY `transaction_id` (`transaction_id`,`amount`,`additional_fee`,`subtotal`),
KEY `invoice_id_2` (`invoice_id`,`process_status`,`status`),
KEY `process_status` (`process_status`),
KEY `transaction_id_2` (`transaction_id`,`invoice_id`,`amount`,`additional_fee`,`subtotal`,`process_status`,`status`)
) ENGINE=InnoDB AUTO_INCREMENT=543606 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_i;
Let's find the 50 invoices first, then reach for the rest of the stuff:
SELECT ... (all the current stuff)
FROM ( SELECT invoice_id
FROM invoices
WHERE status = 1
ORDER BY created_at DESC
LIMIT 50
) AS ids
JOIN invoices AS i USING(invoice_id)
LEFT JOIN ... ON ...
LEFT JOIN ... ON ...
GROUP BY invoice_id
ORDER BY created_id DESC
These composite indexes may help:
i: (status, created_at, invoice_id)
it: (status, process_status, invoice_id)
ii: (invoice_id, quantity, subtotal, status, item_id)
Redundant index:
KEY `variant_id` (`variant_id`) -- since this is the prefix of others
Because of PRIMARY KEY(transaction_id), the keys starting with transaction_id are redundant.
Summary:
I provided a "covering index" with the columns in the right order to allow finding the 50 ids directly from that index.
The rest of the work is limited to the 50 rows (or sets of rows, since I suspect that the rest are many:1 with respect to invoices.)
Your EXPLAIN shows lots of "Rows" and a "filesort" -- indicating that it gathered everything, then did the group by, then sorted, and finally, peeled of 50.
I suspect the "filesort" was really two sorts. See EXPLAIN FORMAT=JSON ... to find out.
I have this query:
SELECT
c.*,
cv.views
FROM
content AS c
JOIN
content_views AS cv ON cv.content = c.record_num
WHERE
c.enabled = 1
ORDER BY
cv.views
Quite simple, but it's really slow... Is there a way to make it faster ?
This is my EXPLAIN:
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE c ref enabled_2,enabled enabled 4 const 23947 Using temporary; Using filesort
1 SIMPLE cv eq_ref PRIMARY PRIMARY 4 c.record_num 1
EDIT 2016-02-24
Please note that usually, I use a LIMIT so the number of records returned in the EXPLAIN isn't entirely accurate, however for the sake of simplicity and because the performance doesn't change with the LIMIT or without it, I have removed it.
As requested in the comments, this is the result of my SHOW CREATE TABLE. As you can see, one of my table is MyISAM while the other is InnoDB.
CREATE TABLE `content` (
`title` varchar(255) NOT NULL DEFAULT '',
`filename` varchar(255) NOT NULL DEFAULT '',
`filename_2` varchar(255) NOT NULL,
`filename_3` varchar(255) NOT NULL,
`orig_filename` varchar(255) NOT NULL,
`trailer_filename` varchar(255) NOT NULL,
`thumbnail` varchar(255) NOT NULL DEFAULT '',
`embed` text NOT NULL,
`description` text NOT NULL,
`paysite` int(11) NOT NULL DEFAULT '0',
`keywords` varchar(255) NOT NULL,
`model` varchar(255) NOT NULL DEFAULT '',
`scheduled_date` date NOT NULL DEFAULT '0000-00-00',
`date_added` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`encoded_date` datetime NOT NULL,
`rating` int(5) NOT NULL DEFAULT '0',
`length` int(11) NOT NULL DEFAULT '0',
`submitter` int(11) NOT NULL DEFAULT '0',
`ip` varchar(15) NOT NULL,
`approved` int(11) NOT NULL DEFAULT '0',
`hotlinked` varchar(1024) NOT NULL,
`plug_url` varchar(255) NOT NULL,
`enabled` int(11) NOT NULL DEFAULT '0',
`main_thumb` int(11) NOT NULL DEFAULT '3',
`xml` varchar(32) NOT NULL,
`photos` int(11) NOT NULL DEFAULT '0',
`mobile` varchar(255) NOT NULL,
`modeltmp` varchar(255) NOT NULL,
`movie_width` int(11) NOT NULL,
`movie_height` int(11) NOT NULL,
`token` varchar(255) DEFAULT NULL,
`source_thumb_url` varchar(255) NOT NULL,
`related` varchar(1024) NOT NULL,
`force_related` varchar(255) NOT NULL,
`record_num` int(11) NOT NULL AUTO_INCREMENT,
`webvtt_src` text NOT NULL,
`category_thumb` int(11) NOT NULL,
`related_date` date NOT NULL,
`publish_ready` tinyint(1) NOT NULL,
PRIMARY KEY (`record_num`),
KEY `encoded_date` (`encoded_date`,`photos`,`enabled`),
KEY `filename` (`filename`),
KEY `scheduled_date` (`scheduled_date`),
KEY `enabled_2` (`enabled`,`length`,`photos`),
KEY `enabled` (`enabled`,`encoded_date`,`photos`),
KEY `rating` (`rating`,`enabled`,`photos`),
KEY `token` (`token`),
KEY `submitter` (`submitter`),
FULLTEXT KEY `keywords` (`keywords`,`title`),
FULLTEXT KEY `title` (`title`),
FULLTEXT KEY `description` (`description`),
FULLTEXT KEY `keywords_2` (`keywords`)
) ENGINE=MyISAM AUTO_INCREMENT=124207 DEFAULT CHARSET=latin1
CREATE TABLE `content_views` (
`views` int(11) NOT NULL,
`content` int(11) NOT NULL,
PRIMARY KEY (`content`),
KEY `views` (`views`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
For this query:
SELECT c.*, cv.views
FROM content c JOIN
content_views cv
ON cv.content = c.record_num
WHERE c.enabled = 1
ORDER BY cv.views;
The best indexes are probably content(enabled, record_num) and content_views(content, views). I am guessing that the performance even with these indexes will be similar to what you have now.
I have a rather slow query that I'd like to optimize. EXPLAIN shows 'using temporary; using filesort'. I tried a few solutions and, doing without an ORDER BY, even managed to get rid of the 'using filesort'. But is there a way to avoid 'using temporary; using filesort' entirely, without sacrificing the ORDER BY?
This is my query:
SELECT `tags`.`name`,
`tags`.`tag_id`,
COUNT(*) AS `qty_products`
FROM `products_subsubcategories`
JOIN `products_tags` ON `products_subsubcategories`.`product_id` = `products_tags`.`product_id`
JOIN `products` ON `products_subsubcategories`.`product_id` = `products`.`product_id`
JOIN `tags` ON `products_tags`.`tag_id` = `tags`.`tag_id`
WHERE `products_subsubcategories`.`subsubcategory_id` = 55
AND `tags`.`type` = 'brand'
AND `products`.`dont_display` = 0
GROUP BY `tags`.`tag_id`
ORDER BY `tags`.`order`,
`tags`.`name`;
The subsubcategory 55 is dynamic user input.
This is the EXPLAIN result:
id select_type table type possible_keys key key_len ref rows filtered Extra
1 SIMPLE products_subsubcategories ref PRIMARY,subsubcategory_id subsubcategory_id 4 const 3982 100.00 Using temporary; Using filesort
1 SIMPLE tags ALL PRIMARY,type NULL NULL NULL 679 78.94 Using where; Using join buffer
1 SIMPLE products eq_ref PRIMARY,dont_display PRIMARY 4 mbb.products_subsubcategories.product_id 1 100.00 Using where
1 SIMPLE products_tags eq_ref PRIMARY,tag_id PRIMARY 8 mbb.products.product_id,mbb.tags.tag_id 1 100.00 Using where; Using index
(When I replace ORDER BY ... by ORDER BY NULL, the 'using filesort' disapperars. I could sort the result with PHP afterwards, although it's more convenient with MySQL, of course ...)
My tables look like this:
CREATE TABLE IF NOT EXISTS `products_subsubcategories` (
`position` smallint(5) unsigned NOT NULL DEFAULT '0',
`product_id` int(10) unsigned NOT NULL,
`subsubcategory_id` int(10) unsigned NOT NULL,
PRIMARY KEY (`product_id`,`subsubcategory_id`),
KEY `subsubcategory_id` (`subsubcategory_id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE IF NOT EXISTS `products_tags` (
`product_id` int(10) unsigned NOT NULL,
`tag_id` int(10) unsigned NOT NULL,
PRIMARY KEY (`product_id`,`tag_id`),
KEY `tag_id` (`tag_id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE IF NOT EXISTS `products` (
`article_number` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`date` date DEFAULT NULL,
`delivery_time` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`description` text COLLATE utf8_unicode_ci NOT NULL,
`dont_display` tinyint(1) unsigned NOT NULL DEFAULT '0',
`ean` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`image_error` tinyint(1) NOT NULL DEFAULT '0',
`image_is_downloaded` tinyint(1) NOT NULL DEFAULT '0',
`image_url` varchar(400) COLLATE utf8_unicode_ci NOT NULL,
`image_url_170_134` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`image_url_original_size` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`is_deleted` tinyint(1) unsigned NOT NULL DEFAULT '0',
`is_duplicate` tinyint(1) unsigned NOT NULL DEFAULT '0',
`is_not_associated_to_category` tinyint(1) unsigned NOT NULL DEFAULT '0',
`is_not_associated_to_subcategory` tinyint(1) unsigned NOT NULL DEFAULT '0',
`is_not_associated_to_subsubcategory` tinyint(1) unsigned NOT NULL DEFAULT '0',
`last_association` datetime DEFAULT NULL,
`last_completion_by_ean` datetime DEFAULT NULL,
`matching_age` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_brand` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_category` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_color` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_gender` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_keywords` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_main_category` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_size` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_subcategory` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`matching_subsubcategory` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`old_price` decimal(7,2) unsigned NOT NULL DEFAULT '0.00',
`price` decimal(7,2) unsigned NOT NULL DEFAULT '0.00',
`product_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`product_list_id` int(10) unsigned NOT NULL DEFAULT '0',
`qty_overall_clicks` int(10) unsigned NOT NULL DEFAULT '0',
`shipping` decimal(7,2) unsigned NOT NULL DEFAULT '0.00',
`shop_url` varchar(400) COLLATE utf8_unicode_ci NOT NULL,
`title` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`vendor_id` int(10) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`product_id`),
KEY `article_number` (`article_number`),
KEY `dont_display` (`dont_display`),
KEY `ean` (`ean`),
KEY `is_deleted` (`is_deleted`),
KEY `is_duplicate` (`is_duplicate`),
KEY `is_not_associated_to_category` (`is_not_associated_to_category`),
KEY `is_not_associated_to_subcategory` (`is_not_associated_to_subcategory`),
KEY `is_not_associated_to_subsubcategory` (`is_not_associated_to_subsubcategory`),
KEY `product_list_id` (`product_list_id`),
KEY `vendor_id` (`vendor_id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=1084370;
CREATE TABLE IF NOT EXISTS `tags` (
`display_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`image_url` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`order` int(10) unsigned NOT NULL DEFAULT '0',
`tag_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`type` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
PRIMARY KEY (`tag_id`),
KEY `type` (`type`),
KEY `name` (`name`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=1084;
I would suggest attempting a query that doesn't use JOINs, simply for the fact that you don't use the JOINs for anything other than getting a count.
Try the following:
SELECT
t.name,
t.tag_id,
(
SELECT COUNT(*) FROM product_tags pt
INNER JOIN product_subcategories ps
ON ps.product_id = pt.product_id
INNER JOIN product p
ON p.product_id = pt.product_id
WHERE pt.tag_id = t.tag_id
AND p.dont_display = 0
AND ps.subsubcategory_id = 55
) AS qty_products
FROM tags t
WHERE
t.type = 'brand'
AND EXISTS (
SELECT * FROM product_tags pt
INNER JOIN product_subcategories ps
ON ps.product_id = pt.product_id
INNER JOIN product p
ON p.product_id = pt.product_id
WHERE pt.tag_id = t.tag_id
AND p.dont_display = 0
AND ps.subsubcategory_id = 55
)
ORDER BY t.order,t.name
In this way, you only query the tags table, getting the results back in order initially. Then for each record you check if any are in subsubcategory 55, and otherwise skip that tag.
This should improve your query greatly, unless there are an enormous number of tags (and even then it still might improve things.)
Another improvement you can make is the one Kickstart suggested in the comments: add a covering index to the tags table:
ALTER TABLE tags
ADD INDEX `type_order_name` (`type`,`order`,`name`)
If you aren't familiar with multipart keys, just know that internally they are effectively stored as a concatenation of each of the columns, in the order the columns are listed in the key definition.
Therefore as long as you provide the type in the WHERE clause, the tags will be stored ordered by order and name, just as this query wants them. This will result in very fast sorting (because they will already be sorted in the index).
i have a query as given below
SELECT MONTHNAME( f.receipt_date ) AS MONTH ,
SUM( CASE WHEN fm.fee_name = 'University Fees' THEN f.fee_amount END ) AS A
FROM fee_type_masters fm
INNER JOIN student_fee_collections f ON fm.id = f.fee_type
GROUP BY MONTH( f.receipt_date )
and my tables are like
CREATE TABLE `fee_type_masters` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`fee_name` varchar(255) DEFAULT NULL,
`fee_type` varchar(255) DEFAULT NULL,
`year` varchar(255) DEFAULT NULL,
`student_type` int(11) DEFAULT NULL,
`due_date` varchar(255) DEFAULT NULL,
`amount` float DEFAULT NULL,
`accounts_master_id` varchar(255) DEFAULT NULL,
`comments` varchar(255) DEFAULT NULL,
`status` varchar(255) DEFAULT NULL,
`sem` varchar(255) DEFAULT NULL,
`degree_id` int(11) DEFAULT NULL,
`approve_needed` varchar(255) DEFAULT NULL,
`concession_allowed` varchar(255) DEFAULT NULL,
`created_at` datetime DEFAULT NULL,
`updated_at` datetime DEFAULT NULL,
`relation_id` int(11) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=latin1
CREATE TABLE `student_fee_collections` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`student_id` int(11) DEFAULT NULL,
`fee_type` int(11) DEFAULT NULL,
`fee_amount` int(11) DEFAULT '0',
`due` int(11) DEFAULT '0',
`received` int(11) DEFAULT '0',
`concession` int(11) DEFAULT '0',
`receipt_no` int(11) DEFAULT NULL,
`receipt_date` date DEFAULT NULL,
`received_by` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`amount` int(11) DEFAULT '0',
`late_fee` int(11) DEFAULT '0',
`pay_mode` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`comments` text COLLATE utf8_unicode_ci,
`created_at` datetime DEFAULT NULL,
`updated_at` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `student_id` (`fee_type`,`student_id`)
) ENGINE=InnoDB AUTO_INCREMENT=325 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
each is having more than 20000 rows
and my explain plan is like
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE fm ALL PRIMARY NULL NULL NULL 7000 Using temporary; Using filesort
1 SIMPLE f ref student_id student_id 5 emsnew.fm.id 28000 Using where
any one please tell me how to optimize the query or rewrite.
Create Index on fee_type column in student_fee_collections and recheck the output of Explain Query. And update the same in your question.
Create Index on fee_type column in student_fee_colle
and run the following query
SELECT MONTHNAME( f.receipt_date ) AS MONTH ,
SUM( CASE WHEN fm.fee_name = 'University Fees' THEN f.fee_amount END ) AS A
FROM fee_type_masters fm
INNER JOIN student_fee_collections f USE index(fee_type_IDX) ON fm.id = f.fee_type
GROUP BY MONTH( f.receipt_date )
I have few queries and am not able to figure out how to optimize them,
QUERY 1
select *
from t_twitter_tracking
where classified is null and tweetType='ENGLISH'
order by id limit 500;
QUERY 2
Select
count(*) as cnt,
DATE_FORMAT(CONVERT_TZ(wrdTrk.createdOnGMTDate,'+00:00','+05:30'),'%Y-%m-%d')
as dat
from
t_twitter_tracking wrdTrk
where
wrdTrk.word like ('dell')
and CONVERT_TZ(wrdTrk.createdOnGMTDate,'+00:00','+05:30')
between '2010-12-12 00:00:00' and '2010-12-26 00:00:00'
group by dat;
Both these queries run on the same table,
CREATE TABLE `t_twitter_tracking` (
`id` BIGINT(20) NOT NULL AUTO_INCREMENT,
`word` VARCHAR(200) NOT NULL,
`tweetId` BIGINT(100) NOT NULL,
`twtText` VARCHAR(800) NULL DEFAULT NULL,
`language` TEXT NULL,
`links` TEXT NULL,
`tweetType` VARCHAR(20) NULL DEFAULT NULL,
`source` TEXT NULL,
`sourceStripped` TEXT NULL,
`isTruncated` VARCHAR(40) NULL DEFAULT NULL,
`inReplyToStatusId` BIGINT(30) NULL DEFAULT NULL,
`inReplyToUserId` INT(11) NULL DEFAULT NULL,
`rtUsrProfilePicUrl` TEXT NULL,
`isFavorited` VARCHAR(40) NULL DEFAULT NULL,
`inReplyToScreenName` VARCHAR(40) NULL DEFAULT NULL,
`latitude` BIGINT(100) NOT NULL,
`longitude` BIGINT(100) NOT NULL,
`retweetedStatus` VARCHAR(40) NULL DEFAULT NULL,
`statusInReplyToStatusId` BIGINT(100) NOT NULL,
`statusInReplyToUserId` BIGINT(100) NOT NULL,
`statusFavorited` VARCHAR(40) NULL DEFAULT NULL,
`statusInReplyToScreenName` TEXT NULL,
`screenName` TEXT NULL,
`profilePicUrl` TEXT NULL,
`twitterId` BIGINT(100) NOT NULL,
`name` TEXT NULL,
`location` VARCHAR(100) NULL DEFAULT NULL,
`bio` TEXT NULL,
`url` TEXT NULL COLLATE 'latin1_swedish_ci',
`utcOffset` INT(11) NULL DEFAULT NULL,
`timeZone` VARCHAR(100) NULL DEFAULT NULL,
`frenCnt` BIGINT(20) NULL DEFAULT '0',
`createdAt` DATETIME NULL DEFAULT NULL,
`createdOnGMT` VARCHAR(40) NULL DEFAULT NULL,
`createdOnServerTime` DATETIME NULL DEFAULT NULL,
`follCnt` BIGINT(20) NULL DEFAULT '0',
`favCnt` BIGINT(20) NULL DEFAULT '0',
`totStatusCnt` BIGINT(20) NULL DEFAULT NULL,
`usrCrtDate` VARCHAR(200) NULL DEFAULT NULL,
`humanSentiment` VARCHAR(30) NULL DEFAULT NULL,
`replied` BIT(1) NULL DEFAULT NULL,
`replyMsg` TEXT NULL,
`classified` INT(32) NULL DEFAULT NULL,
`createdOnGMTDate` DATETIME NULL DEFAULT NULL,
`locationDetail` TEXT NULL,
`geonameid` INT(11) NULL DEFAULT NULL,
`country` VARCHAR(255) NULL DEFAULT NULL,
`continent` CHAR(2) NULL DEFAULT NULL,
`placeLongitude` FLOAT NULL DEFAULT NULL,
`placeLatitude` FLOAT NULL DEFAULT NULL,
PRIMARY KEY (`id`),
INDEX `id` (`id`, `word`),
INDEX `createdOnGMT_index` (`createdOnGMT`) USING BTREE,
INDEX `word_index` (`word`) USING BTREE,
INDEX `location_index` (`location`) USING BTREE,
INDEX `classified_index` (`classified`) USING BTREE,
INDEX `tweetType_index` (`tweetType`) USING BTREE,
INDEX `getunclassified_index` (`classified`, `tweetType`) USING BTREE,
INDEX `timeline_index` (`word`, `createdOnGMTDate`, `classified`) USING BTREE,
INDEX `createdOnGMTDate_index` (`createdOnGMTDate`) USING BTREE,
INDEX `locdetail_index` (`country`, `id`) USING BTREE,
FULLTEXT INDEX `twtText_index` (`twtText`)
)
COLLATE='utf8_general_ci'
ENGINE=MyISAM
ROW_FORMAT=DEFAULT
AUTO_INCREMENT=12608048;
The table has more than 10 million records. How can I optimize it?
EDITED
Explain on 2nd query
"id";"select_type";"table";"type";"possible_keys";"key";"key_len";"ref";"rows";"Extra"
"1";"SIMPLE";"wrdTrk";"range";"word_index,word_createdOnGMT";"word_index";"602";NULL;"222847";"Using where; Using temporary; Using filesort"
Regards,
Rohit
In Query2, I suggest that:
1. remove DATE_FORMAT and CONVERT_TZ. You can process in PHP to be an output or between's condition.
2. like ('dell'): I don't see any '%', so you can use wrdTrk.word = 'dell' to let it faster.
The convert_tz in the where condition needs to be removed,
Select
count(*) as cnt,
DATE_FORMAT(CONVERT_TZ(wrdTrk.createdOnGMTDate,'+00:00','+05:30'),'%Y-%m-%d')
as dat
from
t_twitter_tracking wrdTrk
where
wrdTrk.word like ('dell')
and CONVERT_TZ(wrdTrk.createdOnGMTDate,'+00:00','+05:30')
between '2010-12-12 00:00:00' and '2010-12-26 00:00:00'
group by dat;
This will lead to comparing each row and finding out the right result, hence a tremendous improvement in the query result. Just passed the converted data to the query.
Select
count(*) as cnt,
DATE_FORMAT(CONVERT_TZ(wrdTrk.createdOnGMTDate,'+00:00','+05:30'),'%Y-%m-%d')
as dat
from
t_twitter_tracking wrdTrk
where
wrdTrk.word like ('dell')
and CONVERT_TZ(wrdTrk.createdOnGMTDate,'+00:00','+05:30')
between '2010-12-12 00:00:00' and '2010-12-26 00:00:00'
group by dat;