I have a basic query that runs great on small datasets but when you hit 100,000 results, the query takes forever (1.9 seconds) to run. Please help with possible indexing because maybe I missed something (have already indexed quite extensively). Below is the query
SELECT
count( * ) AS counts
FROM
pages pg
INNER JOIN products AS p
ON p.page_id = pg.id
INNER JOIN pages_description AS pgd
ON pg.id = pgd.page_id
and pgd.language_id = 1
WHERE
pgd.active = 1
AND pgd.deleted = 0
AND pg.type = 4
AND pg.created_date between '2018-01-30 06:00:00' and '2018-12-30 09:00:00'
AND pg.modified_date between '2018-01-30 07:00:00' and '2018-12-30 09:00:00'
CREATE TABLE `pages` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`parent` int(11) DEFAULT NULL,
`type` int(11) DEFAULT NULL,
`template_id` int(11) DEFAULT NULL,
`link` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`local_left` tinyint(1) DEFAULT NULL,
`sort_order` int(11) DEFAULT NULL,
`faceted_search` tinyint(1) DEFAULT NULL,
`created_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`modified_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`) USING BTREE,
KEY `idx_parent` (`parent`) USING BTREE,
KEY `idx_type` (`type`) USING BTREE,
KEY `idx_template` (`template_id`) USING BTREE,
KEY `idx_local_left` (`local_left`) USING BTREE,
KEY `idx_sort` (`sort_order`) USING BTREE,
KEY `idx_faceted_search` (`faceted_search`) USING BTREE,
KEY `idx_dates` (`type`,`created_date`,`modified_date`,`id`) USING BTREE
) ENGINE=MyISAM AUTO_INCREMENT=149352 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE `pages_description` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`page_id` int(11) NOT NULL,
`language_id` int(11) NOT NULL DEFAULT '1',
`name` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`menu_name` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`content` text COLLATE utf8_unicode_ci,
`sub_content` text COLLATE utf8_unicode_ci,
`seo_url` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`h1` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`keywords` text COLLATE utf8_unicode_ci,
`title_tag` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`meta_description` varchar(390) COLLATE utf8_unicode_ci DEFAULT NULL,
`canonical_tag` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`active` tinyint(1) DEFAULT NULL,
`deleted` tinyint(1) DEFAULT '0',
PRIMARY KEY (`id`) USING BTREE,
UNIQUE KEY `unq_page_lang` (`page_id`,`language_id`) USING BTREE,
KEY `idx_name` (`name`) USING BTREE,
KEY `idx_menu_name` (`menu_name`) USING BTREE,
KEY `idx_active` (`active`) USING BTREE,
KEY `idx_deleted` (`deleted`) USING BTREE,
KEY `idx_lang` (`language_id`),
KEY `idx_page` (`page_id`) USING BTREE,
KEY `idx_active_deleted_id` (`active`,`deleted`,`page_id`,`language_id`) USING BTREE
) ENGINE=MyISAM AUTO_INCREMENT=149319 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE `products` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`page_id` int(11) NOT NULL,
`on_hand` int(11) NOT NULL DEFAULT '0',
`buy` tinyint(1) NOT NULL DEFAULT '0',
`rfq` tinyint(1) NOT NULL DEFAULT '1',
`model` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`parent_type` int(11) NOT NULL DEFAULT '1',
`image` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`image_alt` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`image_title` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`base_price` decimal(10,2) DEFAULT NULL,
`length` decimal(10,5) DEFAULT NULL,
`width` decimal(10,5) DEFAULT NULL,
`height` decimal(10,5) DEFAULT NULL,
`weight` decimal(10,5) DEFAULT NULL,
`tax_class` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`ready_to_ship` tinyint(1) DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
UNIQUE KEY `unq_page_id` (`page_id`) USING BTREE
) ENGINE=MyISAM AUTO_INCREMENT=148391 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
I know one might think that 1.9 seconds is forever but this is a web app and needs to be responsive. I have added the schema for the tables in question.
I was able to figure it out. I can't believe I missed this. I optimized the tables and the query was then cut down to 350ms.
Related
First query:
SELECT
u.id,
u.first_name,
u.last_name,
u.tazkera_id,
cu.relation_type,
CASE csp.`shift`
WHEN '1' THEN 'Morning'
WHEN '2' THEN 'Afternoon'
else 'Neither'
END AS shift
FROM `course_user` as cu
LEFT JOIN `courses` as c ON c.id = cu.`course_id`
LEFT JOIN (
select users.id,
users.first_name,
users.last_name,
users.tazkera_id
FROM `users`
) as u ON u.id = cu.`user_id`
left JOIN `course_schedule_prefs` as csp ON csp.`user_id` = cu.`user_id`
Where cu.relation_type = 1 group by cu.`user_id`;
Second Query:
SELECT
u.id,
u.first_name,
u.last_name,
u.tazkera_id,
cu.relation_type,
CASE csp.`shift`
WHEN '1' THEN 'Morning'
WHEN '2' THEN 'Afternoon'
else 'Neither'
END AS shift
FROM `course_user` as cu
LEFT JOIN `courses` as c ON c.id = cu.`course_id`
LEFT JOIN users as u ON u.id = cu.`user_id`
left JOIN `course_schedule_prefs` as csp ON csp.`user_id` = cu.`user_id`
Where cu.relation_type = 1 group by cu.`user_id`;
Tables :
users:
CREATE TABLE `users` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`first_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`last_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`email` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`password` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`is_password_reset` int(11) NOT NULL DEFAULT '0',
`login_ip` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`login_date` datetime DEFAULT NULL,
`last_login_ip` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`last_login_date` datetime DEFAULT NULL,
`is_email_address_verified` int(11) NOT NULL DEFAULT '0',
`failed_login_attempts` int(11) DEFAULT NULL,
`last_failed_login_date` datetime DEFAULT NULL,
`tazkera_id` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`province_code` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`district_code` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`village` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`home_address` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`promote_unique_user_id` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`created_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`remember_token` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
`forgot_token` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
`email_confirm_token` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`district_other` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`father_name` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`training_provider_id` int(10) unsigned DEFAULT NULL,
`is_profile_completed` int(11) DEFAULT NULL,
`active` int(11) DEFAULT '1',
`picture` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`small_picture` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`last_poll` bigint(20) DEFAULT NULL,
`last_password_reset` date DEFAULT NULL,
`presence_status` int(11) DEFAULT NULL,
`auto_presence` int(11) NOT NULL DEFAULT '0',
`last_activity` timestamp NULL DEFAULT NULL,
`trash` tinyint(4) NOT NULL DEFAULT '0',
`login_approval` tinyint(4) NOT NULL DEFAULT '0',
`lang` varchar(2) COLLATE utf8_unicode_ci NOT NULL DEFAULT 'en',
`cover` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`small_cover` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`forum_reputation` int(11) NOT NULL DEFAULT '0',
`forum_suspension_date` date DEFAULT NULL,
`forum_status` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `users_email_unique` (`email`),
KEY `users_training_provider_id_foreign` (`training_provider_id`),
CONSTRAINT `users_training_provider_id_foreign` FOREIGN KEY (`training_provider_id`) REFERENCES `training_providers` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB AUTO_INCREMENT=1000038 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
course_users:
CREATE TABLE `course_user` (
`course_id` int(10) unsigned DEFAULT NULL,
`user_id` int(10) unsigned DEFAULT NULL,
`relation_type` int(11) DEFAULT NULL,
`created_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
KEY `course_user_course_id_foreign` (`course_id`),
KEY `course_user_user_id_foreign` (`user_id`),
CONSTRAINT `course_user_course_id_foreign` FOREIGN KEY (`course_id`) REFERENCES `courses` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT `course_user_user_id_foreign` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
courses;
CREATE TABLE `courses` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`curriculum_id` int(10) unsigned DEFAULT NULL,
`training_center_id` int(10) unsigned DEFAULT NULL,
`training_coordinator_id` int(10) unsigned DEFAULT NULL,
`focal_point_id` int(10) unsigned DEFAULT NULL,
`start_date` date DEFAULT NULL,
`end_date` date DEFAULT NULL,
`start_time` time DEFAULT NULL,
`end_time` time DEFAULT NULL,
`created_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`shift` int(11) DEFAULT NULL,
`stage_id` int(10) unsigned DEFAULT NULL,
`stream_id` int(10) unsigned DEFAULT NULL,
`unit_id` int(10) unsigned DEFAULT NULL,
`master_trainer_id` int(10) unsigned DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`pre_test_generated` int(11) NOT NULL DEFAULT '0',
`post_test_generated` int(11) NOT NULL DEFAULT '0',
`conduct_days` int(11) DEFAULT NULL,
`no` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`evaluation_available` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `courses_curriculum_id_foreign` (`curriculum_id`),
KEY `courses_training_coordinator_id_foreign` (`training_coordinator_id`),
KEY `courses_focal_point_id_foreign` (`focal_point_id`),
KEY `courses_stage_id_foreign` (`stage_id`),
KEY `courses_stream_id_foreign` (`stream_id`),
KEY `courses_unit_id_foreign` (`unit_id`),
KEY `courses_master_trainer_id_foreign` (`master_trainer_id`),
KEY `courses_training_center_id_foreign` (`training_center_id`),
CONSTRAINT `courses_curriculum_id_foreign` FOREIGN KEY (`curriculum_id`) REFERENCES `curriculums` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT `courses_focal_point_id_foreign` FOREIGN KEY (`focal_point_id`) REFERENCES `users` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT `courses_master_trainer_id_foreign` FOREIGN KEY (`master_trainer_id`) REFERENCES `users` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT `courses_stage_id_foreign` FOREIGN KEY (`stage_id`) REFERENCES `stages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT `courses_stream_id_foreign` FOREIGN KEY (`stream_id`) REFERENCES `streams` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT `courses_training_center_id_foreign` FOREIGN KEY (`training_center_id`) REFERENCES `training_centers` (`id`),
CONSTRAINT `courses_training_coordinator_id_foreign` FOREIGN KEY (`training_coordinator_id`) REFERENCES `users` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT `courses_unit_id_foreign` FOREIGN KEY (`unit_id`) REFERENCES `units` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB AUTO_INCREMENT=139 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
course_schedule_prefs;
CREATE TABLE `course_schedule_prefs` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`cunduct_days` int(11) DEFAULT NULL,
`training_centers` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`shift` int(11) NOT NULL,
`created_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`user_id` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `course_schedule_prefs_user_id_foreign` (`user_id`),
CONSTRAINT `course_schedule_prefs_user_id_foreign` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
My question is , which one of this query is good in term of performance and accuracy. Why ?
first query: I join directly with table; select all columns of table.
Second query: I used subquery inside of join; select only those element which I need it.
Note : My main question is about users table which is in join
There are too many possibilities here. There is no simple, 'straight', answer.
With LEFT JOIN (instead of JOIN), the Optimizer cannot start with the subquery, which it would usually like to do.
If there is a WHERE clause and it is useful for filtering rows, the Optimizer would like to start with the table that is filtered the most. Your WHERE clause smells like a flag, which is usually not very useful.
If a subquery has a GROUP BY or LIMIT, then it may significantly shrink the number of rows to work with. (Not in your case.) This might make the subquery approach better.
If cu has INDEX(relation_type, user_id), the Optimizer may decide that handling all of the WHERE and GROUP BY leads to the best bet, thereby starting with cu as the 'first' table.
In old versions of MySQL, subqueries in JOIN ( SELECT ... ) had no way of getting an index, thereby leading to inefficient table scans of that tmp table. In new versions (starting with 5.6, I think), the Optimizer takes the extra step (which costs something) to generate the optimal index ((id) in this case) after creating the tmp table and before JOINing it.
More
2 more table definitions needed.
Do you have the indexes I suggested?
InnoDB really needs a PRIMARY KEY - See many:many for advice on course_user.
This is my SQL code which links users to items based on tutorials:
CREATE TABLE IF NOT EXISTS `users` (
`user_id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`user_name` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`user_password_hash` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`user_email` varchar(254) COLLATE utf8_unicode_ci NOT NULL,
`user_access_level` tinyint(3) unsigned NOT NULL DEFAULT '0',
`user_active` tinyint(1) NOT NULL DEFAULT '0',
`user_activation_hash` varchar(40) COLLATE utf8_unicode_ci DEFAULT NULL,
`user_password_reset_hash` char(40) COLLATE utf8_unicode_ci DEFAULT NULL,
`user_password_reset_timestamp` bigint(20) DEFAULT NULL,
`user_failed_logins` tinyint(1) NOT NULL DEFAULT '0',
`user_last_failed_login` int(10) DEFAULT NULL,
`user_registration_datetime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`user_registration_ip` varchar(39) COLLATE utf8_unicode_ci NOT NULL DEFAULT '0.0.0.0',
PRIMARY KEY (`user_id`),
UNIQUE KEY `user_name` (`user_name`),
UNIQUE KEY `user_email` (`user_email`)
) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE IF NOT EXISTS `item` (
`item_id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`item_title` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`item_location` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`item_description` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`item_datetime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`item_status` int(1) unsigned NOT NULL,
PRIMARY KEY (`item_id`)
) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
http://sqlfiddle.com/#!9/fd011
I'm getting a bit confused about how to link the items to the user. It seems like I need something called a foreign key on the items, a bit like this in my item table:
FOREIGN KEY (user_id) REFERENCES user(ID)
I can't seem to get it to compile and query successfully. Can anyone please show me the right way to associate the items with the user.
You need to add the user_id column to the items table along with the constraint:
CREATE TABLE IF NOT EXISTS `users` (
`user_id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`user_name` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`user_password_hash` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`user_email` varchar(254) COLLATE utf8_unicode_ci NOT NULL,
`user_access_level` tinyint(3) unsigned NOT NULL DEFAULT '0',
`user_active` tinyint(1) NOT NULL DEFAULT '0',
`user_activation_hash` varchar(40) COLLATE utf8_unicode_ci DEFAULT NULL,
`user_password_reset_hash` char(40) COLLATE utf8_unicode_ci DEFAULT NULL,
`user_password_reset_timestamp` bigint(20) DEFAULT NULL,
`user_failed_logins` tinyint(1) NOT NULL DEFAULT '0',
`user_last_failed_login` int(10) DEFAULT NULL,
`user_registration_datetime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`user_registration_ip` varchar(39) COLLATE utf8_unicode_ci NOT NULL DEFAULT '0.0.0.0',
PRIMARY KEY (`user_id`),
UNIQUE KEY `user_name` (`user_name`),
UNIQUE KEY `user_email` (`user_email`)
) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE IF NOT EXISTS `item` (
`item_id` int(11) unsigned NOT NULL AUTO_INCREMENT,
user_id int unsigned,
`item_title` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`item_location` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`item_description` varchar(64) COLLATE utf8_unicode_ci NOT NULL,
`item_datetime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`item_status` int(1) unsigned NOT NULL,
PRIMARY KEY (`item_id`),
FOREIGN KEY (user_id) REFERENCES users(user_id)
) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
The SQL Fiddle is here.
I should point out a few other things:
Pay attention to the database engine you are using. MyISAM doesn't actually enforce the relationship.
Having weird dates as the default value is probably less useful than just using NULL.
I'm not sure if there is a value to having explicit collations for every character definition, unless your database is going to be supporting a wide variety of collations.
Don't use single quotes for numeric constants. So, if a value is declared as a tinyint, set the default ot 0 not '0' (this doesn't affect performance in a CREATE TABLE statement; it is just misleading).
#GordonLindoff's solution is one method, but that assumes that each item belongs to exactly one user, and an item cannot be referenced by multiple users. If you have a many-to-many relationship, where a user can have multiple items and an item can be referenced by multiple users, then you need a third table that links them together:
CREATE TABLE IF NOT EXISTS `user_item` (
`user_id` int(11) unsigned NOT NULL,
`item_id` int(11) unsigned NOT NULL,
PRIMARY KEY (`user_item`,`item_id`),
FOREIGN KEY (user_id) REFERENCES users(user_id),
FOREIGN KEY (item_id) REFERENCES items(item_id)
)ENGINE=Innodb DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
The Foreign Key constraints enforce that for every row in user_item that the user_id exists in users, and the item_id exists in items. And as was mentioned in a previous comment, that you will need Innodb to have the foreign key constraints enforced.
I have this query that doesn't run under odd circumstances. It's doing two joins, I have two indexes of the same name on two different tables (but this shouldn't matter). When I remove one of the indexes from either tables, the query runs fine. Something else that makes it run is removing the order by clause, but I definitely need that. When I run show processlist the state is stuck on "Copying to tmp table". The indexes I'm talking about are channelID.
the query
SELECT twitterTweets.*,
COUNT(twitterRetweets.id) AS retweets,
sTwitter.followers,
sTwitter.date
FROM twitterTweets
LEFT JOIN twitterRetweets
ON twitterTweets.id = twitterTweetsID
JOIN sTwitter
ON DATE(twitterTweets.dateCreated) = sTwitter.date
AND twitterTweets.channelID = sTwitter.channelID
WHERE twitterTweets.channelID = 32
AND type = 'tweet'
AND DATE(dateCreated) >= '2013-12-05'
AND DATE(dateCreated) <= '2014-01-05'
GROUP BY twitterTweets.id
ORDER BY dateCreated
explain results
1 SIMPLE sTwitter ref channelID channelID 5 const 1162 Using where; Using temporary; Using filesort
1 SIMPLE twitterTweets ref channelID channelID 5 const 17456 Using where
1 SIMPLE twitterRetweets ref twitterTweetsID twitterTweetsID 5 social.twitterTweets.id 3
create for the three tables
CREATE TABLE `twitterTweets` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`clientID` int(11) DEFAULT NULL,
`divisionID` int(11) DEFAULT NULL,
`accountID` int(11) DEFAULT NULL,
`channelID` int(11) DEFAULT NULL,
`type` enum('tweet','reply','direct in','direct out') COLLATE utf8_unicode_ci DEFAULT 'tweet',
`subType` enum('reply beginning','retweet beginning','reply middle','retweet middle') COLLATE utf8_unicode_ci DEFAULT NULL,
`tweetID` bigint(20) DEFAULT NULL,
`tweet` text COLLATE utf8_unicode_ci,
`replies` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`hash` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`retweet` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`source` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`media` text COLLATE utf8_unicode_ci,
`tweetUrls` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`expandedUrls` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`replyToStatus` bigint(20) DEFAULT NULL,
`user` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`followers` int(11) DEFAULT NULL,
`following` int(11) DEFAULT NULL,
`updates` int(11) DEFAULT NULL,
`group1` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`campaign` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`segment` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`destination` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
`retweets` smallint(6) DEFAULT '0',
`favorites` smallint(6) DEFAULT NULL,
`dateCreated` datetime DEFAULT NULL,
`dateAdded` datetime DEFAULT NULL,
`dateUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `channelID` (`channelID`)
) ENGINE=MyISAM AUTO_INCREMENT=296264 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE `twitterRetweets` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`clientID` int(11) DEFAULT NULL,
`divisionID` int(11) DEFAULT NULL,
`accountID` int(11) DEFAULT NULL,
`channelID` int(11) DEFAULT NULL,
`twitterTweetsID` int(11) DEFAULT NULL,
`tweetID` bigint(20) DEFAULT NULL,
`screenName` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`followers` int(11) DEFAULT NULL,
`following` int(11) DEFAULT NULL,
`updates` int(11) DEFAULT NULL,
`favorites` smallint(6) DEFAULT NULL,
`dateAdded` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `twitterTweetsID` (`twitterTweetsID`)
) ENGINE=MyISAM AUTO_INCREMENT=93821 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE `sTwitter` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`clientID` int(11) DEFAULT NULL,
`divisionID` int(11) DEFAULT NULL,
`accountID` int(11) DEFAULT NULL,
`channelID` int(11) DEFAULT NULL,
`following` int(11) DEFAULT '0',
`followers` int(11) DEFAULT '0',
`updates` int(11) DEFAULT '0',
`date` date DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `channelID` (`channelID`)
) ENGINE=MyISAM AUTO_INCREMENT=35615 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
So how do I make this thing run with both indexes in place? Do I have to ignore one of them?
I have next log.
Please somebody explain me where is the problem.
My table deals is 142906 why is it taking so much time?
# Query_time: 5.524629 Lock_time: 0.000059 Rows_sent: 3 Rows_examined: 142906
SET timestamp=1381963341;
SELECT *,1 as distance FROM deals
WHERE end_date > '1381959736'
GROUP BY title
ORDER BY COALESCE(distance, 999999999) , distance ASC LIMIT 0 , 3;
Here is ddl.
CREATE TABLE `deals` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`sku` varchar(180) NOT NULL DEFAULT '',
`price` decimal(8,2) DEFAULT NULL,
`retail_price` decimal(8,2) DEFAULT NULL,
`category` int(1) DEFAULT '4',
`advertiser` varchar(120) DEFAULT NULL,
`image_url` varchar(255) DEFAULT NULL,
`tiks` int(5) DEFAULT NULL,
`buy_url` varchar(255) DEFAULT NULL,
`description` text,
`views` int(6) NOT NULL DEFAULT '1',
`title` varchar(100) DEFAULT NULL,
`brand` varchar(100) DEFAULT NULL,
`api` varchar(50) DEFAULT NULL,
`discount` int(2) DEFAULT NULL,
`black_list` smallint(1) DEFAULT '0',
`gender` tinyint(1) DEFAULT NULL,
`sort` tinyint(1) DEFAULT NULL,
`is_home` tinyint(1) DEFAULT NULL,
`is_quick_shop` tinyint(1) DEFAULT NULL,
`date_add` int(11) DEFAULT NULL,
`is_sale` tinyint(1) DEFAULT NULL,
`is_new` tinyint(1) DEFAULT NULL,
`is_brand_show` tinyint(1) DEFAULT NULL,
`date_modified` int(11) NOT NULL DEFAULT '0',
`modifier_id` smallint(2) DEFAULT NULL,
`modified` smallint(1) DEFAULT NULL,
`longitute` decimal(10,8) DEFAULT NULL,
`latitute` decimal(10,8) DEFAULT NULL,
`is_deal` tinyint(1) DEFAULT NULL,
`best_seller` tinyint(1) DEFAULT NULL,
`home_cat` int(2) DEFAULT NULL,
`end_date` int(11) DEFAULT NULL,
`temp_category` varchar(120) DEFAULT NULL,
`update_cron` smallint(1) unsigned DEFAULT '1',
`alias` varchar(120) DEFAULT NULL,
`found_by` text,
`fb_share` smallint(3) DEFAULT NULL,
`tw_share` smallint(3) DEFAULT NULL,
`pin_share` smallint(3) DEFAULT NULL,
`google_share` smallint(3) DEFAULT NULL,
`last_tiked` int(11) DEFAULT NULL,
`is_simple` smallint(1) DEFAULT '0',
`dealer_id` int(3) DEFAULT NULL,
`clicks` int(5) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `sku` (`sku`),
KEY `tiks` (`tiks`),
KEY `category` (`category`),
KEY `bests` (`best_seller`),
KEY `ne` (`is_new`),
KEY `qu` (`is_quick_shop`),
KEY `end_date` (`end_date`),
KEY `lat` (`latitute`),
KEY `lon` (`longitute`),
KEY `alias` (`alias`),
FULLTEXT KEY `title` (`title`),
FULLTEXT KEY `desc` (`description`),
FULLTEXT KEY `brand` (`brand`),
FULLTEXT KEY `advertiser` (`advertiser`),
FULLTEXT KEY `found_by` (`found_by`)
) ENGINE=MyISAM AUTO_INCREMENT=1861942 DEFAULT CHARSET=utf8
Assuming end_date is a date, convert the number into a date type using FROM_UNIXTIME()
SELECT *, 1 as distance
FROM deals
WHERE end_date > FROM_UNIXTIME(1381959736)
GROUP BY title
ORDER BY COALESCE(distance, 999999999), distance ASC
LIMIT 0, 3;
That way mysql doesn't have the cast every row's end_date to a string, which is very slow. It can also use an index if one exists, so make sure you have one:
create index deals_end_date on deals(end_date);
The goal is to synchronize the customers data and his table structure to our table structure.
The table structure is the following:
CREATE TABLE IF NOT EXISTS `BildSerie_W2L` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`BildserieID` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Prislista` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`BildKod` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`AvdKlassID` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Status` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=24 ;
CREATE TABLE IF NOT EXISTS `classes` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`client_id` int(10) unsigned NOT NULL,
`class` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`year` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`created_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`id`),
KEY `classes_client_id_foreign` (`client_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=9 ;
CREATE TABLE IF NOT EXISTS `Elever_W2L` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`X_ElevID` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Efternamn` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Förnamn` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`GataMedNr` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`PostNr` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Postort` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`TelefonMobil` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Epost1` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Epost2` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`X_AvdKlassId` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Status` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=18 ;
CREATE TABLE IF NOT EXISTS `Elev_Bild_BildSerie_W2L` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`Elev_Bild_Bildserie_ID` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`BildSerieID` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`ElevID` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Status` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=26 ;
CREATE TABLE IF NOT EXISTS `image_codes` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`client_id` int(10) unsigned DEFAULT NULL,
`code` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
`created_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`id`),
UNIQUE KEY `image_codes_code_unique` (`code`),
KEY `image_codes_client_id_foreign` (`client_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=20 ;
CREATE TABLE IF NOT EXISTS `image_series` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`image_code_id` int(10) unsigned NOT NULL,
`pricelist_id` int(10) unsigned NOT NULL DEFAULT '1',
`class_id` int(10) unsigned DEFAULT NULL,
`image_serie` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`slug` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`year` int(11) NOT NULL,
`created_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`id`),
KEY `image_series_image_code_id_foreign` (`image_code_id`),
KEY `image_series_pricelist_id_foreign` (`pricelist_id`),
KEY `image_series_class_id_foreign` (`class_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AUTO_INCREMENT=3 ;
The goal is to take the data in "BildSerie_W2L" and populate it in to image_series.
My query looks like this:
SELECT `BildSerie_W2L`.*, `classes`.id, `image_codes`.id as ImageID, `Elever_W2L`.Förnamn, `Elever_W2L`.Efternamn
FROM `BildSerie_W2L`
INNER JOIN `image_codes` ON `BildSerie_W2L`.BildKod = `image_codes`.code
INNER JOIN `classes` ON `BildSerie_W2L`.AvdKlassID = `classes`.class
INNER JOIN `Elev_Bild_BildSerie_W2L` ON `Elev_Bild_BildSerie_W2L`.BildSerieID = `BildSerie_W2L`.BildserieID
INNER JOIN `Elever_W2L` ON `Elever_W2L`.X_ElevID = `Elev_Bild_BildSerie_W2L`.ElevID
WHERE `BildSerie_W2L`.Status = 0
However it gives back duplicated results and not all the data in BildSerie_W2L...