I have a query which is generated via PHP framework core function. I don't have control to change the query. Therefore i will need to perform optimization on server side i.e. mysql to execute this query in efficient time. I have applied some indices but still it is taking around 4-5 seconds and ideally it should take 1-1.5 seconds. Following is the query:
(
SELECT rr.rt_bids_aos_quotes_relaos_quotes_idb AS so_id,
rr.sales_order_sequence sequence,
so.*
FROM rt_bids_aos_quotes_rel AS rr
INNER JOIN aos_quotes AS so ON so.id = rr.rt_bids_aos_quotes_relaos_quotes_idb
WHERE rr.deleted = 0
AND rr.rt_bids_aos_quotes_relrt_bids_ida='490395-403600-b'
)
UNION
(
SELECT ra.rt_bids_aos_quotes_altaos_quotes_idb AS so_id,
'' AS sequence,
so.*
FROM rt_bids_aos_quotes_alternate AS ra
INNER JOIN aos_quotes AS so ON so.id = ra.rt_bids_aos_quotes_altaos_quotes_idb
WHERE ra.deleted = 0
AND ra.rt_bids_aos_quotes_altrt_bids_ida='490395-403600-b'
)
Following is the Explain query result:Image_here
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY rt_bids_aos_quotes_rel const idx_rt_bids_aos_quotes_relrt_bids_ida_deleted,rt_bids_aos_quotes_rel_alt,idx_rt_bids_aos_quotes_rel_rt_bids_aos_quotes_relaos_quotes_idb idx_rt_bids_aos_quotes_relrt_bids_ida_deleted 113 const,const 1 NULL
1 PRIMARY so ALL NULL NULL NULL NULL 631950 Using where
2 UNION NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables
NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL Using temporary
Show create table results :
CREATE TABLE `rt_bids_aos_quotes_rel` (
`id` varchar(36) NOT NULL,
`date_modified` datetime DEFAULT NULL,
`deleted` tinyint(1) DEFAULT '0',
`rt_bids_aos_quotes_relrt_bids_ida` varchar(36) DEFAULT NULL,
`rt_bids_aos_quotes_relaos_quotes_idb` varchar(36) DEFAULT NULL,
`sales_order_sequence` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_rt_bids_aos_quotes_relrt_bids_ida_deleted` (`deleted`,`rt_bids_aos_quotes_relrt_bids_ida`),
KEY `rt_bids_aos_quotes_rel_alt` (`rt_bids_aos_quotes_relrt_bids_ida`,`rt_bids_aos_quotes_relaos_quotes_idb`),
KEY `idx_rt_bids_aos_quotes_rel_rt_bids_aos_quotes_relaos_quotes_idb` (`rt_bids_aos_quotes_relaos_quotes_idb`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `aos_quotes` (
`id` char(36) NOT NULL,
`name` varchar(255) DEFAULT NULL,
`date_entered` datetime DEFAULT NULL,
`date_modified` datetime DEFAULT NULL,
`modified_user_id` char(36) DEFAULT NULL,
`created_by` char(36) DEFAULT NULL,
`description` text,
`deleted` tinyint(1) DEFAULT '0',
`assigned_user_id` char(36) DEFAULT NULL,
`approval_issue` text,
`billing_account_id` char(36) DEFAULT NULL,
`billing_contact_id` char(36) DEFAULT NULL,
`billing_address_street` varchar(150) DEFAULT NULL,
`billing_address_city` varchar(100) DEFAULT NULL,
`billing_address_state` varchar(100) DEFAULT NULL,
`billing_address_postalcode` varchar(20) DEFAULT NULL,
`billing_address_country` varchar(255) DEFAULT NULL,
`shipping_address_street` varchar(150) DEFAULT NULL,
`shipping_address_city` varchar(100) DEFAULT NULL,
`shipping_address_state` varchar(100) DEFAULT NULL,
`shipping_address_postalcode` varchar(20) DEFAULT NULL,
`shipping_address_country` varchar(255) DEFAULT NULL,
`expiration` date DEFAULT NULL,
`number` int(11) NOT NULL,
`opportunity_id` char(36) DEFAULT NULL,
`template_ddown_c` text,
`total_amt` decimal(26,6) DEFAULT NULL,
`total_amt_usdollar` decimal(26,6) DEFAULT NULL,
`subtotal_amount` decimal(26,6) DEFAULT NULL,
`subtotal_amount_usdollar` decimal(26,6) DEFAULT NULL,
`discount_amount` decimal(26,6) DEFAULT NULL,
`discount_amount_usdollar` decimal(26,6) DEFAULT NULL,
`tax_amount` decimal(26,6) DEFAULT NULL,
`tax_amount_usdollar` decimal(26,6) DEFAULT NULL,
`shipping_amount` decimal(26,6) DEFAULT NULL,
`shipping_amount_usdollar` decimal(26,6) DEFAULT NULL,
`shipping_tax` varchar(100) DEFAULT NULL,
`shipping_tax_amt` decimal(26,6) DEFAULT NULL,
`shipping_tax_amt_usdollar` decimal(26,6) DEFAULT NULL,
`total_amount` decimal(26,6) DEFAULT NULL,
`total_amount_usdollar` decimal(26,6) DEFAULT NULL,
`currency_id` char(36) DEFAULT NULL,
`stage` varchar(100) DEFAULT 'Draft',
`term` varchar(100) DEFAULT NULL,
`terms_c` text,
`approval_status` varchar(100) DEFAULT NULL,
`invoice_status` varchar(100) DEFAULT 'Not Invoiced',
`subtotal_tax_amount` decimal(26,6) DEFAULT NULL,
`subtotal_tax_amount_usdollar` decimal(26,6) DEFAULT NULL,
`bid_id` char(36) DEFAULT NULL,
`alt` varchar(255) DEFAULT NULL,
`group_desc` longtext,
`hold` tinyint(1) DEFAULT '0',
`order_amount` decimal(26,2) DEFAULT NULL,
`order_description` longtext,
`status` varchar(100) DEFAULT NULL,
`type` varchar(255) DEFAULT NULL,
`contract_id` char(36) DEFAULT NULL,
`customer_discount` decimal(26,2) DEFAULT NULL,
`customer_markup` decimal(26,2) DEFAULT NULL,
`markup_inv_type` decimal(26,2) DEFAULT NULL,
`location_id` char(36) DEFAULT NULL,
`active` varchar(100) DEFAULT 'Active',
`city` varchar(255) DEFAULT NULL,
`rt_jobs_id` char(36) DEFAULT NULL,
`system_type` varchar(36) DEFAULT NULL,
`com_address` varchar(255) DEFAULT NULL,
`com_city` varchar(255) DEFAULT NULL,
`com_mapsco` varchar(255) DEFAULT NULL,
`com_state_zip` varchar(255) DEFAULT NULL,
`job_type` varchar(100) DEFAULT NULL,
`calc_labor` varchar(100) DEFAULT 'Item_Install_amt',
`comission` decimal(10,4) DEFAULT '0.0000',
`hourly_labor_rate` decimal(8,4) DEFAULT NULL,
`labor_percentage_of_price` decimal(12,2) DEFAULT NULL,
`overhead` decimal(12,4) DEFAULT '0.0000',
`profit` decimal(12,4) DEFAULT '0.0000',
`pricing_checkbox` tinyint(1) DEFAULT '0',
`pkg_package_id` char(36) DEFAULT NULL,
`dh_factor` decimal(6,2) DEFAULT '0.00',
`addl_builder_price` decimal(12,4) DEFAULT '0.0000',
`billing_notes` longtext,
`billing_notes_home` longtext,
`builder_percentage` decimal(12,4) DEFAULT '0.0000',
`discount` decimal(12,4) DEFAULT '0.0000',
`jobs_contact_homeowner_id` char(36) DEFAULT NULL,
`mortage` varchar(100) DEFAULT 'mortage',
`oh_percentage` decimal(12,2) DEFAULT '0.00',
`origin` varchar(255) DEFAULT NULL,
`package_items` decimal(10,2) DEFAULT NULL,
`package_items_unit_left` decimal(10,2) DEFAULT '0.00',
`selected_package_units` decimal(10,2) DEFAULT '0.00',
`jobs_account_superintendent_id` char(36) DEFAULT NULL,
`subdivision_selected_id` char(36) DEFAULT NULL,
`subdivision_selected_name` varchar(255) DEFAULT NULL,
`department` varchar(255) DEFAULT 'Dallas',
`locked` tinyint(1) DEFAULT '0',
`billed_amount` decimal(26,2) DEFAULT '0.00',
`billed_percentage` decimal(26,2) DEFAULT '0.00',
`retained_amount` decimal(26,2) DEFAULT '0.00',
`selected_package_amount` decimal(26,2) DEFAULT '0.00',
`builder_amount` decimal(26,2) DEFAULT '0.00',
`home_owner_amount` decimal(26,2) DEFAULT '0.00',
`builderContractAmount` decimal(26,2) DEFAULT '0.00',
`homeOwnerContractAmount` decimal(26,2) DEFAULT '0.00',
`ho_pct` decimal(26,6) DEFAULT '0.000000',
`builder_pct` decimal(26,6) DEFAULT '0.000000',
`labor_pct` decimal(26,6) DEFAULT '0.000000',
`mortgage` tinyint(1) DEFAULT '0',
`ho_order` tinyint(1) DEFAULT '0',
`home_owner_sale_order` tinyint(1) DEFAULT '0',
`directly_created_contract` tinyint(1) DEFAULT '0',
`crew_manager_id` char(36) DEFAULT NULL,
`estimate_date` date DEFAULT NULL,
`complete` tinyint(1) DEFAULT '0',
`complete_note` text,
`plan_estimate_date` date DEFAULT NULL,
`plan_manager_id` char(36) DEFAULT NULL,
`lock_bid_id` char(36) DEFAULT NULL,
`documents_id` text,
`no_item_change` tinyint(1) DEFAULT '0',
`tagged_at_yard` tinyint(1) DEFAULT '0',
`soap_created_so` tinyint(1) DEFAULT '0',
`soap_created_so_amount` decimal(10,2) DEFAULT '0.00',
`subcon_vendor` varchar(255) DEFAULT NULL,
`plan_complete` varchar(100) DEFAULT 'In Progress',
`plan_required` tinyint(1) DEFAULT '0',
`plan_important` varchar(255) DEFAULT '0',
`confirm` varchar(100) DEFAULT 'Unconfirm',
`designer_notes` text,
`plan_due_date` date DEFAULT NULL,
`plan_required_so` tinyint(1) DEFAULT '0',
`plan_request` tinyint(1) DEFAULT '0',
`material_hold_status` varchar(255) DEFAULT NULL,
`wh_warehouse_id` char(36) DEFAULT NULL,
`work_order_created` tinyint(1) DEFAULT '0',
`maintenance_status` varchar(100) DEFAULT NULL,
`classification_type` varchar(255) DEFAULT NULL,
`pricing_type` varchar(255) DEFAULT NULL,
`estimate_end_date` date DEFAULT NULL,
`install_date` date DEFAULT NULL,
`is_matched` tinyint(1) DEFAULT '0',
`builder_discount` decimal(10,2) DEFAULT '0.00',
`ho_discount` decimal(10,2) DEFAULT '0.00',
`locate_required` tinyint(1) DEFAULT '0',
`priority` varchar(100) DEFAULT NULL,
`is_no_charged` tinyint(1) DEFAULT '0',
`no_charge_reasons` varchar(255) DEFAULT NULL,
`no_charge_users` varchar(50) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_aos_quotes_type` (`type`),
KEY `idx_aos_quotes_rt_jobs_id` (`rt_jobs_id`),
KEY `idx_aos_quotes_id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE TABLE `rt_bids_aos_quotes_alternate` (
`id` varchar(36) NOT NULL,
`date_modified` datetime DEFAULT NULL,
`deleted` tinyint(1) DEFAULT '0',
`rt_bids_aos_quotes_altrt_bids_ida` varchar(36) DEFAULT NULL,
`rt_bids_aos_quotes_altaos_quotes_idb` varchar(36) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_rt_bids_aos_quotes_altrt_bids_ida_deleted` (`deleted`,`rt_bids_aos_quotes_altrt_bids_ida`),
KEY `rt_bids_aos_quotes_alt` (`rt_bids_aos_quotes_altrt_bids_ida`,`rt_bids_aos_quotes_altaos_quotes_idb`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
Please advise how can i improve it.
Your default charset for aos_quotes is latin1, but the default charset for the other tables is utf8. When you JOIN based on comparing two strings with different collations, they can't use the index, so they are forced to do a table-scan. I have no doubt that's what's slowing down your query.
When I use your tables as is, I get this EXPLAIN for the so tables:
*************************** 2. row ***************************
id: 1
select_type: PRIMARY
table: so
partitions: NULL
type: ALL
possible_keys: NULL
key: NULL
key_len: NULL
ref: NULL
rows: 1
filtered: 100.00
Extra: Using where
When I convert your aos_quotes table to utf8, I get this EXPLAIN for the so tables:
*************************** 2. row ***************************
id: 1
select_type: PRIMARY
table: so
partitions: NULL
type: const
possible_keys: PRIMARY,idx_aos_quotes_id
key: PRIMARY
key_len: 108
ref: const
rows: 1
filtered: 100.00
Extra: NULL
The "type: PRIMARY" is much better than "type: ALL".
So you need to convert your aos_quotes table to utf8.
See How do I change a MySQL table to UTF-8?
Other possible improvements:
A PRIMARY KEY is a UNIQUE key is a KEY. So KEY idx_aos_quotes_id (id) is redundant and could (should) be dropped.
I see VARCHAR(36); that usually smacks of random UUIDs. But your query shows something else. What is the situation. (The randomness of standard UUIDs is a performance problem in large tables.)
I see lots of TEXT columns and SELECT .. so.* fetching all of them. If you don't need them all, don't fetch them all. Each one is potentially an extra disk hit.
UNION defaults to UNION DISTINCT, which involves a de-dup pass. If you don't need that, then UNION ALL would be faster.
(Not a speed issue, just a readability issue.) Can you remove the clutter of the "rt_bids_aos_quotes_" prefix?
If rt_bids_aos_quotes_alternate and rt_bids_aos_quotes_rel are many:many mappings, see my blog for several performance tips.
decimal(26,6) takes 12 bytes. Do you need that much precision and range? Or would some other datatype do? Smaller -> more cacheable -> less I/O -> faster. (Rule of Thumb: DECIMAL(m,n) occupies about m/2 bytes. INT and FLOAT occupy exactly 4 bytes. TINYINT: 1 byte.)
In addition to fixing the CHARSET for id, fix the inconsistency between CHAR(36) and VARCHAR(36), especially if your ids are only 15 characters. (Again, space may lead to speed.)
I see UNIQUE(deleted, ida) -- Does this mean that there can be 1 or 2 rows with a given ida? If there can be only one (either deleted or not), then make (ida) the PK.
Related
I have a table containing more than 1 million data and I am trying to fetch using a date range. I have indexed the date column and still its searching the entire rows. Can someone give a best solution to fix this up.
/** Table Structure **/
CREATE TABLE `claim_history` (
`claim_history_id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`consultation_id` varchar(50) NOT NULL,
`member_id` int(11) unsigned DEFAULT NULL,
`card_number` char(18) DEFAULT NULL,
`member_name` varchar(200) DEFAULT NULL,
`network_code` int(11) unsigned NOT NULL DEFAULT '0',
`mobile_number` varchar(50) DEFAULT NULL,
`soap_number` varchar(50) DEFAULT NULL,
`diagnosis_id` varchar(50) DEFAULT NULL,
`diagnosis_code` varchar(50) DEFAULT NULL,
`diagnosis_description` text,
`activity_id` int(11) unsigned DEFAULT NULL,
`activity_code` varchar(50) DEFAULT NULL,
`activity_description` text,
`activity_comments` text,
`activity_quantity` int(11) unsigned NOT NULL DEFAULT '0',
`is_erx` tinyint(1) NOT NULL DEFAULT '0',
`lab_id` int(11) unsigned DEFAULT NULL,
`lab_name` varchar(100) DEFAULT NULL,
`session_no` tinyint(1) unsigned NOT NULL DEFAULT '0',
`net_amount` decimal(10,2) DEFAULT NULL,
`copay_pct` decimal(10,2) DEFAULT NULL,
`copay_amt` decimal(10,2) DEFAULT NULL,
`total_cost` decimal(10,2) DEFAULT NULL,
`is_edited` tinyint(1) unsigned DEFAULT '0',
`edit_comments` text,
`is_assigned_mcc` tinyint(1) DEFAULT '0'
`mcc_user` int(11) unsigned DEFAULT NULL,
`mcc_user_name` varchar(50) DEFAULT NULL,
`rule_name` varchar(200) DEFAULT NULL,
`mcu_assigned_time` timestamp NULL DEFAULT NULL,
`mcu_action_performed_time` timestamp NULL DEFAULT NULL,
`mcu_open_time` timestamp NULL DEFAULT NULL,
`is_reappealed` tinyint(1) unsigned NOT NULL DEFAULT '0',
`reappeal_count` int(11) unsigned NOT NULL,
`denial_code` varchar(50) DEFAULT NULL,
`denial_description` text,
`rejection_comments` text,
`justification_comment` text,
`justification_reply` text,
`justification_count` tinyint(1) unsigned NOT NULL DEFAULT '0',
`edit_comment_reply` text,
`is_referral` tinyint(1) unsigned NOT NULL DEFAULT '0',
`is_physiotherapy` tinyint(1) unsigned NOT NULL DEFAULT '0',
`facility_id` int(11) unsigned DEFAULT NULL,
`provider_code` varchar(50) DEFAULT NULL,
`provider_name` varchar(100) DEFAULT NULL,
`doctor_user_id` int(11) DEFAULT NULL,
`doctor_name` varchar(100) DEFAULT NULL,
`referral_doctor_name` varchar(100) DEFAULT NULL,
`referral_clinic_name` varchar(100) DEFAULT NULL,
`pic_id` int(11) unsigned DEFAULT NULL,
`pic_name` varchar(100) DEFAULT NULL,
`ig_id` int(11) unsigned DEFAULT NULL,
`ig_name` varchar(100) DEFAULT NULL,
`is_active` tinyint(1) NOT NULL DEFAULT '1',
`history_created_on` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`history_last_modified_on` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`created_on` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`created_by` int(11) unsigned NOT NULL DEFAULT '0',
`last_modified_on` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`last_modified_by` int(11) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`claim_history_id`),
KEY `idx_consultation_id` (`consultation_id`),
KEY `idx_card_number` (`member_id`,`card_number`),
KEY `idx_provider` (`facility_id`),
KEY `idx_soap_number` (`soap_number`),
KEY `idx_created_on` (`created_on`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;
/** SQL QUERY **/
SELECT fields FROM claim_history
WHERE created_on BETWEEN '2017-01-01 00:00:00'
AND '2017-05-01 00:00:00';
The query you provided has a syntax error.
Dependent on the range you put into your query, the query optimiser might use the index or not.
Try a really short range like one day and then use one year, you'll probably already see the difference.
Check this thread for more details:
MySQL ignores my index on a timestamp column
I am running the following query in MySQL Workbench:
UPDATE directorylistings INNER JOIN postcodelatlng ON PostalCode = postcode
set directoryListings.Latitude = postcodelatlng.latitude, directoryListings.Longitude = postcodelatlng.longitude
WHERE PostalCode = postcode;
Table directorylistings has 9000 records, postcodelatlng has nearly 2 million records.
directorylistings has a postcode (zip code) field and I'm looking this up in postcodelatlng to retrieve and update the latitude and longitude fields in directorylistings.
Ater timeout issues I found I should change DBMS connection read time out, I set this to 36000 (10 hours). The query has been running of 5 hours so far.
My question is, is this time reasonable? Should I be indexing something? Could the query be optimised?
I did make a 2 record version of the postcodelatlng and ran the query, 16 rows were correctly changed in 0.04 seconds, so I know the query is at least workable.
I have looked at a number of Q/A's on indexing and optimising, but it's taken me over a day to get this far and if it fails again I would like to know what approach to follow. I can't keep trying things and waiting hours for the failure! I bet this query should really take seconds. I'm using a MacBook Pro with a 2.5GHz i7, 16 GB RAM and ssd drive.
Many Thanks.
cpu is at 99%. I aborted after 12 hours! to run the show create tables:
CREATE TABLE `postcodelatlng` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`postcode` varchar(8) NOT NULL,
`latitude` decimal(18,15) NOT NULL,
`longitude` decimal(18,15) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=1734490 DEFAULT CHARSET=latin1
CREATE TABLE `directorylistings` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`Active` tinyint(1) DEFAULT NULL,
`Tag11` tinyint(1) DEFAULT NULL,
`Minyan` tinyint(1) DEFAULT NULL,
`Latitude` double DEFAULT NULL,
`Longitude` double DEFAULT NULL,
`AffiliatedDate` double DEFAULT NULL,
`RegisteredDate` datetime DEFAULT NULL,
`AccountID` varchar(10) DEFAULT NULL,
`ShowListing` tinyint(1) DEFAULT NULL,
`Bloguser` tinyint(1) DEFAULT NULL,
`Trusted` tinyint(1) DEFAULT NULL,
`ClassifiedUser` tinyint(1) DEFAULT NULL,
`HostingID` int(11) DEFAULT NULL,
`EndDate` double DEFAULT NULL,
`EndPromoDate` datetime DEFAULT NULL,
`BaseID` int(11) DEFAULT NULL,
`Banner` varchar(50) DEFAULT NULL,
`PrivateListing` tinyint(1) DEFAULT NULL,
`Reserved1` tinyint(1) DEFAULT NULL,
`Reserved2` tinyint(1) DEFAULT NULL,
`Reserved3` tinyint(1) DEFAULT NULL,
`Reserved4` tinyint(1) DEFAULT NULL,
`Reserved5` tinyint(1) DEFAULT NULL,
`CommunityOrg` tinyint(1) DEFAULT NULL,
`Tag1` tinyint(1) DEFAULT NULL,
`Tag2` tinyint(1) DEFAULT NULL,
`Tag3` tinyint(1) DEFAULT NULL,
`Tag4` tinyint(1) DEFAULT NULL,
`Tag5` tinyint(1) DEFAULT NULL,
`Tag6` tinyint(1) DEFAULT NULL,
`Tag7` tinyint(1) DEFAULT NULL,
`Tag8` tinyint(1) DEFAULT NULL,
`Tag9` tinyint(1) DEFAULT NULL,
`Tag10` tinyint(1) DEFAULT NULL,
`Tag12` tinyint(1) DEFAULT NULL,
`Tag13` tinyint(1) DEFAULT NULL,
`Tag14` tinyint(1) DEFAULT NULL,
`ShomerShabbos` tinyint(1) DEFAULT NULL,
`CategoryID1` int(11) DEFAULT NULL,
`CategoryID2` int(11) DEFAULT NULL,
`CategoryID3` int(11) DEFAULT NULL,
`CategoryID4` int(11) DEFAULT NULL,
`CategoryID5` int(11) DEFAULT NULL,
`CategoryID6` int(11) DEFAULT NULL,
`HiddenCategory` int(11) DEFAULT NULL,
`Company` varchar(100) DEFAULT NULL,
`Slogan` varchar(250) DEFAULT NULL,
`Description` varchar(2000) DEFAULT NULL,
`CharShown` int(11) DEFAULT NULL,
`Address` varchar(100) DEFAULT NULL,
`Unit` varchar(50) DEFAULT NULL,
`Address2` varchar(150) DEFAULT NULL,
`Streetno` varchar(50) DEFAULT NULL,
`Building` varchar(50) DEFAULT NULL,
`Parade` varchar(50) DEFAULT NULL,
`Locality` varchar(50) DEFAULT NULL,
`City` varchar(50) DEFAULT NULL,
`Province` varchar(50) DEFAULT NULL,
`PostalCode` varchar(10) DEFAULT NULL,
`Country` varchar(25) DEFAULT NULL,
`PhoneNumber` varchar(70) DEFAULT NULL,
`Ext` varchar(25) DEFAULT NULL,
`PhoneNumber2` varchar(70) DEFAULT NULL,
`FaxNumber` varchar(30) DEFAULT NULL,
`CellNumber` varchar(30) DEFAULT NULL,
`WebUrl` varchar(100) DEFAULT NULL,
`Email` varchar(75) DEFAULT NULL,
`Password` varchar(30) DEFAULT NULL,
`ContactName` varchar(100) DEFAULT NULL,
`Keywords` varchar(355) DEFAULT NULL,
`Dcount` int(11) DEFAULT NULL,
`LocationID` int(11) DEFAULT NULL,
`Comments` varchar(1500) DEFAULT NULL,
`ApplicantName` varchar(200) DEFAULT NULL,
`ApplicantPhone` varchar(200) DEFAULT NULL,
`ApplicantEmail` varchar(200) DEFAULT NULL,
`Photo1` varchar(250) DEFAULT NULL,
`pRating` int(11) DEFAULT NULL,
`sRating` int(11) DEFAULT NULL,
`vRating` int(11) DEFAULT NULL,
`Private` tinyint(1) NOT NULL DEFAULT '0',
`VeryPrivate` tinyint(1) NOT NULL DEFAULT '0',
`SemiSecure` tinyint(1) NOT NULL DEFAULT '0',
`Secure` tinyint(1) NOT NULL DEFAULT '0',
`Secret` tinyint(1) NOT NULL DEFAULT '0',
`TrustHigh` tinyint(1) NOT NULL DEFAULT '0',
`FL` tinyint(1) NOT NULL DEFAULT '1',
`GJ` tinyint(1) NOT NULL DEFAULT '1',
`Recommended` tinyint(1) NOT NULL DEFAULT '0',
`NonGeo` tinyint(1) NOT NULL DEFAULT '0',
`Edited` datetime DEFAULT NULL,
`Editor` varchar(40) DEFAULT NULL,
PRIMARY KEY (`ID`),
UNIQUE KEY `ID_UNIQUE` (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=27167 DEFAULT CHARSET=utf8
mysql Ver 14.14 Distrib 5.1.73, for redhat-linux-gnu (x86_64) using readline 5.1
I am taking over a project. It is very old and the original programmer is long gone. No one has any idea why certain decisions were made.
The following query runs (on my Mac) in 9.5 seconds but if I remove the last JOIN then it drops to 2.5 seconds. What is wrong with that last JOIN?
select `ttl`.`id` AS `id`,
`ttl`.`name` AS `name`,
`ttl`.`updated_at` AS `last_update_on`,
`ttl`.`user_id` AS `list_creator`,
`ttl`.`retailer_nomination_list` AS `nomination_list`,
`ttl`.`created_at` AS `created_on`,
count(distinct `tlb`.`title_id`) AS `title_count`
from `haha_title_lists` `ttl`
left join `haha_title_list_to_users` `tltu` on((`ttl`.`id` = `tltu`.`title_list_id`))
left join `users` `u` on((`tltu`.`user_id` = `u`.`id`))
left join `users` `u2` on((`tltu`.`user_id` = `u2`.`id`))
left join `haha_title_list_to_venues` `tlv` on((`ttl`.`id` = `tlv`.`title_list`))
left join `haha_venue_properties` `tvp` on((`tlv`.`venue_id` = `tvp`.`id`))
join `haha_title_list_to_books` `tlb` on((`ttl`.`id` = `tlb`.`title_list_id`))
join `wawa_title` `ot` on((`tlb`.`title_id` = `ot`.`title_id`))
group by `ttl`.`id`;
The tables:
CREATE TABLE `haha_title_list_to_books` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`title_id` int(11) NOT NULL,
`title_list_id` int(11) NOT NULL,
`sdk` varchar(15) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`created_at` datetime NOT NULL,
`promo_start_date` date DEFAULT NULL,
`promo_end_date` date DEFAULT NULL,
`promo_price` float DEFAULT NULL,
`confirmations` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`nominations` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`title_note` text COLLATE utf8_unicode_ci,
`executed` int(11) DEFAULT NULL,
`event_created` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_promo_start_date` (`promo_start_date`),
KEY `idx_promo_end_date` (`promo_end_date`),
KEY `idx_title_list_to_books_title_id` (`title_id`),
KEY `idx_title_list_to_books_title_list_id` (`title_list_id`)
) ENGINE=MyISAM AUTO_INCREMENT=21847 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
and:
CREATE TABLE `wawa_title` (
`title_id` int(11) NOT NULL AUTO_INCREMENT,
`title` varchar(200) DEFAULT NULL,
`title_alpha` varchar(25) NOT NULL,
`display_title` varchar(200) NOT NULL,
`subtitle` text NOT NULL,
`sdk10` varchar(13) DEFAULT '',
`sdk13` varchar(15) DEFAULT NULL,
`primary_sdk13` varchar(15) DEFAULT NULL,
`asin` varchar(10) DEFAULT NULL,
`pub_season` varchar(15) NOT NULL,
`pub_year` varchar(15) NOT NULL,
`bisac1` varchar(15) NOT NULL,
`bisac2` varchar(15) NOT NULL,
`bisac3` varchar(15) NOT NULL,
`barcode` varchar(30) DEFAULT NULL,
`dewey_decimal` varchar(15) NOT NULL,
`lib_of_congress` varchar(15) NOT NULL,
`spanish_language` tinyint(4) NOT NULL,
`target_audience` tinyint(3) unsigned DEFAULT NULL,
`language` varchar(20) DEFAULT NULL,
`edition` varchar(45) DEFAULT NULL,
`pages` int(11) DEFAULT NULL,
`number_in_series` int(11) DEFAULT NULL,
`trimsize` varchar(10) DEFAULT NULL,
`filesize` varchar(10) DEFAULT NULL,
`duration_hours` int(11) DEFAULT NULL,
`duration_minutes` int(11) DEFAULT NULL,
`discs` int(11) DEFAULT NULL,
`download` date DEFAULT NULL,
`size_unit` varchar(15) NOT NULL DEFAULT '',
`digitization_date` date NOT NULL,
`us_on_sale_date` date NOT NULL,
`aus_on_sale_date` date NOT NULL,
`can_on_sale_date` date NOT NULL,
`uk_on_sale_date` date NOT NULL,
`us_list_price` varchar(10) NOT NULL,
`aus_list_price` varchar(10) NOT NULL,
`can_list_price` varchar(10) NOT NULL,
`uk_list_price` varchar(10) NOT NULL,
`isPrimary` varchar(1) DEFAULT NULL,
`modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`modifier` int(11) NOT NULL,
`activated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`active` varchar(3) NOT NULL DEFAULT 'N',
`flagged_string` text,
`created` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`assets_id` varchar(20) DEFAULT NULL,
`book_details` text,
`book_keynote` text,
`exclude_goodreads` char(1) NOT NULL DEFAULT 'N',
`series_description` text,
`review_quote1` text,
`territory_id` int(11) DEFAULT '27',
`featured_newsletter_id` tinyint(3) unsigned DEFAULT '0',
`retailer_discovery_check` datetime DEFAULT NULL,
`suppress_retailer_approval` tinyint(1) DEFAULT '0',
`suppress_retailer_approval_reason` varchar(255) DEFAULT NULL,
`ebb_description` text CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`slug` varchar(150) DEFAULT NULL,
`legacy_slug` varchar(150) DEFAULT NULL,
`us_agency_price` varchar(10) DEFAULT NULL,
`firebrand_title_id` int(11) DEFAULT NULL,
`ebb_label` varchar(200) DEFAULT NULL,
`ebb_end_sale_date` date DEFAULT NULL,
`ebb_downprice` decimal(10,2) DEFAULT NULL,
`book_club` varchar(1) DEFAULT NULL,
`best_seller` varchar(1) DEFAULT NULL,
`award_winner` varchar(1) DEFAULT NULL,
`discovery` char(1) NOT NULL DEFAULT 'Y',
`narrator_id` int(11) DEFAULT NULL,
`suppress_series_data` varchar(255) DEFAULT NULL,
PRIMARY KEY (`title_id`),
KEY `active_index` (`active`),
KEY `fk_title_series_id_idx` (`series_id`),
KEY `series_id` (`series_id`),
KEY `idx_title_sdk13` (`sdk13`),
KEY `idx_title_active_isprimary` (`active`,`isPrimary`),
KEY `bisac1` (`bisac1`),
KEY `bisac2` (`bisac2`),
KEY `bisac3` (`bisac3`),
KEY `idx_primary_sdk13` (`primary_sdk13`),
KEY `idx_territory_id` (`territory_id`),
CONSTRAINT `fk_title_series_id` FOREIGN KEY (`series_id`) REFERENCES `wawa_series` (`series_id`) ON DELETE NO ACTION ON UPDATE NO ACTION
) ENGINE=InnoDB AUTO_INCREMENT=19700 DEFAULT CHARSET=utf8 |
If I remove this line:
join `wawa_title` `ot` on((`tlb`.`title_id` = `ot`.`title_id`))
The query speed drops from 9.5 seconds to 2.5 seconds. Not great, but a huge improvement.
And yet, both tables have indexes on table_id, so why would that line be a problem?
I notice that one table is InnoDB and the other is MyISAM. Would that have an effect?
Do not JOIN to tables that you don't use.
A JOIN often "explodes" the number of rows, then a GROUP BY like you have reels in the number of rows. To see this, leave all the JOINs there, but remove the GROUP BY. See how many rows you get.
To avoid part of that explosion, change
count(distinct `tlb`.`title_id`) AS `title_count`
to
( SELECT count(distinct `title_id`)
FROM `haha_title_list_to_books`
WHERE `ttl`.`id` = `title_list_id`
) AS `title_count`
and remove the current JOIN to tlb.
Mixing MyISAM and InnoDB should not have any direct impact on this SELECT. However, you should consider moving all of your tables to InnoDB.
I have just included an index to a MyISAM table and the cardinality appears to be 1 million something which is exactly the number of rows in the table. In fact it needed to be 315 because distinct count returns 315. And then I have realized that there are 3 other such indexes (equal to tables's row size), and all their distinct counts are way below. So, what could be the reason these indexes' carnalities are equal to table's row size?
Note: There are 97 columns and 27 indexes.
The table:
CREATE TABLE `isemri` (
`RECNO` int(12) NOT NULL AUTO_INCREMENT,
`MUSTERI_RECNO` int(11) NOT NULL,
`MUSTERI_PER_RECNO` int(11) DEFAULT NULL,
`YIL` smallint(4) unsigned zerofill NOT NULL DEFAULT '0000',
`ISEMRI_NO` int(6) NOT NULL DEFAULT '0',
`BOLGE` int(6) unsigned NOT NULL,
`URUN_BARKOD` varchar(13) NOT NULL DEFAULT '''''',
`ISTEK_TIPI` tinyint(1) DEFAULT '0',
`MUSTERI_NO` bigint(20) DEFAULT NULL,
`SERVIS_CAGRI_TAR` datetime DEFAULT NULL,
`MUSTERI_PER` varchar(30) DEFAULT NULL,
`SERVIS_BAS_TAR` datetime DEFAULT NULL,
`SERVIS_BIT_TAR` datetime DEFAULT NULL,
`ISTEGI_ALAN` varchar(15) DEFAULT NULL,
`URUN_CIHAZ` smallint(6) DEFAULT NULL,
`URUN_CIHAZ_ADI` varchar(40) DEFAULT NULL,
`URUN_MARKA` smallint(6) DEFAULT NULL,
`URUN_MARKA_ADI` varchar(20) DEFAULT NULL,
`URUN_MODEL` varchar(35) DEFAULT NULL,
`URUN_MODEL_ADI` varchar(150) DEFAULT NULL,
`URUN_SERI_NO` varchar(100) DEFAULT NULL,
`SERVIS_VEREN` varchar(50) DEFAULT NULL,
`SERVIS_TURU` tinyint(2) NOT NULL DEFAULT '1',
`ARIZA_ACIKLAMASI` mediumtext,
`ARIZASERVIS_ACIKLAMA` varchar(250) DEFAULT NULL,
`ACILIYET_DERECESI` tinyint(2) NOT NULL DEFAULT '2',
`SERVIS_DURUMU` tinyint(2) NOT NULL DEFAULT '1',
`GERI_BILDIRIM` tinyint(1) NOT NULL DEFAULT '0',
`KON_GEL_DON_TAR` datetime DEFAULT NULL,
`GERI_BILDIRIM_TARIHI` datetime DEFAULT NULL,
`TAHMINI_TESLIM_TARIHI` datetime DEFAULT NULL,
`YAPILANLAR` varchar(255) DEFAULT NULL,
`YAPILANLAR_M` mediumtext,
`SERVIS_YERI` varchar(20) DEFAULT NULL,
`TOPLAM` float(12,2) DEFAULT NULL,
`KDV_TOP` float(12,2) DEFAULT NULL,
`GENEL_TOPLAM` float(12,2) DEFAULT NULL,
`INSERT_USER` varchar(15) DEFAULT NULL,
`GIRIS_KARGO_NO` varchar(10) DEFAULT NULL,
`INSERT_DATE` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`UPDATE_USER` varchar(10) DEFAULT NULL,
`UPDATE_DATE` datetime DEFAULT NULL,
`BARKOD` int(6) NOT NULL DEFAULT '0',
`SERVIS_BITIS` tinyint(1) DEFAULT '0',
`GARANTI_BELGENO` varchar(12) DEFAULT NULL,
`GARANTI_TARIHI` date DEFAULT NULL,
`EL_ISEMRI_NO` int(11) DEFAULT '0',
`ONAY_TARIHI` datetime DEFAULT NULL,
`ONAYLAYAN` varchar(30) DEFAULT NULL,
`ALAN1` varchar(20) DEFAULT NULL,
`ALAN2` varchar(20) DEFAULT NULL,
`ALAN3` varchar(20) DEFAULT NULL,
`ALAN4` varchar(20) DEFAULT NULL,
`DOLAR_KURU` float DEFAULT '2',
`EURO_KURU` float DEFAULT '2',
`KDV` tinyint(1) DEFAULT '0',
`BILGILENDIR` tinyint(1) DEFAULT '0',
`TESLIM_SEKLI` varchar(10) DEFAULT NULL,
`TESLIM_NOTU` varchar(150) DEFAULT NULL,
`ARIZA_TIP_NO` smallint(6) DEFAULT NULL,
`ARIZASERVIS_TIP` smallint(6) DEFAULT NULL,
`GETIREN_KARGO_FRM` varchar(50) DEFAULT NULL,
`SERV_ALAN1` varchar(25) DEFAULT NULL,
`SERV_ALAN2` varchar(25) DEFAULT NULL,
`DOSYA__` varchar(250) DEFAULT NULL,
`HAKEDIS_DURUM` tinyint(1) unsigned DEFAULT '0',
`HAKEDIS_FATURANO` int(11) DEFAULT NULL,
`HAKEDIS_FATURA_DURUM` int(1) DEFAULT '0',
`ONAY_DURUM` tinyint(1) unsigned DEFAULT '0',
`MONTAJ_KODU` int(6) DEFAULT NULL,
`urun_id` int(11) DEFAULT NULL,
`gondKargoFirmasi` tinyint(2) DEFAULT NULL,
`gondKargoNo` int(11) DEFAULT NULL,
`gondIrsNo` varchar(30) DEFAULT NULL,
`gondGelTar` date DEFAULT NULL,
`BAYI_KODU` varchar(50) DEFAULT '0',
`HAKEDIS_TARIHI` datetime DEFAULT NULL,
`HAKEDIS_ONAY_PERSONEL` varchar(30) DEFAULT NULL,
`HURDA_CTRL` tinyint(2) DEFAULT '0',
`DOSYA` tinyint(2) DEFAULT '0',
`RANDEVU_TARIHI` datetime DEFAULT NULL,
`MUSTERI_ACIKLAMA` varchar(200) DEFAULT NULL,
`PLASIYER` varchar(50) DEFAULT NULL,
`FIRMA_SERINO` varchar(20) DEFAULT NULL,
`MONTAJ_KART_NO` varchar(50) DEFAULT NULL,
`SERINO2` varchar(50) DEFAULT NULL,
`SATICI_FIRMA` varchar(50) DEFAULT NULL,
`URUN_DEGISIM_DURUMU` int(11) DEFAULT NULL,
`URUN_HASAR_DURUMU` int(6) DEFAULT NULL,
`BOLGE_2` varchar(12) DEFAULT NULL,
`KILIT` int(2) DEFAULT NULL,
`ON_ONAY` int(11) DEFAULT NULL,
`RANDEVU_BITIS_TARIHI` datetime DEFAULT NULL COMMENT 'iş emri randavü bitiş tarihi',
`URUN_KALITE` tinyint(4) DEFAULT NULL,
`BOLGE_X` varchar(255) DEFAULT NULL,
`REWORK` enum('0','1') DEFAULT '0' COMMENT 'Rework olup olmadığını gösteriyor.',
PRIMARY KEY (`RECNO`),
KEY `MUSTERI_NO` (`MUSTERI_NO`) USING BTREE,
KEY `EL_ISEMRI_NO` (`EL_ISEMRI_NO`) USING BTREE,
KEY `MUSTERI_RECNO` (`MUSTERI_RECNO`) USING BTREE,
KEY `ISEMRI_NO` (`ISEMRI_NO`) USING BTREE,
KEY `ALAN1` (`ALAN1`) USING BTREE,
KEY `URUN_SERI_NO` (`URUN_SERI_NO`) USING BTREE,
KEY `SERVIS_TURU` (`SERVIS_TURU`) USING BTREE,
KEY `SERVIS_DURUMU` (`SERVIS_DURUMU`) USING BTREE,
KEY `ISTEK_TIPI` (`ISTEK_TIPI`) USING BTREE,
KEY `urun_id` (`urun_id`) USING BTREE,
KEY `URUN_CIHAZ` (`URUN_CIHAZ`) USING BTREE,
KEY `HAKEDIS_DURUM` (`HAKEDIS_DURUM`) USING BTREE,
KEY `HAKEDIS_TARIHI` (`HAKEDIS_TARIHI`) USING BTREE,
KEY `HAKEDIS_ONAY_PERSONEL` (`HAKEDIS_ONAY_PERSONEL`) USING BTREE,
KEY `SERVIS_BAS_TAR` (`SERVIS_BAS_TAR`) USING BTREE,
KEY `BOLGE_2` (`BOLGE_2`) USING BTREE,
KEY `URUN_MARKA` (`URUN_MARKA`) USING BTREE,
KEY `URUN_MODEL` (`URUN_MODEL`) USING BTREE,
KEY `MUSTERI_PER` (`MUSTERI_PER`) USING BTREE,
KEY `MONTAJ_KODU` (`MONTAJ_KODU`) USING BTREE,
KEY `BOLGE` (`BOLGE`),
KEY `HAKEDIS_FATURANO` (`HAKEDIS_FATURANO`),
KEY `REWORK` (`REWORK`) USING BTREE,
KEY `SERVIS_BITIS` (`SERVIS_BITIS`) USING BTREE,
KEY `HAKEDIS_FATURA_DURUM` (`HAKEDIS_FATURA_DURUM`),
KEY `MUSTERI_PER_RECNO` (`MUSTERI_PER_RECNO`) USING BTREE
) ENGINE=MyISAM AUTO_INCREMENT=1335476 DEFAULT CHARSET=utf8 PACK_KEYS=0 ROW_FORMAT=DYNAMIC
SHOW TABLE STATUS:
Name: isemri
Engine: MyISAM
Version: 10
Rpw_format: Dynamic
Rows: 1328150
Avg_row_length: 459
Data_length: 610624628
Max_data_length: 281474976710655
Index_length: 420404224
Data_free: 0
Auto_icrement: 1335384
Create_time: 2016-02-12 16:22:12
Update_time: 2016-02-13 10:43:59
Check_time: 2016-02-12 17:52:23
Collation: utf8_general_ci
Checksum:
Create_options: pack_keys=0 row_format=DYNAMIC
Comment:
Distinct count:
URUN_SERI_NO: 1012754
SERVIS_BAS_TAR: 1300066
BOLGE_2: 5
MUSTERI_PER_RECNO: 319
I have a table that was inherited from a different system and one of the fields has the UUID function enabled so no matter what ID I generate and try to insert, the table creates a completely different one automatically.
I would like to use a PHP function to create the ID instead but I can't workout how to remove the UUID function from the field.
I am not sure what information will be needed to help you so please feel free to ask.
The field in question is
id, char(36)
Table definaition is..
CREATE TABLE `users` (
`id` char(36) NOT NULL,
`user_name` varchar(60) default NULL,
`user_hash` varchar(32) default NULL,
`diary_weekly_view` varchar(500) NOT NULL,
`week_start` date NOT NULL,
`diary_monthly_view` varchar(300) NOT NULL,
`diary_view` int(1) NOT NULL default '1',
`account_search` varchar(1000) NOT NULL,
`cases_search` varchar(500) NOT NULL,
`serials_search` varchar(500) NOT NULL,
`type` varchar(8) NOT NULL,
`email` varchar(255) NOT NULL,
`notes` varchar(3000) NOT NULL,
`authenticate_id` varchar(100) default NULL,
`sugar_login` tinyint(1) default '1',
`first_name` varchar(30) default NULL,
`last_name` varchar(30) default NULL,
`reports_to_id` char(36) default NULL,
`is_admin` tinyint(1) default '0',
`receive_notifications` tinyint(1) default '1',
`date_entered` datetime NOT NULL,
`date_modified` datetime NOT NULL,
`modified_user_id` char(36) default NULL,
`created_by` char(36) default NULL,
`phone_office` varchar(50) default NULL,
`phone_mobile` varchar(50) default NULL,
`status` varchar(25) default NULL,
`address_street` varchar(150) default NULL,
`address_city` varchar(100) default NULL,
`address_state` varchar(100) default NULL,
`address_country` varchar(25) default NULL,
`address_postalcode` varchar(9) default NULL,
`user_preferences` text,
`deleted` tinyint(1) NOT NULL default '0',
`portal_only` tinyint(1) default '0',
`employee_status` varchar(25) default NULL,
`is_group` tinyint(1) default '0',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf
OK, sussed it, I removed the index and then recreated the index with INDEX