Related
I'm having trouble optimizing this really big query and I can't change the table structure except creating additional indexes and small adjustments.
SELECT
'Fattura Prodotti Postali' AS `type`,
SUM(dpd.qta) AS `products_count_quantity`,
COUNT(dpd.IDlavorazione_dett) AS `products_count`,
GROUP_CONCAT(DISTINCT dp.prod_totali - CAST(dp.opzione1 AS UNSIGNED) SEPARATOR ' |-| ') AS `process_products_count`,
GROUP_CONCAT(DISTINCT dp.IDdistinta) AS `product_code`,
GROUP_CONCAT(DISTINCT dp.data_distinta) AS `process_date`,
GROUP_CONCAT(DISTINCT dp.IDesito) AS `process_status_id`,
GROUP_CONCAT(DISTINCT dp.note) AS `process_note`,
GROUP_CONCAT(DISTINCT dp.IDlavorazione) AS `unique_id`,
SUM(dpd.tariffa) AS `products_total`,
SUM(IF(o.IDdoc IS NULL,1,0)*dpd.qta) AS `tobill_count_quantity`,
SUM(IF(NOT o.IDdoc IS NULL,1,0)*dpd.qta) AS `billed_count_quantity`,
SUM(IF(o.IDdoc IS NULL,1,0)) AS `tobill_count`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa,0)) AS `tobill_total`,
SUM(IF(NOT o.IDdoc IS NULL,1,0)) AS `billed_count`,
SUM(IF(NOT o.IDdoc IS NULL, dpd.tariffa,0)) AS `billed_total`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa,0)-IF(NOT o.IDdoc IS NULL, dpd.tariffa,0)) AS `bill_diff`,
COUNT(dpd.IDlavorazione_dett) AS `products_count`,
SUM(dpd.tariffa) AS `products_total`,
SUM(dpd.tariffa*(dpd.iva/100)) AS `products_vat`,
SUM(dpd.tariffa*(dpd.sconto/100.0)) AS `products_discount`,
SUM(dpd.tariffa*(1.0-dpd.sconto/100.0)*(dpd.iva/100)) AS `products_discount_vat`,
SUM(dpd.tariffa*(1+(dpd.iva/100.0))) AS `products_total_vat`,
SUM(ROUND(dpd.tariffa*(1.0-dpd.sconto/100.0),5)) AS `products_total_discount`,
SUM(dpd.tariffa*(1.0-dpd.sconto/100.0)*(1+(dpd.iva/100.0))) AS `products_total_discount_vat`,
SUM(dpd.qta) AS `products_quantity`,
SUM(IF(o.IDdoc IS NULL,1,0)) AS `tobill_count`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa,0)) AS `tobill_total`,
SUM(IF(o.IDdoc IS NULL, dpd.tariffa*(1+(dpd.iva/100.0)),0)) AS `tobill_total_vat`,
SUM(IF(NOT o.IDdoc IS NULL,1,0)) AS `billed_count`,
SUM(IF(NOT o.IDdoc IS NULL, dpd.tariffa,0)) AS `billed_total`,
SUM(IF(NOT o.IDdoc IS NULL, dpd.tariffa*(1+(dpd.iva/100.0)),0)) AS `billed_total_vat`
FROM doc_prodottipostali_dett dpd
INNER JOIN tracking t ON (dpd.IDlavorazione_dett=t.product_id)
LEFT JOIN prodotti_pp ppp ON (dpd.IDprodotto=ppp.IDprodotto)
LEFT JOIN categorie_pp cpp ON (ppp.categoria=cpp.IDcategoria)
LEFT JOIN categorie_pp cppp ON (CAST(dpd.IDcategoria AS UNSIGNED)=cppp.IDcategoria)
INNER JOIN doc_prodottipostali dp ON (dpd.IDlavorazione=dp.IDlavorazione)
LEFT JOIN ordini o ON (dpd.IDfattura=o.IDdoc)
WHERE
(
(dp.tipo = 'PT' AND t.date >= '2022-05-15 00:00:00' AND t.date <= '2022-07-26 23:59:59') AND
((IF(dpd.IDprodotto>0, cpp.codice, IF(CAST(dpd.IDcategoria AS UNSIGNED)>0, cppp.codice, 'NONE')) NOT IN ('LAW','AR','CAD','EMESSOCAD') OR IF(dpd.IDprodotto>0, cpp.codice, IF(CAST(dpd.IDcategoria AS UNSIGNED)>0, cppp.codice, 'NONE')) IS NULL)) AND
((t.last IN (-1,2,3,6,7,10,11,34,35,130,131,258,514,4098,4354,8194,8450))) AND
(((dp.opzione2 = 'PI' AND dp.data_distinta < '2022-08-25')) OR ((dp.data_distinta >= '2022-08-25')) OR ((o.IDdoc >= '1'))) AND
(((t.last IN (-1,2,3,6,7,10,11,34,35,130,131,258,514,4098,4354,8194,8450))))
) AND
(
((NOT t.last IN (-1,4,5,6,7,20,21,132,133,149,516,532,1028,1157,8197)))
)
GROUP BY dpd.IDlavorazione
HAVING (1=1 AND ((tobill_count > '0')))
ORDER BY dp.data_distinta ASC;
The create tables are as following:
CREATE TABLE `doc_prodottipostali_dett` (
`IDlavorazione_dett` int(11) NOT NULL AUTO_INCREMENT,
`IDlavorazione` int(11) NOT NULL,
`IDdistinta` varchar(45) NOT NULL,
`IDcategoria` varchar(45) NOT NULL,
`IDprodotto` int(11) NOT NULL,
`codiceabarre` varchar(255) NOT NULL,
`codiceavviso` varchar(255) NOT NULL DEFAULT '',
`IDvettore` int(11) NOT NULL,
`IDlistino` int(11) NOT NULL,
`rif` varchar(45) NOT NULL,
`IDmittente` int(11) NOT NULL,
`IDdestinatario` int(11) NOT NULL,
`ufficio_mittente` varchar(45) NOT NULL,
`nome_lavoro` varchar(255) NOT NULL,
`IDpostino` int(11) DEFAULT NULL,
`note` longtext NOT NULL,
`allegati` char(1) NOT NULL,
`utente` varchar(45) NOT NULL,
`peso` decimal(10,5) NOT NULL,
`tariffa` decimal(10,5) NOT NULL,
`sconto` float NOT NULL DEFAULT 0,
`iva` int(11) NOT NULL,
`qta` int(11) NOT NULL,
`am` char(1) NOT NULL,
`cp` char(1) NOT NULL,
`eu` char(1) NOT NULL,
`aa` char(10) NOT NULL,
`ee` char(10) NOT NULL,
`stato` int(11) NOT NULL,
`lavorato` int(1) NOT NULL,
`IDesito` int(11) NOT NULL,
`esito` varchar(45) NOT NULL,
`data_op` datetime NOT NULL,
`fatturato` int(11) NOT NULL,
`data_fatt` date NOT NULL,
`IDfattura` int(11) NOT NULL,
`data` datetime DEFAULT NULL,
`data_ar` datetime DEFAULT NULL,
`nome_ar` varchar(255) DEFAULT NULL,
`file_ar` varchar(255) DEFAULT NULL,
`IDflusso_dett` int(11) NOT NULL DEFAULT 0,
`type` varchar(2) NOT NULL DEFAULT '',
`typology` varchar(2) NOT NULL DEFAULT '',
`related_id` int(11) NOT NULL DEFAULT 0,
`related_ar_id` int(11) NOT NULL DEFAULT 0,
`unregistered` int(1) NOT NULL DEFAULT 0,
`total_attachments` int(11) NOT NULL DEFAULT 0,
`repeated_recipient` int(4) DEFAULT NULL,
`law_tomanage` tinyint(4) NOT NULL DEFAULT 0,
`law_towork` int(11) NOT NULL DEFAULT 0,
`law_toprint` int(11) NOT NULL DEFAULT 0,
`notlaw_tocomplete` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`IDlavorazione_dett`),
KEY `IDvettore` (`IDvettore`),
KEY `IDmittente` (`IDmittente`),
KEY `IDdestinatario` (`IDdestinatario`),
KEY `IDlavorazione` (`IDlavorazione`),
KEY `IDfattura` (`IDfattura`),
KEY `codiceabarre` (`codiceabarre`),
KEY `rif` (`rif`),
KEY `IDpostino` (`IDpostino`),
KEY `IDcategoria` (`IDcategoria`),
KEY `IDprodotto` (`IDprodotto`),
KEY `IDlistino` (`IDlistino`),
KEY `codiceavviso` (`codiceavviso`),
KEY `IDflusso_dett` (`IDflusso_dett`),
KEY `typology` (`typology`),
KEY `nome_ar` (`nome_ar`),
KEY `unregistered` (`unregistered`),
KEY `related_ar_id` (`related_ar_id`),
KEY `related_id` (`related_id`) USING BTREE,
KEY `type` (`type`),
KEY `stato` (`stato`),
KEY `lavorato` (`lavorato`),
KEY `law_tomanage` (`law_tomanage`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `tracking` (
`tracking_id` int(11) NOT NULL AUTO_INCREMENT,
`product_id` int(11) NOT NULL,
`status_id` int(11) NOT NULL,
`lat` double NOT NULL DEFAULT 0,
`lng` double NOT NULL DEFAULT 0,
`entity_id` int(11) NOT NULL,
`sub_entity_id` int(11) NOT NULL DEFAULT 0,
`date` datetime NOT NULL,
`note` text DEFAULT NULL,
`last` int(11) NOT NULL DEFAULT 0,
`date_last` datetime NOT NULL DEFAULT current_timestamp(),
`tracking_rel` int(11) NOT NULL DEFAULT 0,
`quantity_from` int(11) NOT NULL DEFAULT 1,
`quantity_to` int(11) NOT NULL DEFAULT 1,
`price` decimal(14,4) NOT NULL DEFAULT 0.0000,
`price_unit` decimal(14,4) NOT NULL DEFAULT 0.0000,
`vat` decimal(14,4) NOT NULL DEFAULT 0.0000,
`invoice_id` int(11) NOT NULL DEFAULT 0,
`management_status` int(11) NOT NULL DEFAULT 0,
`package_id` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`tracking_id`),
KEY `product_id` (`product_id`),
KEY `status_id` (`status_id`),
KEY `entity_id` (`entity_id`),
KEY `sub_entity_id` (`sub_entity_id`),
KEY `date` (`date`),
KEY `last` (`last`),
KEY `package_id` (`package_id`),
KEY `management_status` (`management_status`),
KEY `date_last` (`date_last`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `prodotti_pp` (
`IDprodotto` int(10) unsigned NOT NULL AUTO_INCREMENT,
`codice` varchar(255) NOT NULL DEFAULT '',
`codiceabarre` varchar(255) NOT NULL DEFAULT '',
`descrizione` longtext NOT NULL,
`prezzo` decimal(10,2) NOT NULL DEFAULT 0.00,
`prezzo_2` decimal(10,2) NOT NULL DEFAULT 0.00,
`prezzo_3` decimal(10,2) NOT NULL DEFAULT 0.00,
`UM` char(3) NOT NULL DEFAULT '',
`peso` decimal(10,2) NOT NULL DEFAULT 0.00,
`datapv` date NOT NULL DEFAULT '0000-00-00',
`iva` decimal(10,0) NOT NULL DEFAULT 0,
`fornitore` varchar(45) NOT NULL DEFAULT '',
`categoria` int(10) unsigned NOT NULL DEFAULT 0,
`trasp` decimal(10,2) NOT NULL DEFAULT 0.00,
`dettaglio` longtext NOT NULL,
`ico` varchar(255) NOT NULL DEFAULT '',
`foto` varchar(255) NOT NULL DEFAULT '',
`visibility_web` char(1) NOT NULL DEFAULT '',
`composito` char(1) NOT NULL DEFAULT '',
`sottocat` int(10) unsigned NOT NULL DEFAULT 0,
`tipologia` int(10) NOT NULL DEFAULT 0,
`marchio` int(10) unsigned NOT NULL DEFAULT 0,
`disp` char(1) NOT NULL DEFAULT '',
`vetrina1` char(1) NOT NULL DEFAULT '',
`vetrina2` char(1) NOT NULL DEFAULT '',
`click` decimal(10,0) NOT NULL DEFAULT 0,
`IDnote` int(11) NOT NULL DEFAULT 1,
PRIMARY KEY (`IDprodotto`),
KEY `categoria` (`categoria`),
KEY `codice` (`codice`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
CREATE TABLE `categorie_pp` (
`IDcategoria` int(10) unsigned NOT NULL AUTO_INCREMENT,
`IDmadre` int(11) NOT NULL DEFAULT 0,
`codice` varchar(255) NOT NULL,
`nome` varchar(45) NOT NULL DEFAULT '',
`visibility_web` char(1) NOT NULL DEFAULT '',
PRIMARY KEY (`IDcategoria`),
KEY `codice` (`codice`),
KEY `IDmadre` (`IDmadre`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
CREATE TABLE `doc_prodottipostali` (
`IDlavorazione` int(11) NOT NULL AUTO_INCREMENT,
`codice_lavorazione` varchar(255) NOT NULL,
`IDdistinta` varchar(45) NOT NULL,
`tipo` varchar(45) NOT NULL,
`data_distinta` date NOT NULL,
`data_lavorazione` date NOT NULL,
`IDcliente` int(11) DEFAULT 1,
`IDpagamento` int(11) NOT NULL,
`note` longtext NOT NULL,
`utente` varchar(45) NOT NULL,
`stato` int(11) NOT NULL,
`data_op` datetime NOT NULL,
`opzioni` varchar(45) NOT NULL,
`rif` varchar(45) NOT NULL,
`opzione1` varchar(45) NOT NULL,
`opzione2` varchar(45) NOT NULL,
`allegati` int(11) NOT NULL DEFAULT 0,
`IDesito` int(11) NOT NULL,
`esito` varchar(45) NOT NULL,
`prod_totali` int(11) NOT NULL,
`prod_accettati` int(11) NOT NULL,
`prod_fatturati` int(11) NOT NULL,
`prod_chiusi` int(11) NOT NULL,
`prod_end_shipping` int(11) NOT NULL,
`stato_fatt` char(1) NOT NULL,
`IDfattura` int(11) NOT NULL DEFAULT 0,
`IDrel` int(11) NOT NULL DEFAULT 0,
`IDflow` int(11) NOT NULL DEFAULT 0,
`laws_tomanage` tinyint(4) NOT NULL DEFAULT 0,
`idSender` int(11) NOT NULL,
PRIMARY KEY (`IDlavorazione`),
KEY `IDdistinta` (`IDdistinta`),
KEY `IDfattura` (`IDfattura`),
KEY `IDrel` (`IDrel`),
KEY `allegati` (`allegati`),
KEY `tipo` (`tipo`),
KEY `IDesito` (`IDesito`),
KEY `laws_tomanage` (`laws_tomanage`),
KEY `data_distinta` (`data_distinta`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
CREATE TABLE `ordini` (
`IDdoc` int(11) NOT NULL AUTO_INCREMENT,
`IDsoggetto` int(11) DEFAULT 1,
`numero` varchar(45) NOT NULL DEFAULT '0',
`numero_web` int(11) NOT NULL,
`datadoc` date NOT NULL DEFAULT '0000-00-00',
`tipodoc` varchar(255) NOT NULL,
`type_doc` varchar(4) NOT NULL DEFAULT '',
`modpag` int(10) unsigned NOT NULL DEFAULT 0,
`div_dest` longtext NOT NULL,
`IDagente` varchar(6) NOT NULL DEFAULT '0',
`IDvettore` int(10) unsigned NOT NULL DEFAULT 0,
`imballo` decimal(20,5) NOT NULL DEFAULT 0.00000,
`colli` decimal(10,0) NOT NULL DEFAULT 0,
`datascad` date NOT NULL DEFAULT '0000-00-00',
`stato` char(1) NOT NULL DEFAULT '',
`note` longtext NOT NULL,
`importo` decimal(20,5) NOT NULL DEFAULT 0.00000,
`tipo` varchar(4) NOT NULL DEFAULT '',
`causale` varchar(45) NOT NULL DEFAULT '',
`azione` char(1) NOT NULL DEFAULT '',
`utente` varchar(45) NOT NULL DEFAULT '',
`data_op` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`acconto` decimal(20,5) NOT NULL DEFAULT 0.00000,
`iva` decimal(20,5) NOT NULL DEFAULT 0.00000,
`sconto` decimal(20,5) NOT NULL DEFAULT 0.00000,
`n_doc_passive` varchar(45) NOT NULL DEFAULT '',
`porto` varchar(45) NOT NULL DEFAULT '',
`dataora_rit` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`ordine_web` char(1) NOT NULL DEFAULT '',
`clonato` char(1) NOT NULL DEFAULT '',
`trasp` decimal(20,5) NOT NULL DEFAULT 0.00000,
`varie` decimal(20,5) NOT NULL DEFAULT 0.00000,
`banca_appo` int(11) NOT NULL DEFAULT 0,
`IDmagazzino` int(11) NOT NULL DEFAULT 0,
`pv` int(11) NOT NULL DEFAULT 0,
`fornitore` int(11) NOT NULL DEFAULT 0,
`qta_colli` decimal(10,0) NOT NULL DEFAULT 0,
`bolli` decimal(20,5) NOT NULL DEFAULT 0.00000,
`agibilitamezzi` varchar(255) NOT NULL DEFAULT '',
`opzioneprezzo` varchar(45) NOT NULL DEFAULT '',
`emailcc` varchar(255) NOT NULL DEFAULT '',
`U_IBAN` varchar(45) NOT NULL,
`stato_fatt` char(1) NOT NULL,
`IDfattura` int(11) NOT NULL,
`data_fatt` date NOT NULL,
`digital_idinvoice` varchar(255) NOT NULL DEFAULT '0',
`digital_idupload` bigint(11) NOT NULL DEFAULT 0,
`digital_status` int(11) NOT NULL DEFAULT 0,
`digital_type` varchar(2) NOT NULL DEFAULT '',
`digital_office` int(11) NOT NULL,
`digital_sectional` char(2) NOT NULL DEFAULT '',
`digital_last_office` int(11) NOT NULL,
`digital_flag` varchar(4) NOT NULL,
`determines_code` varchar(255) NOT NULL,
`determines_date` date NOT NULL,
`determines_id` varchar(255) NOT NULL,
`flag_2` varchar(255) NOT NULL,
`flag` int(11) NOT NULL,
`flag_note` longtext NOT NULL,
`flag_3` varchar(255) NOT NULL,
`type_op` varchar(255) NOT NULL,
`IDpartenza_rel` int(11) NOT NULL,
`IDdestinazione_rel` int(11) NOT NULL,
`doc_insinuazionepassivo` varchar(255) NOT NULL,
`doc_insinuazionepassivo_data` date NOT NULL DEFAULT '0000-00-00',
PRIMARY KEY (`IDdoc`),
KEY `IDsoggetto` (`IDsoggetto`),
KEY `IDfattura` (`IDfattura`),
KEY `pv` (`pv`),
KEY `azione` (`azione`),
KEY `numero` (`numero`),
KEY `tipodoc` (`tipodoc`),
KEY `datadoc` (`datadoc`),
KEY `digital_sectional` (`digital_sectional`),
KEY `digital_type` (`digital_type`),
KEY `digital_office` (`digital_office`),
KEY `digital_status` (`digital_status`),
KEY `type_doc` (`type_doc`),
KEY `flag_2` (`flag_2`),
KEY `flag` (`flag`),
KEY `flag_3` (`flag_3`),
KEY `type_op` (`type_op`)
) ENGINE=InnoDB DEFAULT CHARSET=UTF8;
These are the number of rows for each table:
tracking:42231628
doc_prodottipostali_det:11316150
doc_prodottipostali:40556
ordini:40360
prodotti_pp:52
categorie_pp:30
I've tried making a subquery on the tracking table (the biggest), trying to filter out most of the rows but it didn't work.
This is the explain I get, it looks fine except for the "Using temporary" and "Using filesort".
The problem with the query is that with low numbers of elaborated rows it runs fine, the moment it tries to elaborate more than 1 million it begins to slow down.
Explain
id
select_type
table
type
possible_keys
key
key_len
ref
rows
Extra
1
SIMPLE
t
range
product_id,date,last
date
5
\N
69266
Using index condition; Using where; Using temporary; Using filesort
1
SIMPLE
dpd
eq_ref
PRIMARY,IDlavorazione
PRIMARY
4
db.t.product_id
1
1
SIMPLE
ppp
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDprodotto
1
Using where
1
SIMPLE
cpp
eq_ref
PRIMARY
PRIMARY
4
db.ppp.categoria
1
Using where
1
SIMPLE
cppp
eq_ref
PRIMARY
PRIMARY
4
func
1
Using where
1
SIMPLE
o
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDfattura
1
Using index
1
SIMPLE
dp
eq_ref
PRIMARY,tipo,data_distinta
PRIMARY
4
db.dpd.IDlavorazione
1
Using where
When you try to extend the time range on the tracking table t.date >= '2022-05-15 00:00:00' AND t.date <= '2022-07-26 23:59:59' the optimizer switches to a full table scan.
I've tried forcing the 'date' index on the tracking table but it still slows down with big ranges. The users are supposed to search without the time range too, so an index on the date isn't really the best option.
Update:
I've fixed the malformed query by aggregating the nonggregated fields. The GROUP_CONCAT fields returned the same values and the ONLY_FULL_GROUP_BY setting was disabled, that's why it was working normally.
I'll be more specific on the problems that I'm facing. When a user tries to search with a more extended range of dates, for example from '2022-03-15 00:00:00' to '2022-07-26 23:59:59' the query takes 5:24 minutes to complete with the following explain:
Explain with extended date range:
id
select_type
table
type
possible_keys
key
key_len
ref
rows
Extra
1
SIMPLE
dp
ref
PRIMARY,tipo,data_distinta
tipo
137
const
20400
Using index condition; Using where; Using temporary; Using filesort
1
SIMPLE
dpd
ref
PRIMARY,IDlavorazione
IDlavorazione
4
db.dp.IDlavorazione
115
1
SIMPLE
ppp
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDprodotto
1
Using where
1
SIMPLE
cpp
eq_ref
PRIMARY
PRIMARY
4
db.ppp.categoria
1
Using where
1
SIMPLE
cppp
eq_ref
PRIMARY
PRIMARY
4
func
1
Using where
1
SIMPLE
o
eq_ref
PRIMARY
PRIMARY
4
db.dpd.IDfattura
1
Using where; Using index
1
SIMPLE
t
ref
product_id,date,last
product_id
4
db.dpd.IDlavorazione_dett
1
Using where
Update 2:
Removing the ORDER BY clause removes the 'Using temporary' and 'Using filesort' from the explain, I guess it's an index issue, even if the fields are covered by indexes. After removing the ORDER BY clause the query time is still 5 minutes.
Your query appears to filter your tracking table on IN-lists on last and a date range on date. A multi-column index on those two columns, with the equality-matched column first and the range-matched column second may help you. And your query uses product_id from that table. So this covering index is worth a try.
ALTER TABLE tracking
ADD INDEX last_date (last, date, product_id);
You can look up covering indexes to learn more.
For starters, give these indexes a try:
dpd: INDEX(IDprodotto, IDcategoria, IDlavorazione_dett, IDlavorazione, IDfattura)
dpd: INDEX(IDlavorazione_dett, IDlavorazione, IDprodotto, IDcategoria, IDfattura)
dp: INDEX(tipo, opzione2, data_distinta, IDlavorazione)
dp: INDEX(IDlavorazione)
o: INDEX(IDdoc)
t: INDEX(date, last, product_id)
t: INDEX(product_id)
ppp: INDEX(IDprodotto, categoria)
categorie_pp: INDEX(IDcategoria)
categorie_pp: INDEX(codice, IDcategoria)
When adding a composite index, DROP any plain index(es) with the same leading columns. That is, when you have both INDEX(a) and INDEX(a,b), toss the former.
CAST(dpd.IDcategoria AS UNSIGNED) -- Perhaps you should change the datatype of IDcategoria so you can avoid the conversion here? (That might lead to better index usage.)
The main things slowing down the query (and making it difficult to optimize):
WHERE clause referencing multiple tables.
OR
NOT
GROUP BY after JOINing 7 tables
But I don't see how to improve on any of them.
I have a complex query that is dynamically assembled based upon search criteria. However, in its simplest form, it is still very slow. The main table it runs against has ~10M records. I ran an explain against a 'base' query and the first row of the explain looks bad (at least to a novice dba like me). I have read a couple tutorials about EXPLAIN, but I still am unsure how to fix the query. So, the first row of the results seems to indicate the problem, but I don't know what to do with it. I couldn't make a composite key that long even if I wanted to and some of the field names in that possible_keys column are not even in the patients table. Any help will be greatly appreciated.
id,select_type,table,type,possible_keys,key,key_len,ref,rows,Extra
1,SIMPLE,patients,range,"PRIMARY,location,appt_date,status,radiologist,contract,lastname,paperwork,images_archived,hash,created,document_attached,all_images_archived,last_image_archived,modality,study_uid,company,second_access,firstname,report_delivered,ssn,order_entry_status,dob,tech,doctor,mobile_facility,accession,location_appt_date,location_created,location_lastname,ref,person_seq",location_appt_date,55,NULL,573534,"Using index condition; Using where; Using temporary; Using filesort"
1,SIMPLE,receivable_transactions,ref,patient_seq,patient_seq,4,ris-dev.patients.seq,1,NULL
1,SIMPLE,patients_dispatch,ref,patient_seq,patient_seq,4,ris-dev.patients.seq,1,NULL
1,SIMPLE,mobile_facility,ref,"unique_index,name,location",unique_index,115,"ris-dev.patients.mobile_facility,const",1,"Using where"
1,SIMPLE,mobile_facility_service_areas,eq_ref,PRIMARY,PRIMARY,4,ris-dev.mobile_facility.service_area,1,NULL
Edit: same EXPLAIN, but reformatted to be easier to read:
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE patients range PRIMARY location_appt_date 55 NULL 573534 Using index condition; Using where; Using temporary; Using filesort
location
appt_date
status
radiologist
contract
lastname
paperwork
images_archived
hash
created
document_attached
all_images_archived
last_image_archived
modality
study_uid
company
second_access
firstname
report_delivered
ssn
order_entry_status
dob
tech
doctor
mobile_facility
accession
location_appt_date
location_created
location_lastname
ref
person_seq
1 SIMPLE receivable_transactions ref patient_seq patient_seq 4 ris-dev.patients.seq 1 NULL
1 SIMPLE patients_dispatch ref patient_seq patient_seq 4 ris-dev.patients.seq 1 NULL
1 SIMPLE mobile_facility ref unique_index unique_index 115 ris-dev.patients.mobile_facility,const 1 Using where
name
location
1 SIMPLE mobile_facility_service_areas eq_ref PRIMARY PRIMARY 4 ris-dev.mobile_facility.service_area 1 NULL
The explain is setup against the following query and table structures.
SELECT patients.fax_in_queue, patients.modality, patients.stat, patients.created, patients.seq, patients.lastname,
patients.firstname, patients.appt_date, patients.status, patients.contract, patients.location, patients.unique_hash,
patients.images_archived, patients.report_delivered, patients.doctor, patients.mobile_facility, patients.history,
patients.dob, patients.all_images_archived, patients.order_entry_status, patients.tech, patients.radiologist,
patients.last_image_archived, patients.state, patients.ss_comments, patients.completed, patients.report_status,
patients.have_paperwork, patients.facility_room_number, patients.facility_station_name, patients.facility_bed,
patients.findings_level, patients.document_attached, patients.study_start, patients.company, patients.accession,
patients.number_images, patients.client_number_images, patients.sex, patients.threshhold , GROUP_CONCAT(CONCAT(CONCAT(receivable_transactions.modifier, " "),
receivable_transactions.description) SEPARATOR ", ") AS rt_desc , patients_dispatch.seq AS doc_seq, patients_dispatch.requisition_last_sent,
patients_dispatch.requisition_signed_by_file_seq, patients_dispatch.requisition_signed, patients_dispatch.order_reason, patients_dispatch.order_comments,
patients_dispatch.order_taken, patients_dispatch.order_tech_last_notified, patients_dispatch.order_tech_in_transit, patients_dispatch.order_tech_in,
patients_dispatch.order_tech_out, patients_dispatch.order_tech_ack, patients_dispatch.addr1 AS d_addr1, patients_dispatch.addr2 AS d_addr2,
patients_dispatch.city AS d_city, patients_dispatch.state AS d_state, patients_dispatch.zip AS d_zip, CONCAT(patients.status, order_tech_out,
order_tech_in, order_tech_in_transit) as pseudo_status , mobile_facility.requisition_fax, mobile_facility.station_list, mobile_facility.address1 as mf_addr1,
mobile_facility.address2 as mf_addr2, mobile_facility.city as mf_city, mobile_facility.state as mf_state, mobile_facility.zip as mf_zip,
mobile_facility.phone as mf_phone, mobile_facility.phone2 as mf_phone2, mobile_facility_service_areas.name as mf_service_area
FROM patients LEFT JOIN receivable_transactions ON patients.seq = receivable_transactions.patient_seq
LEFT JOIN patients_dispatch ON patients.seq = patients_dispatch.patient_seq
LEFT JOIN mobile_facility ON patients.location = mobile_facility.location AND patients.mobile_facility = mobile_facility.name
LEFT JOIN mobile_facility_service_areas ON mobile_facility.service_area = mobile_facility_service_areas.seq
WHERE patients.location = "XYZCompany" AND ((patients.appt_date >= '2020-03-19' AND patients.appt_date <= '2020-03-19 23:59:59')
OR (patients.appt_date <= '2020-03-19' AND patients.status < 'X'))
GROUP BY patients.seq DESC
ORDER BY patients.status, patients.order_entry_status, pseudo_status, patients.order_entry_status,patients.lastname);
CREATE TABLE `patients` (
`seq` int(11) NOT NULL AUTO_INCREMENT,
`person_seq` int(11) NOT NULL,
`firstname` varchar(20) NOT NULL DEFAULT '',
`lastname` varchar(30) NOT NULL DEFAULT '',
`middlename` varchar(20) NOT NULL DEFAULT '',
`ref` varchar(50) NOT NULL DEFAULT '',
`location` varchar(50) NOT NULL DEFAULT '',
`doctor` varchar(50) NOT NULL,
`radiologist` varchar(20) NOT NULL DEFAULT '',
`contract` varchar(50) NOT NULL,
`history` mediumtext NOT NULL,
`dob` varchar(15) NOT NULL DEFAULT '0000-00-00',
`appt_date` date NOT NULL DEFAULT '0000-00-00',
`status` tinyint(4) NOT NULL DEFAULT '0',
`tech` varchar(50) NOT NULL DEFAULT '',
`created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`ss_comments` mediumtext NOT NULL,
`mobile_facility` varchar(60) NOT NULL DEFAULT '',
`facility_room_number` varchar(50) NOT NULL,
`facility_bed` varchar(20) NOT NULL,
`facility_station_name` varchar(50) NOT NULL,
`stat` tinyint(4) NOT NULL DEFAULT '0',
`have_paperwork` tinyint(4) NOT NULL DEFAULT '0',
`completed` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`sex` char(1) NOT NULL DEFAULT '',
`unique_hash` varchar(100) NOT NULL DEFAULT '',
`number_images` int(11) NOT NULL DEFAULT '0',
`client_number_images` int(11) NOT NULL,
`images_archived` tinyint(4) NOT NULL DEFAULT '0',
`completed_fax` varchar(10) NOT NULL DEFAULT '0' COMMENT 'This is the number the completed report is faxed to.',
`report_delivered` tinyint(4) NOT NULL DEFAULT '0',
`report_delivered_time` datetime NOT NULL,
`document_attached` tinyint(4) NOT NULL DEFAULT '0',
`modality` varchar(3) NOT NULL,
`last_image_archived` datetime NOT NULL,
`all_images_archived` tinyint(4) NOT NULL DEFAULT '0',
`fax_in_queue` varchar(12) NOT NULL,
`accession` varchar(100) NOT NULL,
`study_uid` varchar(100) NOT NULL,
`order_entry_status` tinyint(4) NOT NULL,
`compare_to` varchar(15) NOT NULL,
`state` varchar(3) NOT NULL,
`company` int(11) NOT NULL,
`second_access` varchar(50) NOT NULL,
`threshhold` datetime NOT NULL,
`report_status` tinyint(4) NOT NULL,
`second_id` varchar(50) NOT NULL,
`rad_alerted` tinyint(4) NOT NULL,
`assigned` datetime NOT NULL,
`findings_level` tinyint(4) NOT NULL,
`report_viewed` tinyint(4) NOT NULL,
`study_received` datetime NOT NULL,
`study_start` datetime NOT NULL,
`study_end` datetime NOT NULL,
`completed_email` varchar(50) NOT NULL,
`completed_send` varchar(255) NOT NULL,
`ssn` varchar(12) NOT NULL,
`exorder_number` varchar(30) NOT NULL,
`exvisit_number` varchar(30) NOT NULL,
`row_updated` timestamp NOT NULL ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`seq`),
KEY `location` (`location`),
KEY `appt_date` (`appt_date`),
KEY `status` (`status`),
KEY `radiologist` (`radiologist`),
KEY `contract` (`contract`),
KEY `lastname` (`lastname`),
KEY `paperwork` (`have_paperwork`),
KEY `images_archived` (`images_archived`),
KEY `hash` (`unique_hash`),
KEY `created` (`created`),
KEY `document_attached` (`document_attached`),
KEY `all_images_archived` (`all_images_archived`),
KEY `last_image_archived` (`last_image_archived`),
KEY `modality` (`modality`),
KEY `study_uid` (`study_uid`),
KEY `company` (`company`),
KEY `second_access` (`second_access`),
KEY `firstname` (`firstname`),
KEY `report_delivered` (`report_delivered`),
KEY `ssn` (`ssn`),
KEY `order_entry_status` (`order_entry_status`),
KEY `dob` (`dob`),
KEY `tech` (`tech`),
KEY `doctor` (`doctor`),
KEY `mobile_facility` (`mobile_facility`),
KEY `accession` (`accession`),
KEY `location_appt_date` (`location`,`appt_date`),
KEY `location_created` (`location`,`created`),
KEY `location_lastname` (`location`,`lastname`),
KEY `ref` (`ref`),
KEY `person_seq` (`person_seq`)
) ENGINE=InnoDB AUTO_INCREMENT=10242952 DEFAULT CHARSET=latin1;
CREATE TABLE `receivable_transactions` (
`seq` int(11) NOT NULL AUTO_INCREMENT,
`patient_seq` int(11) NOT NULL DEFAULT '0',
`cptcode` varchar(15) NOT NULL DEFAULT '',
`modifier` char(2) NOT NULL DEFAULT '',
`description` varchar(100) NOT NULL DEFAULT '',
`amount` decimal(6,2) NOT NULL DEFAULT '0.00',
`type` char(2) NOT NULL DEFAULT '',
`transaction` varchar(10) NOT NULL DEFAULT '',
`radiologist` varchar(20) NOT NULL DEFAULT '',
`status` tinyint(4) NOT NULL DEFAULT '0',
`completed` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
`report_meta_seq` int(11) NOT NULL DEFAULT '0',
`report_header` varchar(255) NOT NULL,
`report_body` blob NOT NULL,
`report_impression` mediumtext NOT NULL,
`report_hide` tinyint(4) NOT NULL,
`radiologist_group` varchar(50) NOT NULL,
`addendum` int(4) NOT NULL DEFAULT '0',
`addendum_type` varchar(20) NOT NULL,
`peer_review` int(4) NOT NULL DEFAULT '0',
`qa_reason` varchar(255) NOT NULL DEFAULT '',
`qa_agree` decimal(2,1) NOT NULL DEFAULT '0.0',
`findings` tinyint(4) NOT NULL,
`comments` mediumtext NOT NULL,
`company` int(11) NOT NULL,
`row_updated` timestamp NOT NULL ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`seq`),
KEY `patient_seq` (`patient_seq`),
KEY `cptcode` (`cptcode`),
KEY `transaction` (`transaction`),
KEY `type` (`type`),
KEY `created` (`created`),
KEY `radiologist` (`radiologist`),
KEY `status` (`status`),
KEY `report_meta_seq` (`report_meta_seq`),
KEY `Billing Check Dropdown` (`status`,`completed`),
KEY `qa_agree` (`qa_agree`),
KEY `peer_review` (`peer_review`),
KEY `addendum` (`addendum`),
KEY `company` (`company`),
KEY `completed` (`completed`)
) ENGINE=InnoDB AUTO_INCREMENT=9380351 DEFAULT CHARSET=latin1;
CREATE TABLE `patients_dispatch` (
`seq` int(11) NOT NULL AUTO_INCREMENT,
`patient_seq` int(11) NOT NULL,
`order_taken` datetime NOT NULL,
`order_taken_by` varchar(50) NOT NULL,
`order_person_calling` varchar(50) NOT NULL,
`order_supervising_physician` varchar(50) NOT NULL,
`order_trip_count` tinyint(4) NOT NULL,
`order_trip_count_max` tinyint(4) NOT NULL,
`order_trip_visit` tinyint(4) NOT NULL,
`order_tech_in` datetime NOT NULL,
`order_tech_out` datetime NOT NULL,
`order_ssn` varchar(12) NOT NULL,
`order_service_request_time` datetime NOT NULL,
`order_reason` varchar(255) NOT NULL,
`order_tech_ack` datetime NOT NULL,
`order_tech_assigned` datetime NOT NULL,
`order_tech_last_notified` datetime NOT NULL,
`requisition_last_sent` datetime NOT NULL,
`requisition_signed` datetime NOT NULL,
`requisition_signed_by` varchar(50) NOT NULL,
`requisition_signed_by_text` varchar(75) NOT NULL,
`requisition_signed_by_file_seq` int(11) NOT NULL,
`order_comments` mediumtext NOT NULL,
`order_tech_in_transit` datetime NOT NULL,
`fasting` tinyint(1) NOT NULL,
`collection_time` time DEFAULT NULL,
`addr1` varchar(100) NOT NULL,
`addr2` varchar(100) NOT NULL,
`city` varchar(30) NOT NULL,
`state` varchar(3) NOT NULL,
`zip` varchar(12) NOT NULL,
`phone` varchar(15) NOT NULL,
`mileage_start` int(11) NOT NULL,
`mileage_end` int(11) NOT NULL,
PRIMARY KEY (`seq`),
KEY `patient_seq` (`patient_seq`)
) ENGINE=InnoDB AUTO_INCREMENT=2261091 DEFAULT CHARSET=latin1;
CREATE TABLE `mobile_facility` (
`seq` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(60) NOT NULL,
`location` varchar(50) DEFAULT NULL,
`address1` varchar(50) NOT NULL,
`address2` varchar(50) NOT NULL,
`city` varchar(50) NOT NULL,
`state` varchar(2) NOT NULL,
`zip` varchar(10) NOT NULL,
`phone` varchar(15) NOT NULL,
`phone2` varchar(15) NOT NULL,
`fax` varchar(110) NOT NULL,
`rads_can_read` text NOT NULL,
`rads_cant_read` text NOT NULL,
`only_techs` text NOT NULL,
`never_modalities` varchar(255) NOT NULL COMMENT 'A serialized list of modalities a facility may not use.',
`station_list` mediumtext NOT NULL,
`email` varchar(255) NOT NULL,
`misc1` varchar(255) NOT NULL,
`latitude` float NOT NULL DEFAULT '0',
`longitude` float NOT NULL DEFAULT '0',
`affiliation` int(11) NOT NULL COMMENT 'mobile_facility_affiliations seq',
`branch` int(11) NOT NULL COMMENT 'mobile_facility_branches seq',
`service_area` int(11) NOT NULL COMMENT 'mobile_facility_service_areas seq',
`other_id` varchar(50) NOT NULL COMMENT 'Usually used for HL7',
`facility_type` varchar(2) DEFAULT NULL,
`no_stat` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Should the facility allow stat priority on patients?',
`facility_notes` varchar(512) DEFAULT NULL,
`requisition_fax` varchar(110) NOT NULL,
`report_template` text NOT NULL,
`all_orders_stat` tinyint(1) NOT NULL,
`sms_notification` varchar(15) NOT NULL,
`tat` varchar(10) NOT NULL,
`npi` varchar(15) NOT NULL,
`NMXR` tinyint(4) NOT NULL DEFAULT '0',
`billing_type` varchar(10) NOT NULL,
`salesman` varchar(75) NOT NULL,
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
`default_bill_to` tinyint(4) NOT NULL DEFAULT '0',
PRIMARY KEY (`seq`),
UNIQUE KEY `unique_index` (`name`,`location`),
KEY `name` (`name`),
KEY `location` (`location`)
) ENGINE=InnoDB AUTO_INCREMENT=155104 DEFAULT CHARSET=latin1;
CREATE TABLE `mobile_facility_service_areas` (
`seq` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL,
`location` varchar(50) NOT NULL,
PRIMARY KEY (`seq`)
) ENGINE=InnoDB AUTO_INCREMENT=841 DEFAULT CHARSET=latin1;
It's only using the index on location, but that only narrows down the search to about a half a million rows. You'd like it to use an index to further narrow down by the appt_date.
However, the use of OR in your WHERE clause is causing a problem. It can't decide how to use the index.
Here's what I suggest:
Drop the index on location because it's redundant with the other indexes that have location as their first column.
Replace the index on location_appt_date with an index on location_appt_date_status.
ALTER TABLE patients
DROP KEY location,
DROP KEY location_appt_date,
ADD KEY location_appt_date_status (location, appt_date, status);
Refactor the query to use UNION instead of OR:
SELECT ... (all the columns you have) ...
FROM (
SELECT * FROM patients USE INDEX (location_appt_date_status)
WHERE location = 'XYZCompany' AND appt_date >= '2020-03-19' AND appt_date < '2020-03-20'
UNION
SELECT * FROM patients USE INDEX (location_appt_date_status)
WHERE location = 'XYZCompany' AND appt_date <= '2020-03-19' AND status < 'X'
) AS p
LEFT JOIN receivable_transactions FORCE INDEX (patient_seq)
ON p.seq = receivable_transactions.patient_seq
LEFT JOIN patients_dispatch FORCE INDEX (patient_seq)
ON p.seq = patients_dispatch.patient_seq
INNER JOIN mobile_facility FORCE INDEX (unique_index)
ON p.location = mobile_facility.location AND p.mobile_facility = mobile_facility.name
INNER JOIN mobile_facility_service_areas FORCE INDEX (PRIMARY)
ON mobile_facility.service_area = mobile_facility_service_areas.seq
GROUP BY p.seq
ORDER BY p.status, p.order_entry_status, pseudo_status, p.order_entry_status, p.lastname
You might not need all the USE INDEX() / FORCE INDEX() optimizer hints I used. I did those because I was testing with empty tables, and that can confuse the optimizer.
Let me focus on the part that affects optimization the most:
FROM patients AS p
LEFT JOIN receivable_transactions AS rt ON p.seq = rt.patient_seq
LEFT JOIN patients_dispatch AS pd ON p.seq = pd.patient_seq
LEFT JOIN mobile_facility AS mf ON p.location = mf.location
AND p.mobile_facility AS mf = mf.name
LEFT JOIN mobile_facility_service_areas AS sa ON mf.service_area = sa.seq
WHERE p.location = "XYZCompany"
AND ((p.appt_date >= '2020-03-19'
AND p.appt_date <= '2020-03-19 23:59:59')
OR (p.appt_date <= '2020-03-19'
AND p.status < 'X')
)
GROUP BY p.seq DESC
ORDER BY p.status, p.order_entry_status, pseudo_status, p.order_entry_status,
p.lastname);
The biggest issue is the OR. It often prevents most optimizations. The usual fix is to turn it into a UNION:
( SELECT ...
FROM .. JOIN ..
WHERE p.location = "XYZCompany"
AND p.appt_date >= '2020-03-19'
AND p.appt_date < '2020-03-19' + INTERVAL 1 DAY
...
)
UNION ALL
( SELECT ...
FROM .. JOIN ..
WHERE p.location = "XYZCompany"
AND p.appt_date <= '2020-03-19'
AND p.status < 'X'
...
)
Each select can benefit from this composite index on patients:
(location, appt_date, status)
The < 'X' is problematic because two ranges (appt_date and status) cannot both be used effectively. What are the possible values of status? If there is only one value before 'X', say 'M', then this would be much better: p.status = 'M' together with another index: (location, status, appt_date)
SELECT lots of stuff, then GROUP BY p.seq -- This is probably create strange results. (Search for ONLY_FULL_GROUP_BY for more discussion). It may be better to first get the patients.seq values (since that is all you are filtering on), then join to the other tables. This would eliminate the GROUP BY, or at least force you to deal with which row to fetch from each of the other tables.
range location_appt_date 55 573534 Using index condition; Using where; Using temporary; Using filesort -- says
55 = 2+50 (for varchar(50)) + 3 (for date) -- neither is NULL.
Based on the 55, I wonder if it is so well optimized that the OR->UNION is not needed.
"Using index condition" is internally called ICP (Index Condition Pushdown) if you want further understanding.
"Using filesort" may be an understatement -- There are probably two sorts, one for GROUP BY, one for ORDER BY. EXPLAIN FORMAT=JSON SELECT ... would make it clear. (And hence my hint that the GROUP BY should be avoided.
You have some redundant indexes (not relevant to much other than disk space): INDEX(a,b), INDEX(a) --> toss INDEX(a).
patients has an awful number of indexes.
The other tables seem to have adequate indexes for your query.
I have 2 search pages that are super slow in getting results.
I did not write the queries, but I know they are not written in an efficient way; I just do not have enough practice in MYSQL to figure out how to make them more efficient.
What should I do to improve the following query?
SELECT DISTINCT m.id AS memberID , m.login , m.age , p.gender
, p.name AS header , p.id AS profileID , p.city , p.state , p.lastlogin
, o.login AS online , c.name AS country , ph.filename_1 AS pic
FROM dt_members AS m
INNER JOIN dt_profile_approved AS p ON m.id=p.member_id
LEFT JOIN dt_privacy AS pv ON m.id=pv.member_id
INNER JOIN dt_countries AS c ON c.id=p.country
LEFT JOIN dt_photos AS ph ON m.id=ph.member_id
LEFT JOIN dt_usersonline AS o ON m.login=o.login
WHERE p.status=1 AND (pv.unsearchable IS NULL OR pv.unsearchable='')
AND p.gender='Female' AND m.age BETWEEN 25 AND 40
ORDER BY p.lastlogin DESC
LIMIT 0, 21;
it is terribly slow and showing 500 error often.
Output of explain:
Show Create Table:
CREATE TABLE `dt_members` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`login` varchar(25) DEFAULT NULL,
`pswd` varchar(20) DEFAULT NULL,
`email` varchar(255) DEFAULT NULL,
`name` varchar(40) DEFAULT NULL,
`gender` varchar(10) DEFAULT NULL,
`age` int(11) DEFAULT NULL,
`country` varchar(255) DEFAULT NULL,
`looking_for` varchar(255) DEFAULT NULL,
`ip_addr` varchar(15) DEFAULT NULL,
`reg_date` int(11) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`system_status` int(11) DEFAULT '0',
`system_status_end` int(11) DEFAULT NULL,
`unlimited` int(11) DEFAULT '0',
`unlimited_end` int(11) DEFAULT NULL,
`matchfinder` int(1) DEFAULT '0',
PRIMARY KEY (`id`),
KEY `login` (`login`),
KEY `pswd` (`pswd`),
KEY `email` (`email`),
KEY `name` (`name`),
KEY `gender` (`gender`),
KEY `age` (`age`),
KEY `country` (`country`),
KEY `looking_for` (`looking_for`),
KEY `ip_addr` (`ip_addr`),
KEY `reg_date` (`reg_date`),
KEY `status` (`status`),
KEY `system_status` (`system_status`),
KEY `system_status_end` (`system_status_end`),
KEY `unlimited` (`unlimited`),
KEY `unlimited_end` (`unlimited_end`),
KEY `matchfinder` (`matchfinder`)
) ENGINE=MyISAM AUTO_INCREMENT=29150 DEFAULT CHARSET=latin1
CREATE TABLE `dt_profile` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`member_id` int(11) DEFAULT NULL,
`country` int(11) DEFAULT NULL,
`state` varchar(255) DEFAULT NULL,
`city` varchar(255) DEFAULT NULL,
`email` varchar(255) DEFAULT NULL,
`name` varchar(255) DEFAULT NULL,
`gender` varchar(20) DEFAULT NULL,
`birth_day` int(11) DEFAULT NULL,
`birth_month` varchar(6) DEFAULT NULL,
`birth_year` int(11) DEFAULT NULL,
`marital_status` int(11) DEFAULT NULL,
`children` int(11) DEFAULT NULL,
`drinking` int(11) DEFAULT NULL,
`smoking` int(11) DEFAULT NULL,
`food` int(11) DEFAULT NULL,
`eye_color` int(11) DEFAULT NULL,
`hair_color` int(11) DEFAULT NULL,
`height` int(11) DEFAULT NULL,
`body_type` int(11) DEFAULT NULL,
`race` int(11) DEFAULT NULL,
`religion` int(11) DEFAULT NULL,
`occupation` int(11) DEFAULT NULL,
`education` int(11) DEFAULT NULL,
`lang_1` int(11) DEFAULT NULL,
`lang_1_rate` int(11) DEFAULT NULL,
`lang_2` int(11) DEFAULT NULL,
`lang_2_rate` int(11) DEFAULT NULL,
`lang_3` int(11) DEFAULT NULL,
`lang_3_rate` int(11) DEFAULT NULL,
`lang_4` int(11) DEFAULT NULL,
`lang_4_rate` int(11) DEFAULT NULL,
`looking_for` varchar(10) DEFAULT NULL,
`age_from` int(11) DEFAULT NULL,
`age_to` int(11) DEFAULT NULL,
`general_info` text,
`appearance_info` text,
`looking_for_info` text,
`status` int(11) DEFAULT NULL,
`finish_status` int(11) DEFAULT NULL,
`not_newbie` int(11) DEFAULT NULL,
`lastlogin` int(10) NOT NULL DEFAULT '0',
`zipcode` varchar(5) NOT NULL DEFAULT '',
`longitude` double DEFAULT NULL,
`latitude` double DEFAULT NULL,
`photo_pass` varchar(25) NOT NULL DEFAULT '',
`view_count` int(11) DEFAULT '0',
`wants_kids` int(11) DEFAULT NULL,
`kids_okay` int(11) DEFAULT NULL,
`relocate_domestic` int(11) DEFAULT NULL,
`relocate_international` int(11) DEFAULT NULL,
`pioneer` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `member_id` (`member_id`),
KEY `country` (`country`),
KEY `state` (`state`),
KEY `city` (`city`),
KEY `email` (`email`),
KEY `name` (`name`),
KEY `gender` (`gender`),
KEY `birth_day` (`birth_day`),
KEY `birth_month` (`birth_month`),
KEY `birth_year` (`birth_year`),
KEY `marital_status` (`marital_status`),
KEY `children` (`children`),
KEY `drinking` (`drinking`),
KEY `smoking` (`smoking`),
KEY `food` (`food`),
KEY `eye_color` (`eye_color`),
KEY `hair_color` (`hair_color`),
KEY `height` (`height`),
KEY `body_type` (`body_type`),
KEY `race` (`race`),
KEY `religion` (`religion`),
KEY `occupation` (`occupation`),
KEY `education` (`education`),
KEY `lang_1` (`lang_1`),
KEY `lang_1_rate` (`lang_1_rate`),
KEY `lang_2` (`lang_2`),
KEY `lang_2_rate` (`lang_2_rate`),
KEY `lang_3` (`lang_3`),
KEY `lang_3_rate` (`lang_3_rate`),
KEY `lang_4` (`lang_4`),
KEY `lang_4_rate` (`lang_4_rate`),
KEY `looking_for` (`looking_for`),
KEY `age_from` (`age_from`),
KEY `age_to` (`age_to`),
KEY `status` (`status`),
KEY `finish_status` (`finish_status`),
KEY `not_newbie` (`not_newbie`),
KEY `lastlogin` (`lastlogin`),
KEY `zipcode` (`zipcode`),
KEY `longitude` (`longitude`),
KEY `latitude` (`latitude`),
KEY `photo_pass` (`photo_pass`),
KEY `view_count` (`view_count`),
KEY `wants_kids` (`wants_kids`),
KEY `kids_okay` (`kids_okay`),
KEY `relocate_domestic` (`relocate_domestic`),
KEY `relocate_international` (`relocate_international`),
KEY `pioneer` (`pioneer`)
) ENGINE=MyISAM AUTO_INCREMENT=18389 DEFAULT CHARSET=latin1
CREATE TABLE `dt_privacy` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`member_id` int(11) DEFAULT NULL,
`online_yn` char(1) DEFAULT NULL,
`vkiss_yn` char(1) DEFAULT NULL,
`profiles_yn` char(1) DEFAULT NULL,
`IM_yn` char(1) DEFAULT NULL,
`featured_yn` char(1) DEFAULT NULL,
`HL_messaged_yn` char(1) DEFAULT NULL,
`HL_im_yn` char(1) DEFAULT NULL,
`HL_viewed_yn` char(1) DEFAULT NULL,
`HL_kissed_yn` char(1) DEFAULT NULL,
`HL_favorite_yn` char(1) DEFAULT NULL,
`unsearchable` char(1) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `member_id` (`member_id`),
KEY `online_yn` (`online_yn`),
KEY `vkiss_yn` (`vkiss_yn`),
KEY `profiles_yn` (`profiles_yn`),
KEY `IM_yn` (`IM_yn`),
KEY `featured_yn` (`featured_yn`),
KEY `HL_messaged_yn` (`HL_messaged_yn`),
KEY `HL_im_yn` (`HL_im_yn`),
KEY `HL_viewed_yn` (`HL_viewed_yn`),
KEY `HL_kissed_yn` (`HL_kissed_yn`),
KEY `HL_favorite_yn` (`HL_favorite_yn`),
KEY `unsearchable` (`unsearchable`)
) ENGINE=MyISAM AUTO_INCREMENT=26305 DEFAULT CHARSET=latin1
CREATE TABLE `dt_countries` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `name` (`name`)
) ENGINE=MyISAM AUTO_INCREMENT=226 DEFAULT CHARSET=latin1
CREATE TABLE `dt_photos` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`member_id` varchar(255) DEFAULT NULL,
`filename_1` varchar(255) DEFAULT NULL,
`filename_2` varchar(255) DEFAULT NULL,
`filename_3` varchar(255) NOT NULL DEFAULT '',
`filename_4` varchar(255) NOT NULL DEFAULT '',
`filename_5` varchar(255) NOT NULL DEFAULT '',
`filename_6` varchar(255) NOT NULL DEFAULT '',
`filename_7` varchar(255) NOT NULL DEFAULT '',
`filename_8` varchar(255) NOT NULL DEFAULT '',
`filename_9` varchar(255) NOT NULL DEFAULT '',
`filename_10` varchar(255) NOT NULL DEFAULT '',
`filename_11` varchar(255) NOT NULL DEFAULT '',
`filename_12` varchar(255) NOT NULL DEFAULT '',
`filename_13` varchar(255) NOT NULL DEFAULT '',
`filename_14` varchar(255) NOT NULL DEFAULT '',
`filename_15` varchar(255) NOT NULL DEFAULT '',
`filename_16` varchar(255) NOT NULL DEFAULT '',
`filename_17` varchar(255) NOT NULL DEFAULT '',
`filename_18` varchar(255) NOT NULL DEFAULT '',
`filename_19` varchar(255) NOT NULL DEFAULT '',
`filename_20` varchar(255) NOT NULL DEFAULT '',
`private_1` tinyint(1) NOT NULL DEFAULT '0',
`private_2` tinyint(1) NOT NULL DEFAULT '0',
`private_3` tinyint(1) NOT NULL DEFAULT '0',
`private_4` tinyint(1) NOT NULL DEFAULT '0',
`private_5` tinyint(1) NOT NULL DEFAULT '0',
`private_6` tinyint(1) NOT NULL DEFAULT '0',
`private_7` tinyint(1) NOT NULL DEFAULT '0',
`private_8` tinyint(1) NOT NULL DEFAULT '0',
`private_9` tinyint(1) NOT NULL DEFAULT '0',
`private_10` tinyint(1) NOT NULL DEFAULT '0',
`private_11` tinyint(1) NOT NULL DEFAULT '0',
`private_12` tinyint(1) NOT NULL DEFAULT '0',
`private_13` tinyint(1) NOT NULL DEFAULT '0',
`private_14` tinyint(1) NOT NULL DEFAULT '0',
`private_15` tinyint(1) NOT NULL DEFAULT '0',
`private_16` tinyint(1) NOT NULL DEFAULT '0',
`private_17` tinyint(1) NOT NULL DEFAULT '0',
`private_18` tinyint(1) NOT NULL DEFAULT '0',
`private_19` tinyint(1) NOT NULL DEFAULT '0',
`private_20` tinyint(1) NOT NULL DEFAULT '0',
`password` varchar(255) NOT NULL DEFAULT '',
`description_1` varchar(255) DEFAULT NULL,
`description_2` varchar(255) DEFAULT NULL,
`description_3` varchar(255) NOT NULL DEFAULT '',
`description_4` varchar(255) NOT NULL DEFAULT '',
`description_5` varchar(255) NOT NULL DEFAULT '',
`description_6` varchar(255) NOT NULL DEFAULT '',
`description_7` varchar(255) NOT NULL DEFAULT '',
`description_8` varchar(255) NOT NULL DEFAULT '',
`description_9` varchar(255) NOT NULL DEFAULT '',
`description_10` varchar(255) NOT NULL DEFAULT '',
`description_11` varchar(255) NOT NULL DEFAULT '',
`description_12` varchar(255) NOT NULL DEFAULT '',
`description_13` varchar(255) NOT NULL DEFAULT '',
`description_14` varchar(255) NOT NULL DEFAULT '',
`description_15` varchar(255) NOT NULL DEFAULT '',
`description_16` varchar(255) NOT NULL DEFAULT '',
`description_17` varchar(255) NOT NULL DEFAULT '',
`description_18` varchar(255) NOT NULL DEFAULT '',
`description_19` varchar(255) NOT NULL DEFAULT '',
`description_20` varchar(255) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
KEY `filename_12` (`filename_12`),
KEY `filename_13` (`filename_13`),
KEY `filename_14` (`filename_14`),
KEY `filename_15` (`filename_15`),
KEY `filename_16` (`filename_16`),
KEY `filename_17` (`filename_17`),
KEY `filename_18` (`filename_18`),
KEY `filename_19` (`filename_19`),
KEY `filename_20` (`filename_20`),
KEY `member_id` (`member_id`),
KEY `filename_1` (`filename_1`),
KEY `filename_2` (`filename_2`),
KEY `filename_3` (`filename_3`),
KEY `filename_4` (`filename_4`),
KEY `filename_5` (`filename_5`),
KEY `filename_6` (`filename_6`),
KEY `filename_7` (`filename_7`),
KEY `filename_8` (`filename_8`),
KEY `filename_9` (`filename_9`),
KEY `filename_10` (`filename_10`),
KEY `private_1` (`private_1`),
KEY `private_2` (`private_2`),
KEY `private_3` (`private_3`),
KEY `private_4` (`private_4`),
KEY `private_5` (`private_5`),
KEY `private_6` (`private_6`),
KEY `private_7` (`private_7`),
KEY `private_8` (`private_8`),
KEY `private_9` (`private_9`),
KEY `private_10` (`private_10`),
KEY `private_11` (`private_11`),
KEY `private_12` (`private_12`),
KEY `private_13` (`private_13`),
KEY `private_14` (`private_14`),
KEY `private_15` (`private_15`),
KEY `private_16` (`private_16`),
KEY `private_17` (`private_17`),
KEY `private_18` (`private_18`),
KEY `private_19` (`private_19`),
KEY `private_20` (`private_20`),
KEY `password` (`password`),
KEY `description_1` (`description_1`),
KEY `description_2` (`description_2`),
KEY `description_3` (`description_3`),
KEY `description_4` (`description_4`),
KEY `description_5` (`description_5`),
KEY `description_6` (`description_6`),
KEY `description_7` (`description_7`),
KEY `description_8` (`description_8`),
KEY `description_9` (`description_9`),
KEY `description_10` (`description_10`),
KEY `description_11` (`description_11`),
KEY `description_12` (`description_12`),
KEY `description_13` (`description_13`),
KEY `description_14` (`description_14`),
KEY `description_15` (`description_15`),
KEY `description_16` (`description_16`),
KEY `description_17` (`description_17`),
KEY `description_18` (`description_18`),
KEY `description_19` (`description_19`),
KEY `description_20` (`description_20`),
KEY `filename_10_2` (`filename_10`),
KEY `filename_10_3` (`filename_10`)
) ENGINE=MyISAM AUTO_INCREMENT=11174 DEFAULT CHARSET=latin1
CREATE TABLE `dt_usersonline` (
`id` int(8) NOT NULL AUTO_INCREMENT,
`timestamp` int(15) NOT NULL DEFAULT '0',
`ip` varchar(40) NOT NULL DEFAULT '',
`login` varchar(25) NOT NULL DEFAULT '',
`userid` int(10) NOT NULL DEFAULT '0',
`session_id` varchar(255) NOT NULL DEFAULT '',
PRIMARY KEY (`id`),
KEY `id_2` (`id`),
KEY `timestamp` (`timestamp`),
KEY `ip` (`ip`),
KEY `login` (`login`),
KEY `userid` (`userid`),
KEY `session_id` (`session_id`)
) ENGINE=MyISAM AUTO_INCREMENT=4424348 DEFAULT CHARSET=latin1
CREATE TABLE `dt_members` (
`id` int(11) NOT NULL AUTO_INCREMENT,...
CREATE TABLE `dt_photos` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`member_id` varchar(255) DEFAULT NULL,..
You are joining on your query as:
LEFT JOIN dt_photos AS ph ON m.id=ph.member_id
Notice how every other time you join on member_id to m.id the column looks like member_id int(11) DEFAULT NULL. Your join criteria should be on the same data type, otherwise it looks like it is doing a full table scan. It's not using the key because it believes that it has to scan the varchar field rather than utilizing indexes for on clause.
Try Altering column member_id to that of m.id from dt_members
It is interesting to note, had you actually tried to create a foreign key constraint, it would not have allowed you to because of the mismatched data types...
Warning: MYISAM table will be locked during the duration of the later table process.
Try moving where conditions into the corresponding join condition:
SELECT DISTINCT m.id AS memberID , m.login , m.age , p.gender
, p.name AS header , p.id AS profileID , p.city , p.state , p.lastlogin
, o.login AS online , c.name AS country , ph.filename_1 AS pic
FROM dt_members AS m
INNER JOIN dt_profile_approved AS p ON m.id=p.member_id
AND p.status=1 AND p.gender='Female' -- Moved from WHERE clause
LEFT JOIN dt_privacy AS pv ON m.id=pv.member_id
AND (pv.unsearchable IS NULL OR pv.unsearchable='') -- Moved from WHERE clause
INNER JOIN dt_countries AS c ON c.id=p.country
LEFT JOIN dt_photos AS ph ON m.id=ph.member_id
LEFT JOIN dt_usersonline AS o ON m.login=o.login
WHERE m.age BETWEEN 25 AND 40
ORDER BY p.lastlogin DESC
LIMIT 0, 21;
While where conditions are made after all joins, join conditions are evaluated during the join, so vast numbers of unnecessary joins can be avoided early.
Although theoretically the query optimizer should do this for you, I have found the mysql can be particularly dense in this regard.
It looks like most of the selectivity on your query is on your dt_profile_approved table. Try creating a compound index on (status, gender, lastlogin)
This should allow an index-only selection and ordering.
Your query does the notorious SELECT a lot ORDER BY something DESC LIMIT tinynumber pattern. This is expensive. Try a deferred join. Start by getting the interesting items from your dt_profile_approved table, like this:
SELECT member_id, lastlogin
FROM dt_profile_approved
WHERE status=1
AND gender='Female'
ORDER BY lastlogin DESC
This subquery can be optimized very cleanly with a compound index on (status, gender, lastlogin, member_id). That's called a covering index. It has a great benefit: no extra sorting is required, because the index is already sorted.
Based on the exact query you showed us, the order in the index of gender and status doesn't matter. But I guess you have another query that looks for males, and you may have one that omits that. So status is likely to be the more selective field across all your queries. (Guessing.)
Then, join that subquery into the rest of your query... that will look something like this.
SELECT DISTINCT
m.id AS memberID , m.login , m.age,
p.gender. p.name AS header , p.id AS profileID ,
p.city , p.state , p.lastlogin,
o.login AS online ,
c.name AS country , ph.filename_1 AS pic
FROM (
SELECT member_id, lastlogin
FROM dt_profile_approved
WHERE status=1
AND gender='Female'
ORDER BY lastlogin DESC
) AS sel
INNER JOIN dt_profile_approved AS p ON sel.member_id = p.member_id
INNER JOIN dt_members AS m ON m.id=sel.member_id
LEFT JOIN dt_privacy AS pv ON m.id=pv.member_id
INNER JOIN dt_countries AS c ON c.id=p.country
LEFT JOIN dt_photos AS ph ON m.id=ph.member_id
LEFT JOIN dt_usersonline AS o ON m.login=o.login
WHERE (pv.unsearchable IS NULL OR pv.unsearchable='')
AND m.age BETWEEN 25 AND 40
ORDER BY sel.lastlogin DESC
LIMIT 0, 21;
If this works, it will be because it can limit the labor of sorting using the index.
I'm not sure why this query is taking 4 minutes to complete:
SELECT
su.sid,u.uid,u.display_name,u.locale
FROM user u
LEFT JOIN subscription_user su ON su.uid = u.uid
ORDER BY u.display_name DESC
LIMIT 0,25;
Well, I know it's due to the order, remove it and it's very fast. If I change to using INNER JOIN instead it's fast but the issue is not all users may be in the subscription_user table.
CREATE TABLE `user` (
`uid` int(11) NOT NULL AUTO_INCREMENT,
`password` varchar(100) DEFAULT NULL,
`user_type` varchar(10) NOT NULL DEFAULT 'user',
`display_name` varchar(50) NOT NULL,
`email` varchar(100) NOT NULL,
`locale` varchar(8) DEFAULT 'en',
`last_login` datetime DEFAULT NULL,
`auth_type` varchar(10) DEFAULT NULL,
`auth_data` varchar(500) DEFAULT NULL,
`inactive` tinyint(4) NOT NULL DEFAULT '0',
`receive_email` tinyint(4) NOT NULL DEFAULT '1',
`stateid` int(10) DEFAULT NULL,
`owner_group_id` int(11) DEFAULT NULL,
`signature` varchar(500) DEFAULT NULL,
`raw_signature` varchar(500) DEFAULT NULL,
`round_robin` smallint(5) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`uid`),
UNIQUE KEY `email` (`email`),
KEY `stateid` (`stateid`) USING BTREE,
KEY `user_type` (`user_type`) USING BTREE,
KEY `name` (`display_name`)
) ENGINE=InnoDB AUTO_INCREMENT=28343 DEFAULT CHARSET=latin1;
CREATE TABLE `subscription_user` (
`sid` varchar(50) NOT NULL,
`uid` int(11) NOT NULL,
`deleted` tinyint(4) NOT NULL DEFAULT '0',
`forum_user` varchar(50) NOT NULL,
PRIMARY KEY (`sid`,`uid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
When you have an SQL query, the index can only really help you if the first column in the index is part of the query.
Your query joins su.uid = u.uid and the optimizer will not be able to use that to reference the first column in the subscription primary key index.
You should either reverse the order of the columns in the primary key, or alternatively, you should add a foreign key index, or an independent index on the uid
i have a query as given below
SELECT MONTHNAME( f.receipt_date ) AS MONTH ,
SUM( CASE WHEN fm.fee_name = 'University Fees' THEN f.fee_amount END ) AS A
FROM fee_type_masters fm
INNER JOIN student_fee_collections f ON fm.id = f.fee_type
GROUP BY MONTH( f.receipt_date )
and my tables are like
CREATE TABLE `fee_type_masters` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`fee_name` varchar(255) DEFAULT NULL,
`fee_type` varchar(255) DEFAULT NULL,
`year` varchar(255) DEFAULT NULL,
`student_type` int(11) DEFAULT NULL,
`due_date` varchar(255) DEFAULT NULL,
`amount` float DEFAULT NULL,
`accounts_master_id` varchar(255) DEFAULT NULL,
`comments` varchar(255) DEFAULT NULL,
`status` varchar(255) DEFAULT NULL,
`sem` varchar(255) DEFAULT NULL,
`degree_id` int(11) DEFAULT NULL,
`approve_needed` varchar(255) DEFAULT NULL,
`concession_allowed` varchar(255) DEFAULT NULL,
`created_at` datetime DEFAULT NULL,
`updated_at` datetime DEFAULT NULL,
`relation_id` int(11) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=latin1
CREATE TABLE `student_fee_collections` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`student_id` int(11) DEFAULT NULL,
`fee_type` int(11) DEFAULT NULL,
`fee_amount` int(11) DEFAULT '0',
`due` int(11) DEFAULT '0',
`received` int(11) DEFAULT '0',
`concession` int(11) DEFAULT '0',
`receipt_no` int(11) DEFAULT NULL,
`receipt_date` date DEFAULT NULL,
`received_by` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`amount` int(11) DEFAULT '0',
`late_fee` int(11) DEFAULT '0',
`pay_mode` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`comments` text COLLATE utf8_unicode_ci,
`created_at` datetime DEFAULT NULL,
`updated_at` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `student_id` (`fee_type`,`student_id`)
) ENGINE=InnoDB AUTO_INCREMENT=325 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
each is having more than 20000 rows
and my explain plan is like
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE fm ALL PRIMARY NULL NULL NULL 7000 Using temporary; Using filesort
1 SIMPLE f ref student_id student_id 5 emsnew.fm.id 28000 Using where
any one please tell me how to optimize the query or rewrite.
Create Index on fee_type column in student_fee_collections and recheck the output of Explain Query. And update the same in your question.
Create Index on fee_type column in student_fee_colle
and run the following query
SELECT MONTHNAME( f.receipt_date ) AS MONTH ,
SUM( CASE WHEN fm.fee_name = 'University Fees' THEN f.fee_amount END ) AS A
FROM fee_type_masters fm
INNER JOIN student_fee_collections f USE index(fee_type_IDX) ON fm.id = f.fee_type
GROUP BY MONTH( f.receipt_date )