optimise mysql join query - mysql

following mysql query taking more time to fetch results so i need help to optimise
SELECT status,
count(status) AS COUNT,
build_id,
results_bu.name,
results_bu.creation_ts,
results_bu.release_date
FROM
(SELECT NHTC.parent_id AS tsuite_id,
NHTC.id AS tcase_id,
NHTC.name AS name,
TPTCV.tcversion_id,
TPTCV.platform_id,
E.build_id,
E.tcversion_number AS VERSION,
TCV.tc_external_id AS external_id,
E.id AS executions_id,
E.status AS status,
(TPTCV.urgency * TCV.importance) AS urg_imp
FROM testplan_tcversions TPTCV
/* GO FOR Absolute LATEST exec ID On BUILD,PLATFORM */
JOIN
(SELECT EE.tcversion_id,
EE.testplan_id,
EE.platform_id,
EE.build_id,
MAX(EE.id) AS id
FROM executions EE
WHERE EE.testplan_id=13954
AND EE.build_id IN
(SELECT id
FROM builds
WHERE testplan_id = 13954
AND active=1
AND (creation_ts BETWEEN DATE('2015-01-01') AND DATE('2015-05-10'))
ORDER BY name ASC)
GROUP BY EE.tcversion_id,
EE.testplan_id,
EE.platform_id,
EE.build_id) AS LEBBP ON LEBBP.testplan_id = TPTCV.testplan_id
AND LEBBP.platform_id = TPTCV.platform_id
AND LEBBP.tcversion_id = TPTCV.tcversion_id
AND LEBBP.testplan_id = 13954
/* Get execution status WRITTEN on DB */
JOIN executions E ON E.id = LEBBP.id
AND E.build_id = LEBBP.build_id
/* Get Test Case info from Test Case Version */
JOIN nodes_hierarchy NHTCV ON NHTCV.id = TPTCV.tcversion_id
/* Get Test Suite info from Test Case */
JOIN nodes_hierarchy NHTC ON NHTC.id = NHTCV.parent_id
/* Get Test Case Version attributes */
JOIN tcversions TCV ON TCV.id = E.tcversion_id
WHERE TPTCV.testplan_id=13954
AND E.build_id IN
(SELECT id
FROM builds
WHERE testplan_id = 13954
AND active=1
AND (creation_ts BETWEEN DATE('2015-01-01') AND DATE('2015-05-10'))
ORDER BY name ASC)
UNION ALL
/* sqlUnion Test suites - not run */
SELECT NHTC.parent_id AS tsuite_id,
NHTC.id AS tcase_id,
NHTC.name AS name,
TPTCV.tcversion_id,
TPTCV.platform_id,
BU.id AS build_id,
TCV.VERSION,
TCV.tc_external_id AS external_id,
COALESCE(E.id,-1) AS executions_id,
COALESCE(E.status,'n') AS status,
(TPTCV.urgency * TCV.importance) AS urg_imp
FROM testplan_tcversions TPTCV
/* Needed to be able to put a value on build_id on output set */
JOIN builds BU ON BU.id IN
(SELECT id
FROM builds
WHERE testplan_id = 13954
AND active=1
AND (creation_ts BETWEEN DATE('2015-01-01') AND DATE('2015-05-10'))
ORDER BY name ASC)
/* GO FOR Absolute LATEST exec ID On BUILD,PLATFORM */
LEFT OUTER JOIN
(SELECT EE.tcversion_id,
EE.testplan_id,
EE.platform_id,
EE.build_id,
MAX(EE.id) AS id
FROM executions EE
WHERE EE.testplan_id=13954
AND EE.build_id IN
(SELECT id
FROM builds
WHERE testplan_id = 13954
AND active=1
AND (creation_ts BETWEEN DATE('2015-01-01') AND DATE('2015-05-10'))
ORDER BY name ASC)
GROUP BY EE.tcversion_id,
EE.testplan_id,
EE.platform_id,
EE.build_id) AS LEBBP ON LEBBP.testplan_id = TPTCV.testplan_id
AND LEBBP.platform_id = TPTCV.platform_id
AND LEBBP.tcversion_id = TPTCV.tcversion_id
AND LEBBP.build_id = BU.id
AND LEBBP.testplan_id = 13954
/* Get execution status WRITTEN on DB */
LEFT OUTER JOIN executions E ON E.build_id = LEBBP.build_id
AND E.testplan_id = TPTCV.testplan_id
AND E.platform_id = TPTCV.platform_id
AND E.tcversion_id = TPTCV.tcversion_id
/* Get Test Case info from Test Case Version */
JOIN nodes_hierarchy NHTCV ON NHTCV.id = TPTCV.tcversion_id
/* Get Test Suite info from Test Case */
JOIN nodes_hierarchy NHTC ON NHTC.id = NHTCV.parent_id
/* Get Test Case Version attributes */
JOIN tcversions TCV ON TCV.id = TPTCV.tcversion_id
WHERE TPTCV.testplan_id=13954
AND BU.id IN
(SELECT id
FROM builds
WHERE testplan_id = 13954
AND active=1
AND (creation_ts BETWEEN DATE('2015-01-01') AND DATE('2015-05-10'))
ORDER BY name ASC)
/* Get REALLY NOT RUN => BOTH LEBBP.id AND E.id NULL */
AND E.id IS NULL
AND LEBBP.id IS NULL) AS RESULT
LEFT OUTER JOIN builds AS results_bu ON results_bu.id=RESULT.build_id
GROUP BY status,
build_id;
results for EXPLAIN as follows
id select_type table type possible_keys key key_len ref rows Extra
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
1 PRIMARY <derived2> ALL 32768 Using temporary; Using filesort
1 PRIMARY results_bu eq_ref PRIMARY PRIMARY 4 result.build_id 1
2 DERIVED <derived3> ALL 32229 Using where
2 DERIVED NHTCV eq_ref PRIMARY,pid_m_nodeorder PRIMARY 4 LEBBP.tcversion_id 1
2 DERIVED NHTC eq_ref PRIMARY PRIMARY 4 bitnami_testlink.NHTCV.parent_id 1
2 DERIVED TPTCV eq_ref testplan_tcversions_tplan_tcversion testplan_tcversions_tplan_tcversion 12 LEBBP.tcversion_id,LEBBP.platform_id 1
2 DERIVED E eq_ref PRIMARY PRIMARY 4 LEBBP.id 1 Using where
2 DERIVED TCV eq_ref PRIMARY PRIMARY 4 bitnami_testlink.E.tcversion_id 1
4 DEPENDENT
SUBQUERY builds unique_subquery PRIMARY,name,testplan_id PRIMARY 4 func 1 Using where
3 DERIVED EE ref executions_idx1 executions_idx1 4 135350 Using where; Using index
3 DERIVED BB eq_ref PRIMARY,name,testplan_id PRIMARY 4 bitnami_testlink.EE.build_id 1 Using where
5 UNION BU index testplan_id 4 569 Using where; Using index
5 UNION TPTCV ref testplan_tcversions_tplan_tcversion testplan_tcversions_tplan_tcversion 4 1472
5 UNION TCV eq_ref PRIMARY PRIMARY 4 bitnami_testlink.TPTCV.tcversion_id 1
5 UNION NHTCV eq_ref PRIMARY,pid_m_nodeorder PRIMARY 4 bitnami_testlink.TPTCV.tcversion_id 1
5 UNION NHTC eq_ref PRIMARY PRIMARY 4 bitnami_testlink.NHTCV.parent_id 1
5 UNION <derived6> ALL 32229 Using where
5 UNION E ref executions_idx1 executions_idx1 16 bitnami_testlink.TPTCV.tcversion_id,
bitnami_testlink.TPTCV.platform_id,
LEBBP.build_id 1 Using where; Not exists
8 DEPENDENT
SUBQUERY builds unique_subquery PRIMARY,name,testplan_id PRIMARY 4 func 1 Using where
7 DEPENDENT
SUBQUERY builds unique_subquery PRIMARY,name,testplan_id PRIMARY 4 func 1 Using where
6 DERIVED EE ref executions_idx1 executions_idx1 4 135350 Using where; Using index
6 DERIVED BB eq_ref PRIMARY,name,testplan_id PRIMARY 4 bitnami_testlink.EE.build_id 1 Using where
UNION RESULT <union2,5> ALL
Actually we are trying to get how many number of execution records passed, failed between period of time for selected Test plan from test link database
Thanks,
Ram

Related

Why does OR in subquery make query so much slower?

I'm using MySQL and have the following query that I was trying to improve:
SELECT
*
FROM
overpayments AS op
JOIN payment_allocations AS overpayment_pa ON overpayment_pa.allocatable_id = op.id
AND overpayment_pa.allocatable_type = 'Overpayment'
JOIN (
SELECT
pa.payment_source_type,
pa.payment_source_id,
ft.conversion_rate
FROM
payment_allocations AS pa
LEFT JOIN line_items AS li ON pa.payment_source_id = li.id
LEFT JOIN credit_notes AS cn ON li.parent_document_id = cn.id
LEFT JOIN financial_transactions AS ft ON (
ft.commercial_document_id = pa.payment_source_id
AND ft.commercial_document_type = pa.payment_source_type
)
OR (
ft.commercial_document_id = cn.id
AND ft.commercial_document_type = 'CreditNote'
)
WHERE
pa.allocatable_type = 'Overpayment'
AND pa.company_id = 14792
AND ft.company_id = 14792
) AS op_bank_transaction_ft ON op_bank_transaction_ft.payment_source_id = overpayment_pa.payment_source_id
AND op_bank_transaction_ft.payment_source_type = overpayment_pa.payment_source_type;
It takes 10s to run. I was able to improve to 0.047s it by removing the OR statement in the subquery and using COALESCE to get the result:
SELECT
*
FROM
overpayments AS op
JOIN payment_allocations AS overpayment_pa ON overpayment_pa.allocatable_id = op.id
AND overpayment_pa.allocatable_type = 'Overpayment'
JOIN (
SELECT
pa.payment_source_type,
pa.payment_source_id,
coalesce(ft_one.conversion_rate, ft_two.conversion_rate)
FROM
payment_allocations AS pa
LEFT JOIN line_items AS li ON pa.payment_source_id = li.id
LEFT JOIN credit_notes AS cn ON li.parent_document_id = cn.id
LEFT JOIN financial_transactions AS ft_one ON (
ft_one.commercial_document_id = pa.payment_source_id
AND ft_one.commercial_document_type = pa.payment_source_type
AND ft_one.company_id = 14792
)
LEFT JOIN financial_transactions AS ft_two ON (
ft_two.commercial_document_id = cn.id
AND ft_two.commercial_document_type = 'CreditNote'
AND ft_two.company_id = 14792
)
WHERE
pa.allocatable_type = 'Overpayment'
AND pa.company_id = 14792
) AS op_bank_transaction_ft ON op_bank_transaction_ft.payment_source_id = overpayment_pa.payment_source_id
AND op_bank_transaction_ft.payment_source_type = overpayment_pa.payment_source_type;
However, I don't really understand why that worked? The original sub query ran very quickly and only returned 2 results, so why would it slow down the query by so much? Explain on the first query returns the following:
# id
select_type
table
partitions
type
possible_keys
key
key_len
ref
rows
filtered
Extra
FIELD13
1
SIMPLE
pa
ref
index_payment_allocations_on_payment_source_id
index_payment_allocations_on_company_id
index_payment_allocations_on_company_id
5
const
191
10.00
Using where
1
SIMPLE
overpayment_pa
ref
index_payment_allocations_on_payment_source_id
index_payment_allocations_on_allocatable_id
index_payment_allocations_on_payment_source_id
5
rails.pa.payment_source_id
1
3.42
Using where
1
SIMPLE
op
eq_ref
PRIMARY
PRIMARY
4
rails.overpayment_pa.allocatable_id
1
100.00
1
SIMPLE
li
eq_ref
PRIMARY
PRIMARY
4
rails.pa.payment_source_id
1
100.00
1
SIMPLE
cn
eq_ref
PRIMARY
PRIMARY
8
rails.li.parent_document_id
1
100.00
Using where; Using index
1
SIMPLE
ft
ALL
transactions_unique_by_commercial_doc
12587878
0.00
Range checked for each record (index map: 0x2)
And for the second I get the following:
# id
select_type
table
partitions
type
possible_keys
key
key_len
ref
rows
filtered
Extra
FIELD13
FIELD14
1
SIMPLE
pa
ref
index_payment_allocations_on_payment_source_id
index_payment_allocations_on_company_id
index_payment_allocations_on_company_id
5
const
191
10.00
Using where
1
SIMPLE
overpayment_pa
ref
index_payment_allocations_on_payment_source_id
index_payment_allocations_on_allocatable_id
index_payment_allocations_on_payment_source_id
5
rails.pa.payment_source_id
1
3.42
Using where
1
SIMPLE
op
eq_ref
PRIMARY
PRIMARY
4
rails.overpayment_pa.allocatable_id
1
100.00
1
SIMPLE
ft_one
ref
transactions_unique_by_commercial_doc
index_financial_transactions_on_company_id
transactions_unique_by_commercial_doc
773
rails.pa.payment_source_id
rails.pa.payment_source_type
1
100.00
Using where
1
SIMPLE
li
eq_ref
PRIMARY
PRIMARY
4
rails.pa.payment_source_id
1
100.00
1
SIMPLE
cn
eq_ref
PRIMARY
PRIMARY
8
rails.li.parent_document_id
1
100.00
Using where; Using index
1
SIMPLE
ft_two
ref
transactions_unique_by_commercial_doc
index_financial_transactions_on_company_id
transactions_unique_by_commercial_doc
773
rails.cn.id
const
1
100.00
Using where
but I don't really know how to interpret those results.
Look at the right side of the last row of your first EXPLAIN. It didn't use an index, and it had to scan through megarows. That's slow. Your second query used indexes for every step of the query, so it was much faster.
If your second query yields correct results, use it and don't look back. Congratulations! You've optimized a query.
OR operations, especially in ON clauses, are harder than usual for the query planner module to satisfy, because they often mean it has to take the union of two separate subqueries. It looks like the planner chose to brute-force it in your case. (brute force === scanning many rows.)
Without knowing your indexes, it's hard to help you further.
Read this to learn more. https://use-the-index-luke.com
These may further speed up the second formulation:
overpayment_pa:
INDEX(payment_source_id, payment_source_type, allocatable_type, allocatable_id)
pa: INDEX(allocatable_type, company_id, payment_source_id, payment_source_type)
financial_transactions:
INDEX(commercial_document_id, commercial_document_type, company_id, conversion_rate)

MySQL query performance improvement

I have a problem with performance with MySQL. How can i improve it?
The situation is following:
Table “backlogsap„ have about 4 mio entries.
Indexes are created
This table have FK and other tables have FK to this table => can’t
create partitions.
This query need about 140 seconds to complete:
select
idmaterial,
materialgroup,
materialgroupcategory,
name,
dispatchgroup,
idsupplier,
group_concat(distinct sellingorganizationname) as sellingorganizationnames,
group_concat(distinct idordertype) as idordertypes,
group_concat(distinct idpositiontype) as idpositiontypes,
sum(newOrUpdated and isCritical) as classA,
sum(newOrUpdated and not isCritical) as classB,
sum(processingstate <3) as classC,
(select count(innerBacklogsAp.idmaterial)
from backlogsap as innerBacklogsAp
where innerBacklogsAp.idmaterial = src.idmaterial and IsDeleted = 0) as countReplacementVehiclerRequests
from
(select
backlogsap.idmaterial as idmaterial,
backlog.processingstate as processingstate,
material.idsupplier as idsupplier,
backlogsap.sellingorganizationname as sellingorganizationname,
backlogsap.idpositiontype as idpositiontype,
backlogsap.idordertype as idordertype,
materialindistributioncenter.dispatchgroup as dispatchgroup,
material.name as name,
material.idmaterialgroup as materialgroup,
materialgroup.idmaterialgroupcategory as materialgroupcategory,
(processingstate = 0 or processingstate = 1) as newOrUpdated,
((cancellation.state is not null and cancellation.state = 0 ) or
(reminderrequest.state is not null and (reminderrequest.state = 2 or reminderrequest.state = 0))
) as isCritical
from backlogsap
join backlog using (idbacklogsap)
left join cancellation using (idcancellation)
left join reminderrequest on backlog.IdReminderRequest = reminderrequest.idreminder
left join material using (idmaterial)
left join materialindistributioncenter using (idmaterial, iddistributioncenter)
left join materialgroup using (idmaterialgroup)
where (idcancellation is null or cancellation.State not in (1)) and
backlogsap.isdeleted = 0 and
backlogsap.idordertype not in ('ZAP', 'ZAK', 'ZAKO', 'ZAKZ', 'ZAPM') and
iddistributioncenter = 1469990
) as src
group by idmaterial
order by classA desc, classB desc, classC, idmaterial desc
Explain
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY <derived3> ALL 26960 Using temporary; Using filesort
3 DERIVED backlogsap index_merge PRIMARY,fk_BacklogSap_OrderType1_idx,
fk_BacklogSap_MaterialInDistributionCenter1_idx,
perform_backlogsap_isdeleted,
fk_BacklogSap_DistributionCenter_idx perform_backlogsap_isdeleted,fk_BacklogSap_DistributionCenter_idx 1,4 35946 Using intersect(perform_backlogsap_isdeleted,fk_BacklogSap_DistributionCenter_idx); Using where
3 DERIVED backlog eq_ref idBacklogSAP_UNIQUE,
fk_Backlog_BacklogSap1_idx,
fk_Backlog_Cancellation1_idx idBacklogSAP_UNIQUE 4 ...backlogsap.IdBacklogSap 1
3 DERIVED cancellation eq_ref PRIMARY PRIMARY 4 ...backlog.IdCancellation 1 Using where
3 DERIVED reminderrequest eq_ref PRIMARY PRIMARY 4 ...backlog.IdReminderRequest 1
3 DERIVED material eq_ref PRIMARY PRIMARY 45 ...backlogsap.IdMaterial 1
3 DERIVED materialindistributioncenter eq_ref PRIMARY,
unqiue_IdDistributionCenter_IdMaterial,
fk_MaterialDistributionCenter_DistributionCenter1_idx,
fk_MaterialDistributionCenter_Material1_idx PRIMARY 49 const,...backlogsap.IdMaterial 1
3 DERIVED materialgroup eq_ref PRIMARY PRIMARY 137 ....material.IdMaterialGroup 1
2 DEPENDENT SUBQUERY innerBacklogsAp ref perform_backlogsap_isdeleted,
idx_backlogsap_IdMaterial idx_backlogsap_IdMaterial 45 func 8 Using where
Solved: created combined Index (idmaterial, IsDeleted)

MySQL query hang

If I execute the following query
select * from medienkatalog_nct.sphinx_index limit 380000, 10000
the query hang forever between status "none" and "closing tables".
explain select * from medienkatalog_nct.sphinx_index;
# id, select_type, table, type, possible_keys, key, key_len, ref, rows, Extra
1 SIMPLE katalognummer index PRIMARY,artikelNr,katalognr,idvideothek artikelNr 4 275870 Using index
1 SIMPLE artikel ref PRIMARY PRIMARY 4 medienkatalog_nct.katalognummer.katalogNr 1
1 SIMPLE artikel eq_ref PRIMARY,id_artikelnr PRIMARY 4 medienkatalog_nct.katalognummer.artikelNr 1
1 SIMPLE relation_kategorie ref PRIMARY PRIMARY 4 medienkatalog_nct.artikel.artikelNr 1 Using index
1 SIMPLE kategorie eq_ref PRIMARY PRIMARY 4 medienkatalog_nct.relation_kategorie.id_kategorie 1
1 SIMPLE relation_inhalt ref PRIMARY PRIMARY 4 medienkatalog_nct.artikel.artikelNr 1 Using index
1 SIMPLE inhalt eq_ref PRIMARY,id_inhalt PRIMARY 4 medienkatalog_nct.relation_inhalt.id_inhalt 1
1 SIMPLE relation_medium ref PRIMARY PRIMARY 4 medienkatalog_nct.katalognummer.artikelNr 1 Using index
1 SIMPLE medium eq_ref PRIMARY PRIMARY 4 medienkatalog_nct.relation_medium.id_medium 1
1 SIMPLE relation_plattform ref PRIMARY PRIMARY 4 medienkatalog_nct.artikel.artikelNr 1 Using index
1 SIMPLE plattform eq_ref PRIMARY PRIMARY 4 medienkatalog_nct.relation_plattform.id_plattform 1
1 SIMPLE relation_hersteller ref PRIMARY PRIMARY 4 medienkatalog_nct.artikel.artikelNr 1 Using index
1 SIMPLE hersteller eq_ref PRIMARY,id_hersteller PRIMARY 4 medienkatalog_nct.relation_hersteller.id_hersteller 1
1 SIMPLE relation_titel eq_ref PRIMARY PRIMARY 4 medienkatalog_nct.katalognummer.artikelNr 1
1 SIMPLE titel eq_ref id_titel id_titel 4 medienkatalog_nct.relation_titel.id_titel 1 Using index
1 SIMPLE relation_freigabe ref PRIMARY PRIMARY 4 medienkatalog_nct.artikel.artikelNr 1 Using index
1 SIMPLE freigabe eq_ref PRIMARY,id_freigabe PRIMARY 4 medienkatalog_nct.relation_freigabe.id_freigabe 1
1 SIMPLE media eq_ref PRIMARY PRIMARY 4 medienkatalog_nct.katalognummer.artikelNr 1
1 SIMPLE artikel_videothek ref PRIMARY,i_katalognr,i_id_videothek PRIMARY 4 medienkatalog_nct.katalognummer.katalogNr 5 Using where
1 SIMPLE videothek eq_ref PRIMARY,INDEX PRIMARY 4 medienkatalog_nct.artikel_videothek.id_videothek 1 Using where
Edit: the SQL query
CREATE
ALGORITHM = UNDEFINED
DEFINER = `dbadmin`#`%`
SQL SECURITY DEFINER
VIEW `sphinx_index` AS
SELECT
((`medienkatalog_nct`.`artikel_videothek`.`id_videothek` * 1000000000) + `medienkatalog_nct`.`artikel`.`artikelNr`) AS `pkey`,
`medienkatalog_nct`.`fkey`(`medienkatalog_nct`.`artikel`.`artikelNr`,
`medienkatalog_nct`.`media`.`key_media`) AS `fkey`,
`medienkatalog_nct`.`artikel_videothek`.`id_videothek` AS `id_videothek`,
`medienkatalog_nct`.`artikel`.`artikelNr` AS `artikelnr`,
`medienkatalog_nct`.`artikel_videothek`.`katalognr` AS `katalognr`,
`medienkatalog_nct`.`media`.`key_media` AS `key_media`,
`medienkatalog_nct`.`media`.`has_werbebanner` AS `has_werbebanner`,
`medienkatalog_nct`.`media`.`has_mailbanner` AS `has_mailbanner`,
`medienkatalog_nct`.`media`.`has_animation` AS `has_animation`,
`medienkatalog_nct`.`media`.`has_klappe` AS `has_klappe`,
`medienkatalog_nct`.`media`.`has_klappeemo` AS `has_klappeemo`,
`medienkatalog_nct`.`media`.`has_podcast` AS `has_podcast`,
IF(ISNULL(`medienkatalog_nct`.`media`.`top20rang`),
100,
IF((`medienkatalog_nct`.`media`.`top20rang` = 0),
50,
`medienkatalog_nct`.`media`.`top20rang`)) AS `top20rang`,
`medienkatalog_nct`.`media`.`has_trailer` AS `has_trailer`,
`medienkatalog_nct`.`media`.`has_traileremo` AS `has_traileremo`,
UNIX_TIMESTAMP(`medienkatalog_nct`.`media`.`verleihstart`) AS `verleihstart`,
`medienkatalog_nct`.`media`.`has_vorschaubanner` AS `has_vorschaubanner`,
TOP_PREIS(`medienkatalog_nct`.`artikel_videothek`.`id_videothek`,
`medienkatalog_nct`.`artikel_videothek`.`katalognr`) AS `is_toppreis`,
IF(((`nachbestellsystem`.`artikel`.`aktiv` = 1)
AND (`nachbestellsystem`.`artikel`.`lieferbar` = 1)
AND ISNULL(`nachbestellsystem`.`artikel`.`dekommissionierung`)),
1,
0) AS `is_kommission`,
`medienkatalog_nct`.`kategorie`.`beschreibung` AS `kategorie`,
`medienkatalog_nct`.`titel`.`titel` AS `titel`,
`medienkatalog_nct`.`titel`.`titel` AS `titel_sort`,
`medienkatalog_nct`.`str_to_ord`(`medienkatalog_nct`.`titel`.`titel`) AS `titel_ord`,
`medienkatalog_nct`.`titel`.`untertitel` AS `untertitel`,
`medienkatalog_nct`.`medium`.`id_medium` AS `id_medium`,
`medienkatalog_nct`.`medium`.`beschreibung` AS `medium`,
`medienkatalog_nct`.`plattform`.`id_plattform` AS `id_plattform`,
`medienkatalog_nct`.`plattform`.`beschreibung` AS `plattform`,
ID_VERSIONS(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `vkey`,
`medienkatalog_nct`.`version`(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `version`,
ID_GENRES(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `gkey`,
GENRE(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `genre`,
UNIX_TIMESTAMP(`medienkatalog_nct`.`artikel`.`artikelstart`) AS `artikelstart`,
UNIX_TIMESTAMP(`medienkatalog_nct`.`artikel_videothek`.`erfassungsdatum`) AS `erfassungsdatum`,
`medienkatalog_nct`.`hersteller`.`id_hersteller` AS `id_hersteller`,
`medienkatalog_nct`.`hersteller`.`beschreibung` AS `hersteller`,
ACTOR(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `darsteller`,
COUNTRY(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `land`,
DIRECTOR(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `regie`,
FORMAT(RANKING(`medienkatalog_nct`.`artikel`.`artikelNr`),
2) AS `sterne`,
COUNT_RANKING(`medienkatalog_nct`.`artikel`.`artikelNr`) AS `bewertungen`,
ANZAHL_VL(`medienkatalog_nct`.`artikel_videothek`.`id_videothek`,
`medienkatalog_nct`.`artikel`.`artikelNr`,
(NOW() - INTERVAL 1 MONTH),
NOW()) AS `anzahl_vl`,
ANZAHL_VK(`medienkatalog_nct`.`artikel_videothek`.`id_videothek`,
`medienkatalog_nct`.`artikel`.`artikelNr`,
(NOW() - INTERVAL 1 MONTH),
NOW()) AS `anzahl_vk`,
`medienkatalog_nct`.`inhalt`.`beschreibung` AS `inhalt`,
`medienkatalog_nct`.`inhalt`.`aufmacher` AS `aufmacher`,
`medienkatalog_nct`.`relation_kategorie`.`id_kategorie` AS `id_kategorie`,
`medienkatalog_nct`.`relation_titel`.`id_titel` AS `id_titel`,
`medienkatalog_nct`.`relation_freigabe`.`id_freigabe` AS `id_freigabe`,
`medienkatalog_nct`.`freigabe`.`beschreibung` AS `freigabe`,
CONCAT(IF((`medienkatalog_nct`.`artikel_videothek`.`ist_verleih` <> 0),
'Leihartikel ',
''),
IF((`medienkatalog_nct`.`artikel_videothek`.`ist_gebraucht` <> 0),
'Gebrauchtartikel Verkaufsartikel ',
''),
IF(((`medienkatalog_nct`.`artikel_videothek`.`ist_verkauf` <> 0)
AND (`medienkatalog_nct`.`artikel_videothek`.`bestand_vk` > 0)),
IF((`medienkatalog_nct`.`artikel_videothek`.`ist_gebraucht` = 0),
'Verkaufsartikel Neuware',
'Neuware'),
'')) AS `artikelart`,
`medienkatalog_nct`.`artikel_videothek`.`deleted` AS `deleted`
FROM
(((((((((((((((((((`medienkatalog_nct`.`artikel`
JOIN `medienkatalog_nct`.`katalognummer` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`katalognummer`.`artikelNr`)))
JOIN `medienkatalog_nct`.`artikel_videothek` ON (((`medienkatalog_nct`.`katalognummer`.`katalogNr` = `medienkatalog_nct`.`artikel_videothek`.`katalognr`)
AND (`medienkatalog_nct`.`katalognummer`.`vtid` IN (0 , `medienkatalog_nct`.`artikel_videothek`.`id_videothek`)))))
LEFT JOIN `medienkatalog_nct`.`relation_kategorie` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`relation_kategorie`.`artikelNr`)))
LEFT JOIN `medienkatalog_nct`.`kategorie` ON ((`medienkatalog_nct`.`relation_kategorie`.`id_kategorie` = `medienkatalog_nct`.`kategorie`.`id_kategorie`)))
LEFT JOIN `medienkatalog_nct`.`relation_inhalt` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`relation_inhalt`.`artikelNr`)))
LEFT JOIN `medienkatalog_nct`.`inhalt` ON ((`medienkatalog_nct`.`relation_inhalt`.`id_inhalt` = `medienkatalog_nct`.`inhalt`.`id_inhalt`)))
LEFT JOIN `medienkatalog_nct`.`relation_medium` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`relation_medium`.`artikelNr`)))
LEFT JOIN `medienkatalog_nct`.`medium` ON ((`medienkatalog_nct`.`relation_medium`.`id_medium` = `medienkatalog_nct`.`medium`.`id_medium`)))
LEFT JOIN `medienkatalog_nct`.`relation_plattform` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`relation_plattform`.`artikelNr`)))
LEFT JOIN `medienkatalog_nct`.`plattform` ON ((`medienkatalog_nct`.`relation_plattform`.`id_plattform` = `medienkatalog_nct`.`plattform`.`id_plattform`)))
LEFT JOIN `medienkatalog_nct`.`relation_hersteller` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`relation_hersteller`.`artikelNr`)))
LEFT JOIN `medienkatalog_nct`.`hersteller` ON ((`medienkatalog_nct`.`relation_hersteller`.`id_hersteller` = `medienkatalog_nct`.`hersteller`.`id_hersteller`)))
LEFT JOIN `medienkatalog_nct`.`relation_titel` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`relation_titel`.`artikelNr`)))
LEFT JOIN `medienkatalog_nct`.`titel` ON ((`medienkatalog_nct`.`relation_titel`.`id_titel` = `medienkatalog_nct`.`titel`.`id_titel`)))
LEFT JOIN `medienkatalog_nct`.`relation_freigabe` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`relation_freigabe`.`artikelNr`)))
LEFT JOIN `medienkatalog_nct`.`freigabe` ON ((`medienkatalog_nct`.`freigabe`.`id_freigabe` = `medienkatalog_nct`.`relation_freigabe`.`id_freigabe`)))
LEFT JOIN `medienkatalog_nct`.`media` ON ((`medienkatalog_nct`.`artikel`.`artikelNr` = `medienkatalog_nct`.`media`.`artikelnr`)))
JOIN `videotaxi`.`videothek` ON ((`medienkatalog_nct`.`artikel_videothek`.`id_videothek` = `videotaxi`.`videothek`.`id_videothek`)))
LEFT JOIN `nachbestellsystem`.`artikel` ON ((`nachbestellsystem`.`artikel`.`katalogNr` = `medienkatalog_nct`.`katalognummer`.`katalogNr`)))
WHERE
(((`medienkatalog_nct`.`artikel_videothek`.`ist_gebraucht` <> 0)
OR (`medienkatalog_nct`.`artikel_videothek`.`ist_verleih` <> 0)
OR (`medienkatalog_nct`.`artikel_videothek`.`ist_verkauf` <> 0))
AND (`videotaxi`.`videothek`.`offen` <> 0)
AND (`medienkatalog_nct`.`artikel_videothek`.`deleted` = 0))
The query worked well for more than 3 years until last week.
Does anybody have ideas?

How can I determine the best place to add indexes in mysql?

I've tried doing some searching, but I am having a bit of trouble getting a grasp on this.
Once I explain a query, how can I look at the information and determine the best place to add an index to speed up the query.
For example I have this query:
SELECT a.app_id, DATE_FORMAT(app_datetime, '%c/%d/%y %H:%i') as app_datetime, app_language
, if((select count(w.app_id) from li_app_cnc as w where w.app_id = a.app_id) > 0 , concat('CNC: ',(select cnc_note from li_app_cnc as w where w.app_id = a.app_id)),
if((select count(x.app_id) from li_app_dnc as x where x.app_id = a.app_id) > 0, concat('DNC: ', (select cancel_note from li_app_dnc as x where x.app_id = a.app_id)),
if((select count(y.app_id) from li_app_canceled as y where y.app_id = a.app_id) > 0, concat('Canceled: ', (select cancel_note from li_app_canceled as y where y.app_id = a.app_id)),
concat(h.emp_firstname, ' ', h.emp_lastname)))) as int_id, app_notes, app_facility, app_department
, app_requesting_person, app_service_provider
, cast(AES_DECRYPT(les_name, '$privatekey') as char) as les_name
, les_dob, cast(AES_DECRYPT(les_medicaid_id, '$privatekey') as char) as les_medicaid_id
, billing_total_time, billing_workorder_received, billing_admin_fee
, billing_notes, created_by, created_on, modified_by, modified_on, wo_entered_by
, t.cancel_code, t.cancel_note
, u.cnc_code, u.cnc_note
, v.cancel_code as dnc_code, v.cancel_note as dnc_note
FROM li_appointments.li_appointments as a
left Join orangehrm_li.hs_hr_employee as h on a.terp_id = h.employee_id
left Join li_appointments.li_app_canceled as t on t.app_id = a.app_id
left Join li_appointments.li_app_cnc as u on u.app_id = a.app_id
left Join li_appointments.li_app_dnc as v on v.app_id = a.app_id
where (app_client_id in (select account_number from li_appointments.li_client_access
where id = $userid) or created_by = '$username')
and date(app_datetime) = date(now())
and a.app_id not in (select f.app_id from li_app_dnc as f)
Which takes about 14 secconds
It is explained as:
1 PRIMARY a ALL 37539 Using where
1 PRIMARY h ALL 1036
1 PRIMARY t eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1
1 PRIMARY u eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1
1 PRIMARY v eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1
9 DEPENDENT SUBQUERY f unique_subquery PRIMARY PRIMARY 4 func 1 Using index
8 DEPENDENT SUBQUERY li_client_access ALL 72 Using where
7 DEPENDENT SUBQUERY y eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1
6 DEPENDENT SUBQUERY y eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1 Using index
5 DEPENDENT SUBQUERY x eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1
4 DEPENDENT SUBQUERY x eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1 Using index
3 DEPENDENT SUBQUERY w eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1
2 DEPENDENT SUBQUERY w eq_ref PRIMARY PRIMARY 4 li_appointments.a.app_id 1 Using index
Some thumb rules:
Index all the fields on which you will define WHERE conditions.
Index all the fields on which you will define relationships.
Index all the fields on which you will define grouping criteria.
Avoid indexing everything in your table. Think before creating the indexes.
(Although it depends on your specific needs, I personally avoid creating indexes on floating point columns)
Obviously, the above rules imply that you must index all the fields that work as primary key or as foreign key (if your tables are normalized, you already must have created the appropriate primary keys; if your tables are not normalized, then normalize them).

SELECT query executes for a very long time occasionally

I have a really wierd problem in my MySQL InnoDB database. I have following query:
SELECT DISTINCT p.idProject AS idProject, p.name AS name, 0 AS isConfirm
FROM Projects p
JOIN team_project tp ON (p.idProject = tp.idProject)
JOIN projtimes pt ON (p.idProject = pt.idProject)
JOIN CalledTimesTbl ctt ON (p.idProject = ctt.idProject)
LEFT JOIN NextCalls nc ON (ctt.idCustomer = nc.idCustomer
AND ctt.idProject = nc.idProject)
WHERE tp.idTeam = 158
AND p.activated = 1
AND current_date >= p.projStart
AND current_date < p.confirmStart
AND pt.invitesCount < pt.maxPerPresentation
AND (nc.idCustomer IS NULL OR nc.nextCall < now())
ORDER BY p.name
Generally the query runs fine, but sometimes - for example when I set tp.idTeam = 147 it runs really slow (like 10 or 20 seconds). When I create alternative team and adjust proper tables values to have the same result with different idTeam value - the query executes in a fraction of second.
I profiled the query and noticed that when query executes slowly - there is one thing that consumes most of the time:
Copying to tmp table | 12.489197
I was a bit surprised that the query creates a tmp table but ok - it creates it every time the query executes - also when it executes fast.
I just add that db is designed well, there are all needed foreign keys, etc.
How to find the source of the slow executions and eliminate it?
EDIT: EXPLAIN results:
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE tp ref unique_row,idTeam idTeam 4 const 56 Using temporary; Using filesort
1 SIMPLE p eq_ref PRIMARY,projStart,confirmStart PRIMARY 4 xxx.tp.idProject 1 Using where
1 SIMPLE pt ref uniq_projtimes uniq_projtimes 4 xxx.tp.idProject 1 Using where; Distinct
1 SIMPLE ctt ref idProject idProject 4 xxx.tp.idProject 3966 Using index; Distinct
1 SIMPLE nc eq_ref PRIMARY,idProject PRIMARY 8 xxx.ctt.idCustomer,xxx.tp.idProject 1 Using where; Distinct
EDIT2: Results of EXPLAIN EXTENDED first for fast query, second for the slow one.
id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE tp ref unique_row,idTeam idTeam 4 const 1 100 Using temporary
1 SIMPLE p eq_ref PRIMARY,projStart,confirmStart PRIMARY 4 xxx.tp.idProject 1 100 Using where
1 SIMPLE pt ref uniq_projtimes uniq_projtimes 4 xxx.tp.idProject 1 100 Using where; Distinct
1 SIMPLE ctt ref idProject idProject 4 xxx.tp.idProject 46199 100 Using index; Distinct
1 SIMPLE nc eq_ref PRIMARY,idProject PRIMARY 8 xxx.ctt.idCustomer,xxx.tp.idProject 1 100 Using index; Distinct
id select_type table type possible_keys key key_len ref rows filtered Extra
1 SIMPLE p eq_ref PRIMARY,projStart,confirmStart PRIMARY 4 xxx.ctt.idProject 1 100 Using where
1 SIMPLE pt ref uniq_projtimes uniq_projtimes 4 xxx.ctt.idProject 1 100 Using where; Distinct
1 SIMPLE tp ref unique_row,idTeam unique_row 8 xxx.pt.idProject,const 1 100 Using where; Using index; Distinct
1 SIMPLE nc eq_ref PRIMARY,idProject PRIMARY 8 xxx.ctt.idCustomer,xxx.tp.idProject 1 100 Using index; Distinct
Try this adjusted query. (It will join less rows)
SELECT DISTINCT p.idProject AS idProject, p.name AS name, 0 AS isConfirm
FROM Projects p
JOIN projtimes pt ON
p.idProject = pt.idProject
AND p.activated = 1
AND current_date >= p.projStart
AND current_date < p.confirmStart
AND pt.invitesCount < pt.maxPerPresentation
JOIN team_project tp ON
p.idProject = tp.idProject
AND tp.idTeam = 158
JOIN CalledTimesTbl ctt ON (p.idProject = ctt.idProject)
LEFT JOIN NextCalls nc ON (ctt.idCustomer = nc.idCustomer
AND ctt.idProject = nc.idProject)
WHERE (nc.idCustomer IS NULL OR nc.nextCall < now())
ORDER BY p.name