Optimization of a large database query - mysql

I would like to ask if there is any way to optimize the following query:
SELECT SQL_BIG_RESULT DISTINCT u.spy_tech,
u.computer_tech,
u.military_tech,
u.defence_tech,
u.shield_tech,
u.energy_tech,
u.hyperspace_tech,
u.combustion_tech,
u.impulse_motor_tech,
u.hyperspace_motor_tech,
u.laser_tech,
u.ionic_tech,
u.buster_tech,
u.intergalactic_tech,
u.expedition_tech,
u.metal_proc_tech,
u.crystal_proc_tech,
u.deuterium_proc_tech,
u.technologia_bankowa,
u.technologia_kolonizacyjna,
u.technologia_optymalizacji,
u.technologia_otchlani,
u.graviton_tech,
SUM(p.small_ship_cargo) as small_ship_cargo,
SUM(p.big_ship_cargo) as big_ship_cargo,
SUM(p.light_hunter) as light_hunter,
SUM(p.heavy_hunter) as heavy_hunter,
SUM(p.crusher) as crusher,
SUM(p.battle_ship) as battle_ship,
SUM(p.colonizer) as colonizer,
SUM(p.recycler) as recycler,
SUM(p.spy_sonde) as spy_sonde,
SUM(p.bomber_ship) as bomber_ship,
SUM(p.solar_satelit) as solar_satelit,
SUM(p.destructor) as destructor,
SUM(p.dearth_star) as dearth_star,
SUM(p.battleship) as battleship,
SUM(p.lune_noir) as lune_noir,
SUM(p.ev_transporter) as ev_transporter,
SUM(p.star_crasher) as star_crasher,
SUM(p.giga_recykler) as giga_recykler,
SUM(p.dm_ship) as dm_ship,
SUM(p.apocalypse) as apocalypse,
SUM(p.annihilator) as annihilator,
SUM(p.devastator) as devastator,
SUM(p.fleet_save) as fleet_save,
u.r_badawczy,
u.r_budowlany,
u.r_militarny,
u.r_obrony,
u.r_paliwowy,
u.r_gospodarczy,
u.r_planetarny,
u.r_magazynowy,
u.r_energetyczny,
u.r_ekonomiczny,
u.r_odkrywczy,
u.r_max,
u.r_speed,
u.r_moon,
u.r_dm,
u.r_terra,
SUM(p.misil_launcher) as misil_launcher,
SUM(p.small_laser) as small_laser,
SUM(p.big_laser) as big_laser,
SUM(p.gauss_canyon) as gauss_canyon,
SUM(p.ionic_canyon) as ionic_canyon,
SUM(p.buster_canyon) as buster_canyon,
SUM(p.small_protection_shield) as small_protection_shield,
SUM(p.big_protection_shield) as big_protection_shield,
SUM(p.planet_protector) as planet_protector,
SUM(p.graviton_canyon) as graviton_canyon,
SUM(p.orbital_station) as orbital_station,
SUM(p.dzialo_pulsacyjne) as dzialo_pulsacyjne,
SUM(p.dzialo_fotonowe) as dzialo_fotonowe,
SUM(p.interceptor_misil) as interceptor_misil,
SUM(p.interplanetary_misil) as interplanetary_misil,
u.id,
u.ally_id,
u.onlinetime,
s.tech_rank AS old_tech_rank,
s.build_rank AS old_build_rank,
s.defs_rank AS old_defs_rank,
s.fleet_rank AS old_fleet_rank,
s.total_rank AS old_total_rank,
s.total_points_old AS total_points_old,
s.fleet_points_old AS fleet_points_old,
s.defs_points_old AS defs_points_old,
s.build_points_old AS build_points_old,
s.tech_points_old AS tech_points_old
FROM test_users as u
LEFT JOIN test_statpoints as s ON s.stat_type = 1 AND s.id_owner = u.id
LEFT JOIN test_planets as p ON u.id = p.id_owner
WHERE u.user_lastip != "127.0.0.1"
AND u.bana = "0"
AND u.authlevel = "0"
AND u.universe = "1"
GROUP BY s.id_owner, u.id
On my VPS, the query takes 8 seconds and the result is about 10,000 results. Although I improved the machine twice, to 8x2,4CPU, 16gb RAM, the query time has not changed.
So I thought to optimize the above-mentioned query or work on VPS configuration and find a bottleneck that can generate such time.
Table STATPOINTS:
Table USERS:
I do not know if I have given you an insight into the tables, when it comes to something else I would ask for guidance.
Explain:

Your query can run faster if you add the following indexes:
create index ix1 on user (bana, authlevel, universe);
create index ix2 on test_statpoints (stat_type, id_owner);
create index ix3 on test_planets (id_owner);
They include the access columns according to the join predicates.

Related

mysql is very slow on LEFT JOIN

I'm converting my entire site from MYSQLi to PDO.
Now I have one problem with one of my larger query's
SELECT
kl.klant_id AS klant_id,
kl.klant_naam AS klant_naam,
kl.klant_adres AS klant_adres,
kl.klant_postcode AS klant_postcode,
kl.klant_plaats AS klant_plaats,
kl.klant_land AS klant_land,
kl.klant_telefoon AS klant_telefoon,
kl.klant_fax AS klant_fax,
kl.klant_email AS klant_email,
kl.klant_contactpersoon AS klant_contactpersoon,
gr.groep_id AS groep_id,
gr.groep_naam AS groep_naam,
IF(ft.factuur_isgroep>0,gr.groep_factuur_adress,ft.factuur_adres) AS factuur_adres,
IF(ft.factuur_isgroep>0,gr.groep_factuur_postbus,ft.factuur_postbus) AS factuur_postbus,
IF(ft.factuur_isgroep>0,gr.groep_factuur_postcode,ft.factuur_postcode) AS factuur_postcode,
IF(ft.factuur_isgroep>0,gr.groep_factuur_plaats,ft.factuur_plaats) AS factuur_plaats,
IF(ft.factuur_isgroep>0,gr.groep_factuur_land,ft.factuur_land) AS factuur_land,
IF(ft.factuur_isgroep>0,gr.groep_factuur_iban,ft.factuur_iban) AS factuur_iban,
IF(ft.factuur_isgroep>0,gr.groep_factuur_bic,ft.factuur_bic) AS factuur_bic,
IF(ft.factuur_isgroep>0,gr.groep_factuur_telefoon,ft.factuur_telefoon) AS factuur_telefoon,
IF(ft.factuur_isgroep>0,gr.groep_factuur_contactpersoon,ft.factuur_contactpersoon) AS factuur_contactpersoon,
IF(ft.factuur_isgroep>0,gr.groep_factuur_bymail,ft.factuur_bymail) AS factuur_bymail,
IF(ft.factuur_isgroep>0,gr.groep_factuur_email,ft.factuur_email) AS factuur_email,
IF(ft.factuur_isgroep>0,gr.groep_extra,ft.factuur_opmerkingen) AS factuur_opmerkingen,
al.alarm_ccs_type AS alarm_ccs,
al.alarm_gprs AS alarm_gprs,
al.alarm_gprs_provider AS alarm_gprs_provider,
al.alarm_gprs_sn AS alarm_gprs_sn,
al.alarm_doormelding AS alarm_doormelding,
al.alarm_onderhoud AS alarm_onderhoud,
al.alarm_onderhoud_laatst AS alarm_onderhoud_laatst,
al.alarm_certificaat AS alarm_certificaat,
al.alarm_cer_state AS alarm_cer_state,
al.alarm_cer_uitgifte AS alarm_cer_uitgifte,
al.alarm_risicoklasse AS alarm_risicoklasse,
mk.pac_naam AS meldkamer_naam,
mk.pac_clientnr AS meldkamer_clientnr,
mk.pac_promnr AS meldkamer_promnr,
mk.pac_backup_clientnr AS meldkamer_backup_clientnr,
mk.pac_backup_promnr AS meldkamer_backup_promnr
FROM klanten AS kl
LEFT JOIN groepen AS gr ON kl.klant_groupid > 0 AND kl.klant_groupid = gr.groep_id
LEFT JOIN alarm AS al ON al.alarm_klantid = kl.klant_id
LEFT JOIN meldkamer AS mk ON mk.pac_klantid = kl.klant_id
LEFT JOIN factuur AS ft ON ft.factuur_klantid = kl.klant_id
ORDER BY klant_id ASC
The query works but on PDO it's very slow. (+/- 3sec while mysqli is less then 1sec)
Is this a known problem of PDO or is there something wrong with my query?
I've read a lot of other poeple with simular problems but they all have more complex query's than this.
Solved Thanks to cale_b. My problem was indexing my columns. I indexed all the columns on i used in the above query and now it's very fast. Thanks.

Why MySQL query from mysql-slow-log is not appearing to be slow when run in PHPMyAdmin?

I have a query responsible for output of clients list that is regularly run by users.
The query is:
SELECT SQL_CALC_FOUND_ROWS p.lname, p.fname, p.patronymic, p.job,
p.post, p.zip, p.city, p.address, p.id, p.additional,
p.people_type_id, p.gender, p.permissions, p.user_id,
p.user_id, p.grading, p.progress, p.full_name, p.b1a,
p.b1b, p.b1c, p.b2a, p.b2b, p.b2c, p.b3a, p.b3b, p.b3c,
p.b4a, p.b4b, p.b4c, p.b5a, p.b5b, p.b5c, p.b6a, p.b6b,
p.b6c, p.b7a, p.b7b, p.b7c, p.b8a, p.b8b, p.b8c, p.b9a,
p.b9b, p.b9c, p.b10a, p.b10b, p.b10c, p.b11a, p.b11b,
p.b11c, p.b12a, p.b12b, p.b12c, p.b13a, p.b13b, p.b13c,
p.b14a, p.b14b, p.b14c, p.b15a, p.b15b, p.b15c, p.b16a,
p.b16b, p.b16c, p.b17a, p.b17b, p.count, p.finish_count,
p.partyId, t.description, a.description, p.contact_through,
DATE_FORMAT(p.next_call, '%d-%m-%Y') as next_call, p.recruiter,
p.b17c, p.l1a, p.l1b, p.l1c, p.l2a, p.l2b, p.l2c, p.l3a,
p.l3b, p.l3c, p.l4a, p.l4b, p.l4c, p.l5a, p.l5b, p.l5c,
p.l6a, p.l6b, p.l6c, p.l7a, p.l7b, p.l7c, p.l8a, p.l8b,
p.l8c, p.l9a, p.l9b, p.l9c, p.l10a, p.l10b, p.l10c, p.l11a,
p.l11b, p.l11c, p.l12a, p.l12b, p.l12c, p.l13a, p.l13b,
p.l13c, p.l14a, p.l14b, p.l14c, p.c1a, p.c1b, p.c1c, p.c2a,
p.c2b, p.c2c, p.c3a, p.c3b, p.c3c, p.c4a, p.c4b, p.c4c,
p.c5a, p.c5b, p.c5c, p.c6a, p.c6b, p.c6c, p.c7a, p.c7b,
p.c7c, p.c8a, p.c8b, p.c8c, p.c9a, p.c9b, p.c9c, p.c10a,
p.c10b, p.c10c, p.c11a, p.c11b, p.c11c, p.c12a, p.c12b,
p.c12c, p.c13a, p.c13b, p.c13c, p.c14a, p.c14b, p.c14c,
p.c15a, p.c15b, p.c15c, p.c16a, p.c16b, p.c16c, p.c17a,
p.c17b, p.c17c, p.c18a, p.c18b, p.c18c, p.c19a, p.c19b,
p.c19c, p.c20a, p.c20b, p.c20c, p.c21a, p.c21b, p.c21c
FROM people p
JOIN people_progress_id a ON p.progress = a.proc_level_id
JOIN people_grading_id t ON p.grading = t.grading_level_id
WHERE p.category=3
and p.status = 1
ORDER BY p.city desc
LIMIT 0, 50;
It regularly gets logged to mysql-slow-query log similarly to this:
# Time: 160718 17:18:32
# User#Host: server[server] # localhost []
# Query_time: 4.098162 Lock_time: 0.000255 Rows_sent: 50 Rows_examined: 1508127
SET timestamp=1468851512;
SELECT SQL_CALC_FOUND_ROWS p.lname...
But when I run this in PHPMyAdmin the execution time is less than a second. Is there any reason why this happens? And how can I speed this up?
Add INDEX(category, status, city)
Rethink dumping that much data out -- do the users really need that much stuff?
Rethink how to get "calc found rows" -- either remove it from the UI, or look into ways of caching and/or approximating the value.
Rethink whether you really need t.description, a.description -- the JOINs are hurting performance (some).
(And, as already mentioned, the Query cache may be confusing the timings. Simply changing LIMIT 0,50 to LIMIT 0,51 will prevent the QC from kicking in. Of course, SQL_NO_CACHE is better.)
You need to change the my.ini file to capture the slow queries. there are following parameters that you need to set up
slow_query_log it should be ON
log_slow_queries = 5; (That means MySql logs queries that takes more than 5 seconds)
Thanks,
Rakesh

How can optimize the query with join

I have a query large query which is taking more time to execute.How can i reduce the execution time of the query using Join.Here is myt query.
SELECT mdi.Member_Id,
dp.Payment1_Date AS Duchess_Payment1_Date,
dp.Payment1_Method AS Duchess_Payment1_Method,
dp.Payment1_By AS Duchess_Payment1_By,
dp.Payment1_Amount AS Duchess_Payment1_Amount,
dp.Payment2_Date AS Duchess_Payment2_Date,
dp.Payment2_Method AS Duchess_Payment2_Method,
dp.Payment2_By AS Duchess_Payment2_By,
dp.Payment2_Amount AS Duchess_Payment2_Amount,
mha.First_Name AS Member_First_Name,
mha.Middle_Name AS Member_Middle_Name,
mha.Last_Name AS Member_Last_Name,
mha.Address AS Member_Address,
mha.City AS Member_City,
mha.State AS Member_State,
mha.Zip AS Member_Zip,
mha.Nickname AS Member_Nickname,
mha.Phone AS Member_Phone,
mha.Mobile AS Member_Mobile,
mha.Email AS Member_Email,
mwa.First_Name AS Work_First_Name,
mwa.Middle_Name AS Work_Middle_Name,
mwa.Last_Name AS Work_Last_Name,
mwa.Address AS Work_Adrress,
mwa.City AS Work_City,
mwa.State AS Work_State,
mwa.Zip AS Work_Zip,
mwa.Phone AS Work_Phone,
(CASE WHEN mha.Preferred=0 THEN 'Work Address' WHEN mha.Preferred=1 THEN 'Home Address' END)AS Preferred,
mmt.Admiral_Title,
mmt.Spouse_Title,
mmt.Couple_Title,
mdi.Year AS Duchess_Year,
mdi.College AS Duchess_College,
mdi.Major AS Duchess_Major,
mdi.Sorority AS Duchess_Sorority,
mdi.Parent_Name AS Duchess_Parent_Name,
mdi.Escort_Name AS Duchess_Escort_Name
FROM duchess_payment dp
JOIN member_duchess_info mdi ON mdi.Duchess_Id = dp.Duchess_Id
JOIN member_home_address mha ON mha.Member_Id = mdi.Member_Id
JOIN member_work_address mwa ON mwa.Member_Id = mdi.Member_Id
JOIN member_mailing_title mmt ON mmt.Member_Id = mdi.Member_Id
ORDER BY dp.Duchess_Id;
this is the way Iam using . Is it corrct?.
when i try to executing that query it is still taking more time.
here is the explain of the query
Use indexes on the items on which you are performing joins e.g Duchess_Id, Member_Id etc

sql query speed. Time response to long

I have problem with my mysl query. Response time is to long. My query:
SELECT SQL_CALC_FOUND_ROWS
t.id,
l.id_produkt_lista,
z.nr_zam,
k.nazwa_fv,
p.nazwa,
p.opis,
p.data_realizacji,
CONCAT(t.d_graf,'</br>', IF(t.D_d_graf IS NOT NULL,
DATE_FORMAT(t.D_d_graf, "%d-%m-%Y"),"")),
CONCAT(t.d_druk,'</br>', IF(t.D_d_druk IS NOT NULL,
DATE_FORMAT(t.D_d_druk, "%d-%m-%Y"),"")),
CONCAT(t.d_zgrz,'</br>', IF(t.D_d_zgrz IS NOT NULL,
DATE_FORMAT(t.D_d_zgrz, "%d-%m-%Y"),""))
FROM zamowienie_produkt_lista l
JOIN zamowienia_zamowienie z ON (l.id_zamowienie = z.id_zamowienie)
JOIN zamowienia_produkt p ON (l.id_produkt = p.id_produkt)
JOIN zamowienia_prac_tmp t ON (l.id_produkt = t.id_produkt)
JOIN zamowienia_klient k ON (z.id_klient = k.id_klient)
WHERE TRUE
AND id_typ_produkt = '1'
AND z.archiwum = 0
ORDER BY t.id
When I use it in phpmyadmin I have to wait about 10 second
Assuming that id_typ_produkt belongs to table zamowienie_produkt_lista, creating following indexes should help:
CREATE INDEX p_1_idx ON zamowienie_produkt_lista
(id_produkt, id_typ_produkt);
CREATE INDEX z_1_idx ON zamowienia_zamowienie
(id_zamowienie, archiwum);
You should also make sure that indexes on all *_id fields for all other joined tables exist.

Track SQL queries that timeout

Is it possible to query SQL Server 2008 R2 for a list of queries that have timedout?
I'm trying to find out which parts of our app are falling over, but for boring and bad reasons that won't get fixed, we have no application level logging.
In lieu of getting some application logging put in, how can I find out which queries are timing out?
to get all the time-outs you will need to use the profiler, however you can run some queries that may help:
show top 10 high CPU queries:
SELECT TOP 10
total_worker_time/execution_count AS Avg_CPU_Time
,execution_count
,total_elapsed_time/execution_count as AVG_Run_Time
,(SELECT
SUBSTRING(text,statement_start_offset/2,(CASE
WHEN statement_end_offset = -1 THEN LEN(CONVERT(nvarchar(max), text)) * 2
ELSE statement_end_offset
END -statement_start_offset)/2
) FROM sys.dm_exec_sql_text(sql_handle)
) AS query_text
FROM sys.dm_exec_query_stats
ORDER BY Avg_CPU_Time DESC
this query will show cahced query plans that "SCAN", change comments for other things, can add filters for UseCount, EstimatedCost, EstimatedCPU, Estimated Rows
;WITH XMLNAMESPACES(DEFAULT N'http://schemas.microsoft.com/sqlserver/2004/07/showplan')
, CachedPlans AS
(SELECT
RelOp.op.value(N'../../#NodeId', N'int') AS ParentOperationID
,RelOp.op.value(N'#NodeId', N'int') AS OperationID
,RelOp.op.value(N'#PhysicalOp', N'varchar(50)') AS PhysicalOperator
,RelOp.op.value(N'#LogicalOp', N'varchar(50)') AS LogicalOperator
,RelOp.op.value(N'#EstimatedTotalSubtreeCost ', N'float') AS EstimatedCost
,RelOp.op.value(N'#EstimateIO', N'float') AS EstimatedIO
,RelOp.op.value(N'#EstimateCPU', N'float') AS EstimatedCPU
,RelOp.op.value(N'#EstimateRows', N'float') AS EstimatedRows
,cp.plan_handle AS PlanHandle
,qp.query_plan AS QueryPlan
,st.TEXT AS QueryText
,cp.cacheobjtype AS CacheObjectType
,cp.objtype AS ObjectType
,cp.usecounts AS UseCounts
FROM sys.dm_exec_cached_plans cp
CROSS APPLY sys.dm_exec_sql_text(cp.plan_handle) st
CROSS APPLY sys.dm_exec_query_plan(cp.plan_handle) qp
CROSS APPLY qp.query_plan.nodes(N'//RelOp') RelOp (op)
)
SELECT
PlanHandle
,ParentOperationID
,OperationID
,PhysicalOperator
,LogicalOperator
,UseCounts
,CacheObjectType
,ObjectType
,EstimatedCost
,EstimatedIO
,EstimatedCPU
,EstimatedRows
,QueryText
FROM CachedPlans
WHERE CacheObjectType = N'Compiled Plan'
AND PhysicalOperator IN ('nothing will ever match this one!'
--,'Assert'
--,'Bitmap'
--,'Clustered Index Delete'
--,'Clustered Index Insert'
,'Clustered Index Scan'
--,'Clustered Index Seek'
--,'Clustered Index Update'
--,'Compute Scalar'
--,'Concatenation'
--,'Constant Scan'
,'Deleted Scan'
--,'Filter'
--,'Hash Match'
,'Index Scan'
--,'Index Seek'
--,'Index Spool'
,'Inserted Scan'
--,'Merge Join'
--,'Nested Loops'
--,'Parallelism'
,'Parameter Table Scan'
--,'RID Lookup'
--,'Segment'
--,'Sequence Project'
--,'Sort'
--,'Stream Aggregate'
--,'Table Delete'
--,'Table Insert'
,'Table Scan'
--,'Table Spool'
--,'Table Update'
--,'Table-valued function'
--,'Top'
)