length of mysql result - mysql

I made this query and i cant figure out why number of characters in chal_rews.rewards field on line 37 returns exactly 341 characters where there are more than 400 characters.
SELECT
GROUP_CONCAT( users.email ) AS group_emails,
GROUP_CONCAT(
CONCAT(
IFNULL(users.first_name,users.email),' ',IFNULL(users.last_name,''),'|',
IFNULL(users.facebook_id,''),'|',
IFNULL(users.email,''),'|',
users.id,'|',
invitations.wined,'|',
invitations.accepted,'|',
invitations.id
)
) AS participants,
owner.id AS owner_id,
owner.email,
owner.facebook_id,
owner.access_token,
owner.first_name,
CONCAT( IFNULL(owner.first_name,owner.email),' ',IFNULL(owner.last_name,'') ) AS owner_fullname,
challenges.id AS challenge_id,
challenges.challenge_type_id,
challenges.period_id,
challenges.period_amount,
challenges.category_item_id,
challenges.metric_amount,
challenges.owner_user_id,
CASE
WHEN challenges.status=2 THEN 'complete'
WHEN ( challenges.status=1 AND challenges.end_date<NOW() ) THEN 'waiting_for_winner'
ELSE challenges.status
END AS status,
DATE_FORMAT(challenges.start_date,'%d %M %Y # %H:%i') AS start_date,
DATE_FORMAT(challenges.end_date,'%d %M %Y # %H:%i') AS end_date,
DATE_FORMAT(challenges.created_at,'%d %M %Y # %H:%i') AS created_at,
DATE_FORMAT(challenges.updated_at,'%d %M %Y # %H:%i') AS updated_at,
challenges.inform,
chal_rews.rewards,
chal_rews.reward_names AS reward_name,
category_items.item_name,
LOWER( CONCAT(fixed_verb,' ',IF(metric_amount=-1,'',CONCAT(metric_amount,' ')),fixed_metric,' ',fixed_text,' ',challenges.period_amount ,' ',periods.name) ) AS challenge_string
FROM
invitations
Left Join users ON invitations.user_id = users.id
Left Join challenges ON invitations.challenge_id = challenges.id
Left Join users AS owner ON owner.id = challenges.owner_user_id
Left Join challenge_types ON challenges.challenge_type_id = challenge_types.id
Left Join periods ON challenges.period_id = periods.id
Left Join category_items ON challenges.category_item_id = category_items.id
Left Join (
SELECT
chal_rews.id,
chal_rews.challenge_id,
chal_rews.reward_item_id,
chal_rews.user_id,
chal_rews.created_at,
chal_rews.updated_at,
GROUP_CONCAT(CONCAT(
IFNULL(users.first_name,users.email),' ',IFNULL(users.last_name,''),'|',
IFNULL(users.facebook_id,''),'|',
IFNULL(users.access_token,''),'|',
IFNULL(users.email,''),'|',
users.id,'|',
IFNULL(chal_rews.id,' '),'|',
IFNULL(chal_rews.reward_item_id,' '),'|',
IFNULL(reward_items.name,' ')
)) AS rewards,
GROUP_CONCAT(reward_items.name) AS reward_names
FROM
chal_rews
Left Join users ON users.id = chal_rews.user_id
Left Join reward_items ON reward_items.id = chal_rews.reward_item_id
GROUP BY
chal_rews.challenge_id
) AS chal_rews ON challenges.id = chal_rews.challenge_id
WHERE 1=1
while if i run only this part of query then rewards field returns exactly what it should i.e. full concatenated string.
SELECT
chal_rews.id,
chal_rews.challenge_id,
chal_rews.reward_item_id,
chal_rews.user_id,
chal_rews.created_at,
chal_rews.updated_at,
GROUP_CONCAT(CONCAT(
IFNULL(users.first_name,users.email),' ',IFNULL(users.last_name,''),'|',
IFNULL(users.facebook_id,''),'|',
IFNULL(users.access_token,''),'|',
IFNULL(users.email,''),'|',
users.id,'|',
IFNULL(chal_rews.id,' '),'|',
IFNULL(chal_rews.reward_item_id,' '),'|',
IFNULL(reward_items.name,' ')
)) AS rewards,
GROUP_CONCAT(reward_items.name) AS reward_names
FROM
chal_rews
Left Join users ON users.id = chal_rews.user_id
Left Join reward_items ON reward_items.id = chal_rews.reward_item_id
GROUP BY
chal_rews.challenge_id
I cant figure out what is wrong. SOS plz...

Bit of a punt this but it might help. If it doesn't then, well, sorry! You might get some joy by tightening up the group by clause at the end of the chal_rews sub-select. Hence:
SELECT
chal_rews.id,
chal_rews.challenge_id,
chal_rews.reward_item_id,
chal_rews.user_id,
chal_rews.created_at,
chal_rews.updated_at,
GROUP_CONCAT(CONCAT(
IFNULL(users.first_name,users.email),' ',IFNULL(users.last_name,''),'|',
IFNULL(users.facebook_id,''),'|',
IFNULL(users.access_token,''),'|',
IFNULL(users.email,''),'|',
users.id,'|',
IFNULL(chal_rews.id,' '),'|',
IFNULL(chal_rews.reward_item_id,' '),'|',
IFNULL(reward_items.name,' ')
)) AS rewards,
GROUP_CONCAT(reward_items.name) AS reward_names
FROM
chal_rews
Left Join users ON users.id = chal_rews.user_id
Left Join reward_items ON reward_items.id = chal_rews.reward_item_id
GROUP BY
chal_rews.id,
chal_rews.challenge_id,
chal_rews.reward_item_id,
chal_rews.user_id,
chal_rews.created_at,
chal_rews.updated_at
It's a little tricky to be sure without the actual tables and data in front of me but I think you will get a more consistent result in the rewards field if you do this. I think the columns that are coming back in the sub-select are inconsistent when you run it as part of the main query compared to when you run it in isolation. I suspect this is being caused by letting MySQL 'choose' a value for the chal_rews.user_id and chal_rews.reward_item_id (which in turn are used to outer join onto the users and reward_items tables - hence bringing back inconsistent data). More formally I reckon you are getting an indeterminate results set when excluding non-aggregated columns from the GROUP BY that are present in the SELECT.
Good luck and post how you get on!

Related

Slow query performance in MySQL

I would like to find out what causes the slow execution of my MySQL query here. It's 1 row fetched in 0.0017s (3.6215s) How can i optimize this?
SELECT hadmlog.hpercode as 'HOSPITAL NUMBER', FLOOR (hadmlog.patage) as 'AGE', CONCAT (hperson.patlast, ', ', hperson.patfirst, ' ',hperson.patmiddle) as 'PROFILE', hcity.ctyname as 'DISTRICT', hadmlog.disdate as 'DISCHARGED DATE'
FROM hadmlog
INNER JOIN hperson ON hadmlog.hpercode=hperson.hpercode
INNER JOIN haddr ON hadmlog.hpercode=haddr.hpercode
INNER JOIN hcity ON haddr.ctycode=hcity.ctycode
WHERE hadmlog.patage BETWEEN '1' AND '4'
AND hperson.patsex = 'M'
AND DATE(hadmlog.disdate) = DATE(curdate())
AND haddr.haddrdte = ( select max(haddrdte)
from haddr
where haddr.hpercode = hperson.hpercode )
ORDER BY Profile;
First, fix the query:
SELECT al.hpercode as HOSPITAL_NUMBER, FLOOR(al.patage) as AGE,
CONCAT_WS(' ', p.patlast, p.patfirst, p.patmiddle) as PROFILE,
c.ctyname as DISTRICT, al.disdate as DISCHARGED_DATE
FROM hadmlog al JOIN
hperson p
ON al.hpercode = p.hpercode JOIN
haddr a
ON al.hpercode = a.hpercode JOIN
hcity c
ON a.ctycode = c.ctycode
WHERE al.patage BETWEEN 1 AND 4 AND
p.patsex = 'M' AND
al.disdate >= curdate() AND
al.disdate < curdate() + INTERVAL 1 day AND
a.haddrdte = (select max(h2.haddrdte)
from haddr h2
where h2.hpercode = p.hpercode
)
ORDER BY Profile;
Some of the changes are cosmetic (such as the table aliases and CONCAT_WS()). The more relevant changes are:
patage appears to be a number, so make the comparisons numbers. Strings can impede the optimizer.
date(disdate) can also impede the optimizer.
Then, for this query, I am guessing that you want an indexes:
hadmlog(disdate, patage)
And then you want indexes on the JOIN keys used for the other tables.

Mysql-cast and convert in joins is not working

In a MySQL table, they have stored return_time as VARCHAR(20) and return_date as a date field. Now I have to compare this based on the given date and time and produce the result. For example, records save in the time column will be like '10:00 PM', '11:00 PM', '02:00 AM' etc and in the date column like '2021-02-02', '2021-01-02' etc
I combined the return_dateand return_time, and compare it with the given date and time like
SELECT *
FROM `travel_infos` as TravelInfo
WHERE TravelInfo.travel_status = 'Open' AND
cast(concat(TravelInfo.return_date, ' ', STR_TO_DATE(TravelInfo.return_time, '%l:%i %p' )) as datetime) > '2021-03-03 14:03'
It outputs me 231 records. travel_infos table has 700 records.
Then I do the join without considering the return_time column
SELECT Part.*,TravelInfo.*,Category.*,Buttons.*
FROM travel_infos As TravelInfo
LEFT JOIN parts as Part ON Part.id = TravelInfo.part_id_1
LEFT JOIN buttons as Buttons ON Part.button_name_id= Buttons.id
LEFT JOIN categories as Category ON Part.category_id = Category.id
WHERE Part.id != 0 AND TravelInfo.id!=0
AND Part.part_type IN('Charter') AND Part.store_id='1'
AND TravelInfo.travel_status = 'Open'
AND TravelInfo.return_date >= '2021-03-03 14:03'
AND Part.is_delete=0
It outputs me 175 records.
So now I want to join by considering both return_date and return_time. So I tried like
SELECT Part.*,TravelInfo.*,Category.*,Buttons.*
FROM travel_infos As TravelInfo
LEFT JOIN parts as Part ON Part.id = TravelInfo.part_id_1
LEFT JOIN buttons as Buttons ON Part.button_name_id= Buttons.id
LEFT JOIN categories as Category ON Part.category_id = Category.id
WHERE Part.id != 0 AND
TravelInfo.id!=0
AND Part.part_type IN('Charter')
AND Part.store_id='1'
AND TravelInfo.travel_status = 'Open'
AND (cast(concat(TravelInfo.return_date, ' ',
STR_TO_DATE(TravelInfo.return_time, '%l:%i %p' )) as datetime)) >= '2021-03-03 14:03' AND Part.is_delete=0
This outputs me 0 records. What went wrong in the 3rd SQL query? Both 1st and 2nd produce the output.
In my 3rd query, I modified the 2nd query WHERE like in the 1st query.
Can someone help me to solve this thank you.
Convert both dates you are comparing to datetimes in the where clause:
SELECT Part.*,TravelInfo.*,Category.*,Buttons.*
FROM travel_infos As TravelInfo
LEFT JOIN parts as Part ON Part.id = TravelInfo.part_id_1
LEFT JOIN buttons as Buttons ON Part.button_name_id= Buttons.id
LEFT JOIN categories as Category ON Part.category_id = Category.id
WHERE Part.id != 0 AND
TravelInfo.id!=0
AND Part.part_type IN('Charter')
AND Part.store_id='1'
AND TravelInfo.travel_status = 'Open'
AND TravelInfo.return_date >= STR_TO_DATE('2021-03-03', '%Y-%m-%d')
AND cast(TravelInfo.return_time as time) >= cast('14:03:00' as time)
AND Part.is_delete=0

MySQL Find Most Recent/Largest Record Per Group by order by

MySQL Find Most Recent/Largest Record Per Group by order by, and how can i minimize / shorten this query and every time it returns the first row value of the group, whereas i like to select the last row value of a group, and sort the values based on ja.id ? I know this is a bad query can anyone suggest or provide me solution to shorten this query . I have used all the necessary Column Indexes in all the tables. How to shorten the query without the use of union all .both the queries in union all are same expect in the where statement.
SELECT
a.previous_status,
a.rejected_status,
a.rejection_reason_text,
a.rejection_reason,
a.rjaId,
a.refer_applied_status,
a.title,
a.playerId,
a.gameId,
a.gamePostDate,
a.game_referal_amount,
a.country,
a.country_name,
a.state,
a.location,
a.state_abb,
a.game_type,
a.appliedId,
a.appliedStatus,
a.admin_review,
a.is_req_referal_check,
a.memberId,
a.appliedEmail,
a.game_id,
a.referred_id,
a.memStateAbb,
a.memState,
a.memZipcode,
a.memCity,
a.memCountryNme,
a.memCountry,
a.appliedMemberName,
a.first_name,
a.primary_contact,
a.last_name,
a.addressbookImage,
a.userImage,
a.last_login,
a.user_experience_year,
a.user_experience_month,
a.current_designation,
a.current_player,
a.appliedDate,
a.addressbook_id,
a.joiningdate,
a.gameStatus,
a.gameReferalAmountType,
a.gameFreezeStatus,
a.gameFreezeMsg,
a.app_assign_back_to_rp_count,
a.applied_source,
a.max_id,
a.gamesApplied,
a.gamesAppliedId,
SUM(a.totalgameApplied) AS totalgameApplied,
a.application_assign_to_rp_status,
a.rpAppliedSource,
a.applied_on
FROM
(
(
SELECT
ja.previous_status,
ja.rejected_status,
ja.rejection_reason_text,
ja.rejection_reason,
rja.id AS rjaId,
rja. STATUS AS refer_applied_status,
jp.title,
jp.user_user_id AS playerId,
jp.id AS gameId,
jp.posted_on AS gamePostDate,
jp.game_referal_amount,
jp.country,
jp.country_name,
jp.state,
jp.location,
jp.state_abb,
jp.game_type,
ja.id AS appliedId,
IFNULL(ja. STATUS, '') AS appliedStatus,
IFNULL(ja.admin_review, '') AS admin_review,
ja.is_req_referal_check,
usr.id AS memberId,
rja.email AS appliedEmail,
rja.game_id,
rja.referred_id,
mem.state_abb AS memStateAbb,
mem.state AS memState,
mem.zipcode AS memZipcode,
mem.city AS memCity,
mem.country_name AS memCountryNme,
mem.country_code AS memCountry,
usc. NAME AS appliedMemberName,
usc.first_name,
IFNULL(
mem.primary_contact,
usc.phone_number
) AS primary_contact,
usc.last_name,
usc.profileimage_path AS addressbookImage,
usr.profile_image AS userImage,
usr.last_login,
mem.user_experience_year,
mem.user_experience_month,
mem.current_designation,
mem.current_player,
rja.create_date AS appliedDate,
rja.addressbook_id,
IFNULL(ja.joining_date, '') AS joiningdate,
jp. STATUS AS gameStatus,
jp.games_referal_amount_type AS gameReferalAmountType,
jp.game_freeze_status AS gameFreezeStatus,
jp.game_freeze_message AS gameFreezeMsg,
ja.app_assign_back_to_rp_count,
ja.applied_source,
MAX(rja.id) AS max_id,
GROUP_CONCAT(
jp.title
ORDER BY
rja.create_date DESC
) AS gamesApplied,
GROUP_CONCAT(DISTINCT(jp.id)) AS gamesAppliedId,
COUNT(DISTINCT(jp.id)) totalgameApplied,
ja.application_assign_to_rp_status,
1 AS rpAppliedSource,
rja.create_date AS applied_on
FROM
(`refer_gameapplied` AS rja)
JOIN `games_post` AS jp ON `jp`.`id` = `rja`.`game_id`
JOIN `user_socialconnections` AS usc ON `rja`.`addressbook_id` = `usc`.`id`
LEFT JOIN `user_user` AS usr ON `usr`.`email` = `rja`.`email`
LEFT JOIN `user_member` AS mem ON `mem`.`user_id` = `usr`.`id`
LEFT JOIN `game_applied` AS ja ON `ja`.`id` = `rja`.`applied_id`
WHERE
`rja`.`referby_id` = '2389'
GROUP BY
`rja`.`email`
)
UNION ALL
(
SELECT
ja.previous_status,
ja.rejected_status,
ja.rejection_reason_text,
ja.rejection_reason,
jr.id AS rjaId,
jrtm. STATUS AS refer_applied_status,
jp.title,
jp.user_user_id AS playerId,
jp.id AS gameId,
jp.posted_on AS gamePostDate,
jp.game_referal_amount,
jp.country,
jp.country_name,
jp.state,
jp.location,
jp.state_abb,
jp.game_type,
ja.id AS appliedId,
IFNULL(ja. STATUS, '') AS appliedStatus,
IFNULL(ja.admin_review, '') AS admin_review,
ja.is_req_referal_check,
usr.id AS memberId,
jrtm.referto_email AS refappliedEmail,
jr.game_id,
jrtm.id,
mem.state_abb AS memStateAbb,
mem.state AS memState,
mem.zipcode AS memZipcode,
mem.city AS memCity,
mem.country_name AS memCountryNme,
mem.country_code AS memCountry,
usc. NAME AS appliedMemberName,
usc.first_name,
IFNULL(
mem.primary_contact,
usc.phone_number
) AS primary_contact,
usc.last_name,
usc.profileimage_path AS addressbookImage,
usr.profile_image AS userImage,
usr.last_login,
mem.user_experience_year,
mem.user_experience_month,
mem.current_designation,
mem.current_player,
jrtm.refer_on AS appliedDate,
jrtm.referto_addressbookid,
IFNULL(ja.joining_date, '') AS joiningdate,
jp. STATUS AS gameStatus,
jp.games_referal_amount_type AS gameReferalAmountType,
jp.game_freeze_status AS gameFreezeStatus,
jp.game_freeze_message AS gameFreezeMsg,
ja.app_assign_back_to_rp_count,
ja.applied_source,
MAX(jrtm.id) AS max_id,
GROUP_CONCAT(
jp.title
ORDER BY
jr.refer_on DESC
) AS gamesApplied,
GROUP_CONCAT(DISTINCT(jp.id)) AS gamesAppliedId,
COUNT(DISTINCT(jp.id)) totalgameApplied,
ja.application_assign_to_rp_status,
2 AS rpAppliedSource,
jrtm.refer_on AS applied_on
FROM
(`game_refer` AS jr)
JOIN `game_refer_to_member` AS jrtm ON `jrtm`.`rid` = `jr`.`id`
JOIN `games_post` AS jp ON `jp`.`id` = `jr`.`game_id`
JOIN `user_socialconnections` AS usc ON `jrtm`.`referto_addressbookid` = `usc`.`id`
LEFT JOIN `user_user` AS usr ON `usr`.`email` = `jrtm`.`referto_email`
LEFT JOIN `user_member` AS mem ON `mem`.`user_id` = `usr`.`id`
LEFT JOIN `game_applied` AS ja ON `ja`.`referred_by` = `jrtm`.`id`
WHERE
`jrtm`.`status` = '1'
AND `jr`.`referby_user_id` = '2389'
AND `jrtm`.`refer_source` NOT IN ('4')
GROUP BY
`jrtm`.`referto_email`
)
) a
GROUP BY
a.appliedEmail
ORDER BY
a.gamesAppliedId DESC
It sounds like a "groupwise-max" problem. I added a tag that you should research.
At least get rid of the columns that are not relevant to the question.
Try tossing the tables other than jrtm and jr to see if the performance problem persists. (I'm thinking that the LEFT JOINs may be red herrings.)
Try with one part of the UNION, then with the other. This may identify which of the two is more of a burden.
Some indexes to add:
rja: (referby_id, create_date)
jrtm: (status, referto_email)
jrtm: (rid, status, referto_email)
jr: (referby_user_id, refer_on)
DISTINCT is not a function. Don't use parents in DISTINCT(jp.id).

Mysql tekes too much time to excute sql query, based on multiple join

My Sql query takes more time to execute from mysql database server . There are number of tables are joined with sb_tblproperty table. sb_tblproperty is main table that contain more than 1,00,000 rows . most of table contain 50,000 rows.
How to optimize my sql query to fast execution. I have also used indexing.
indexing Explain - query - structure
SELECT `t1`.`propertyId`, `t1`.`projectId`,
`t1`.`furnised`, `t1`.`ownerID`, `t1`.`subType`,
`t1`.`fors`, `t1`.`size`, `t1`.`unit`,
`t1`.`bedrooms`, `t1`.`address`, `t1`.`dateConfirm`,
`t1`.`dateAdded`, `t1`.`floor`, `t1`.`priceAmount`,
`t1`.`priceRate`, `t1`.`allInclusive`, `t1`.`booking`,
`t1`.`bookingRate`, `t1`.`paidPercetage`,
`t1`.`paidAmount`, `t1`.`is_sold`, `t1`.`remarks`,
`t1`.`status`, `t1`.`confirmedStatus`, `t1`.`source`,
`t1`.`companyName` as company, `t1`.`monthly_rent`,
`t1`.`per_sqft`, `t1`.`lease_duration`,
`t1`.`lease_commencement`, `t1`.`lock_in_period`,
`t1`.`security_deposit`, `t1`.`security_amount`,
`t1`.`total_area_leased`, `t1`.`lease_escalation_amount`,
`t1`.`lease_escalation_years`, `t2`.`propertyTypeName` as
propertyTypeName, `t3`.`propertySubTypeName` subType,
`t3`.`propertySubTypeId` subTypeId, `Owner`.`ContactName`
ownerName, `Owner`.`companyName`, `Owner`.`mobile1`,
`Owner`.`otherPhoneNo`, `Owner`.`mobile2`,
`Owner`.`email`, `Owner`.`address` as caddress,
`Owner`.`contactType`, `P`.`projectName` as project,
`P`.`developerName` as developer, `c`.`name` as city,
if(t1.projectId="", group_concat( distinct( L.locality)),
group_concat( distinct(L2.locality))) as locality, `U`.`firstname`
addedBy, `U1`.`firstname` confirmedBy
FROM `sb_tblproperty` as t1
JOIN `sb_contact` Owner ON `Owner`.`id` = `t1`.`ownerID`
JOIN `tbl_city` C ON `c`.`id` = `t1`.`city`
JOIN `sb_propertytype` t2 ON `t1`.`propertyType`= `t2`.`propertyTypeId`
JOIN `sb_propertysubtype` t3 ON `t1`.`subType` =`t3`.`propertySubTypeId`
LEFT JOIN `sb_tbluser` U ON `t1`.`addedBy` = `U`.`userId`
LEFT JOIN`sb_tbluser` U1 ON `t1`.`confirmedBy` = `U1`.`userId`
LEFT JOIN `sb_tblproject` P ON `P`.`id` = `t1`.`projectId` LEFT
JOIN `sb_tblpropertylocality` PL ON `t1`.`propertyId` = `PL`.`propertyId`
LEFT JOIN `sa_localitiez` L ON `L`.`id` = `PL`.`localityId`
LEFT JOIN `sb_tblprojectlocality` PROL ON `PROL`.`projectId` = `P`.`id`
LEFT JOIN `sa_localitiez` L2 ON `L2`.`id` = `PROL`.`localityId`
LEFT JOIN `sb_tblfloor` F
ON `F`.`floorName` =`t1`.`floor`
WHERE `t1`.`is_sold` != '1' GROUP BY `t1`.`propertyId`
ORDER BY `t1`.`dateConfirm`
DESC LIMIT 1000
Please provide the EXPLAIN.
Meanwhile, try this:
SELECT ...
FROM (
SELECT propertyId
FROM sb_tblproperty
WHERE `is_sold` = 0
ORDER BY `dateConfirm` DESC
LIMIT 1000
) AS x
JOIN `sb_tblproperty` as t1 ON t1.propertyId = x.propertyId
JOIN `sb_contact` Owner ON `Owner`.`id` = `t1`.`ownerID`
JOIN `tbl_city` C ON `c`.`id` = `t1`.`city`
...
LEFT JOIN `sb_tblfloor` F ON `F`.`floorName` =`t1`.`floor`
ORDER BY `t1`.`dateConfirm` DESC -- yes, again
Together with
INDEX(is_sold, dateConfirm)
How can t1.projectId="" ? Isn't projectId the PRIMARY KEY? (This is one of many reasons for needing the SHOW CREATE TABLE.)
If my suggestion leads to "duplicate" rows (that is, multiple rows with the same propertyId), don't simply add back the GROUP BY propertyId. Instead figure out why, and avoid the need for the GROUP BY. (That is probably the performance issue.)
A likely case is the GROUP_CONCAT. A common workaround is to change from
GROUP_CONCAT( distinct( L.locality)) AS Localities,
...
LEFT JOIN `sa_localitiez` L ON `L`.`id` = `PL`.`localityId`
to
( SELECT GROUP_CONCAT(distinct locality)
FROM sa_localitiez
WHERE id = PL.localityId ) AS Localities
...
# and remove the JOIN

mysql big query optimization

I'd need to optimize the following query which takes up to 10 minutes to run.
Performing the explain it seems to be running on all 350815 rows of the "table_3" table and 1 for all the others.
General rules to place indexes the propper way? Should I think about using multidimensional indexes? Where should I use them at first on the JOINS, the WHERE or the GROUP BY, if I remember right there should be a hierarchy to follow. Also If I have 1 row for all tables but one (in the row column of the explain table) how can I optimize usually my optimization consists in ending up with only one row for all columns but one.
All tables average from 100k to 1000k+ rows.
CREATE TABLE datab1.sku_performance
SELECT
table1.sku,
CONCAT(table1.sku,' ',table1.fk_container ) as sku_container,
table1.price as price,
SUM( CASE WHEN ( table1.fk_table1_status = 82
OR table1.fk_table1_status = 119
OR table1.fk_table1_status = 124
OR table1.fk_table1_status = 141
OR table1.fk_table1_status = 131) THEN 1 ELSE 0 END)
/ COUNT( DISTINCT id_catalog_school_class) as qty_returned,
SUM( CASE WHEN ( table1.fk_table1_status In (23,13,44,65,6,75,8,171,12,166))
THEN 1 ELSE 0 END)
/ COUNT( DISTINCT id_catalog_school_class) as qt,
container.id_container as container_id,
container.idden as container_idden,
container.delivery_badge,
catalog_school.id_catalog_school,
LEFT(catalog_school.flight_fair,2) as departing_country,
catalog_school.weight,
catalog_school.flight_type,
catalog_school.price,
table_3.id_table_3,
table_3.fk_catalog_brand,
MAX( LEFT( table_3.note,3 )) AS supplier,
GROUP_CONCAT( product_number, ' by ',FORMAT(catalog_school_class.quantity,0)
ORDER BY product_number ASC SEPARATOR ' + ') as supplier_prod,
Sum( distinct( catalog_school_class.purch_pri * catalog_school_class.quantity)) AS final_purch_pri,
catalog_groupp.idden as supplier_idden,
catalog_category_details.id_catalog_category,
catalog_category_details.cat1 as product_cat1,
catalog_category_details.cat2 as product_cat2,
COUNT( distinct catalog_school_class.id_catalog_school_class) as setinfo,
datab1.pageviewgrouped.pv as page_views,
Sum(distinct(catalog_school_class.purch_pri * catalog_school_class.quantity)) AS purch_pri,
container_has_table_3.position,
max( table1.created_at ) as last_order_date
FROM
table1
LEFT JOIN container
ON table1.fk_container = container.id_container
LEFT JOIN catalog_school
ON table1.sku = catalog_school.sku
LEFT JOIN table_3
ON catalog_school.fk_table_3 = table_3.id_table_3
LEFT JOIN container_has_table_3
ON table_3.id_table_3 = container_has_table_3.fk_table_3
LEFT JOIN datab1.pageviewgrouped
on table_3.id_table_3 = datab1.pageviewgrouped.url
LEFT JOIN datab1.catalog_category_details
ON datab1.catalog_category_details.id_catalog_category = table_3_has_catalog_minority.fk_catalog_category
LEFT JOIN catalog_groupp
ON table_3.fk_catalog_groupp = catalog_groupp.id_catalog_groupp
LEFT JOIN table_3_has_catalog_minority
ON table_3.id_table_3 = table_3_has_catalog_minority.fk_table_3
LEFT JOIN catalog_school_class
ON catalog_school.id_catalog_school = catalog_school_class.fk_catalog_school
WHERE
table_3.status_ok = 1
AND catalog_school.status = 'active'
AND table_3_has_catalog_minority.is_primary = '1'
GROUP BY
table1.sku,
table1.fk_container;
rows per table :
.table1 960096 to 1.3mn rows
.container 9275 to 13000 rows
.catalog_school 709970 to 1 mn rows
.table_3 709970 to 1 mn rows
.container_has_table_3 709970 to 1 mn rows
.pageviewgrouped 500000 rows
.catalog_school_class 709970 to 1 mn rows
.catalog_groupp 3000 rows
.table_3_has_catalog_minority 709970 to 1 mn rows
.catalog_category_details 659 rows
Too much to put into a single comment, so I'll add here and adjust later as possibly needed... You have LEFT JOINs everywhere, but your WHERE clause is specifically qualifying fields from the Table_3, Catalog_School and Table_3_has_catalog_minority. This by default changes them to INNER JOINs.
With respect to your where clause
WHERE
table_3.status_ok = 1
AND catalog_school.status = 'active'
AND table_3_has_catalog_minority.is_primary = '1'
Which table / column would have the smallest results based on these criteria. ex: Table_3.Status_ok = 1 might have 500k records but table_3_has_catalog_minority.is_primary may only have 65k and catalog_school.status = 'active' may have 430k.
Also, some of your columns are not qualified with the table they are coming from. Can you please confirm... such as "id_catalog_school_class" and "product_number"
SOMETIMES, changing the order of the tables, with good knowledge of the makeup of the data and in MySQL adding a "STRAIGHT_JOIN" keyword can improve performance. This was something I've had in the past working with gov't database of contracts and grants with 20+ million records and joining to about 15+ lookup tables. It went from hanging the server to getting the query finished in less than 2 hrs. Considering the amount of data I was dealing with, that was actually a good time.
AFTER dissecting this thing some, I restructured a bit more for readability, added aliases for table references and changed the order of the query and have some suggested indexes. To help the query, I tried moving the Catalog_School table to the first position and added the STRAIGHT_JOIN. The index is based on the STATUS first to match the WHERE clause, THEN I included the SKU as it is first element of the GROUP BY, then the other columns used to join to the subsequent tables. By having these columns in the index, it can qualify the joins without having to go to the raw data.
By changing the group by to the Catalog_School.SKU instead of table_1.SKU the index from catalog_school can be used to help optimize that. It is the same value since the join from the catalog_school.sku = table_1.sku. I also added index references for table_1 and table_3 that are suggestions -- again, to preemptively qualify the joins without going to the raw data pages of the tables.
I would be interested in knowing the final performance (better or worse) from your data.
TABLE INDEX ON...
catalog_school ( status, sku, fk_table_3, id_catalog_school )
table_1 ( sku, fk_container )
table_3 ( id_table_3, status_ok, fk_catalog_groupp )
SELECT STRAIGHT_JOIN
CS.sku,
CONCAT(CS.sku,' ',T1.fk_container ) as sku_container,
T1.price as price,
SUM( CASE WHEN ( T1.fk_table1_status IN ( 82, 119, 124, 141, 131)
THEN 1 ELSE 0 END)
/ COUNT( DISTINCT CSC.id_catalog_school_class) as qty_returned,
SUM( CASE WHEN ( T1.fk_table1_status In (23,13,44,65,6,75,8,171,12,166))
THEN 1 ELSE 0 END)
/ COUNT( DISTINCT CSC.id_catalog_school_class) as qt,
CS.id_catalog_school,
LEFT(CS.flight_fair,2) as departing_country,
CS.weight,
CS.flight_type,
CS.price,
T3.id_table_3,
T3.fk_catalog_brand,
MAX( LEFT( T3.note,3 )) AS supplier,
C.id_container as container_id,
C.idden as container_idden,
C.delivery_badge,
GROUP_CONCAT( product_number, ' by ',FORMAT(CSC.quantity,0)
ORDER BY product_number ASC SEPARATOR ' + ') as supplier_prod,
Sum( distinct( CSC.purch_pri * CSC.quantity)) AS final_purch_pri,
CGP.idden as supplier_idden,
CCD.id_catalog_category,
CCD.cat1 as product_cat1,
CCD.cat2 as product_cat2,
COUNT( distinct CSC.id_catalog_school_class) as setinfo,
PVG.pv as page_views,
Sum(distinct(CSC.purch_pri * CSC.quantity)) AS purch_pri,
CHT3.position,
max( T1.created_at ) as last_order_date
FROM
catalog_school CS
JOIN table1 T1
ON CS.sku = T1.sku
LEFT JOIN container C
ON T1.fk_container = C.id_container
LEFT JOIN catalog_school_class CSC
ON CS.id_catalog_school = CSC.fk_catalog_school
JOIN table_3 T3
ON CS.fk_table_3 = T3.id_table_3
JOIN table_3_has_catalog_minority T3HCM
ON T3.id_table_3 = T3HCM.fk_table_3
LEFT JOIN datab1.catalog_category_details CCD
ON T3HCM.fk_catalog_category = CCD.id_catalog_category
LEFT JOIN container_has_table_3 CHT3
ON T3.id_table_3 = CHT3.fk_table_3
LEFT JOIN datab1.pageviewgrouped PVG
on T3.id_table_3 = PVG.url
LEFT JOIN catalog_groupp CGP
ON T3.fk_catalog_groupp = CGP.id_catalog_groupp
WHERE
CS.status = 'active'
AND T3.status_ok = 1
AND T3HCM.is_primary = '1'
GROUP BY
CS.sku,
T1.fk_container;