Big query on a big mysql database - mysql

Question

I have a big query to count 3 distinct things on same data table:
SELECT leagues.feed_league_id AS mleague,
leagues.league_id AS mleague_id,
leagues.league_name, countries.feed_name,
(SELECT COUNT(DISTINCT m2.match_id)
FROM `matches` AS m2
LEFT JOIN stats ON stats.match_id=m2.match_id
LEFT JOIN leagues AS l1 ON m2.feed_league_id=l1.feed_league_id
WHERE stats.tipo='Golos1ª2ª' AND
m2.status='FT' AND
(m2.home_ht_score*1+ m2.away_ht_score*1) >1 AND
m2.home_ht_score REGEXP '[0-9]+' AND
m2.away_ht_score REGEXP '[0-9]+' AND
m2.feed_league_id=mleague
) AS Total,
(SELECT COUNT(DISTINCT m3.match_id)
FROM `matches` AS m3
LEFT JOIN stats ON stats.match_id=m3.match_id
LEFT JOIN leagues AS l1 ON m3.feed_league_id=l1.feed_league_id
WHERE stats.tipo='Golos1ª2ª' AND
m3.status='FT' AND
(m3.home_ht_score*1+ m3.away_ht_score*1) >1 AND
(m3.home_ft_score*1+m3.away_ft_score*1)-(m3.home_ht_score*1+ m3.away_ht_score*1)>0 AND
m3.home_ht_score REGEXP '[0-9]+' AND
m3.away_ht_score REGEXP '[0-9]+' AND
m3.feed_league_id=mleague
) AS Golos,
(SELECT COUNT(DISTINCT m2.match_id)
FROM `matches` AS m2
LEFT JOIN stats ON stats.match_id=m2.match_id
LEFT JOIN leagues AS l1 ON m2.feed_league_id=l1.feed_league_id
WHERE DATEDIFF(DATE(NOW()), date)<=4 AND
stats.tipo='Golos1ª2ª' AND
m2.status='FT' AND
(m2.home_ht_score*1+ m2.away_ht_score*1) >1 AND
m2.home_ht_score REGEXP '[0-9]+' AND
m2.away_ht_score REGEXP '[0-9]+' AND
m2.feed_league_id=mleague
) AS LastDays
FROM leagues
LEFT JOIN countries ON countries.country_id=leagues.country_id
GROUP BY leagues.feed_league_id HAVING Total>20
ORDER BY Golos/Total DESC`
Table "leagues" - 828 records
CREATE TABLE IF NOT EXISTS `leagues` (
`league_id` int(11) NOT NULL AUTO_INCREMENT,
`league_name` varchar(255) NOT NULL,
`alt_league_name` varchar(255) NOT NULL,
`league_season` varchar(255) NOT NULL,
`feed_league_id` varchar(8) NOT NULL,
`feed_sub_id` varchar(10) NOT NULL,
`country_id` int(11) NOT NULL,
PRIMARY KEY (`league_id`),
KEY `league_name` (`league_name`),
KEY `league_season` (`league_season`),
KEY `feed_league_id` (`feed_league_id`),
KEY `country_id` (`country_id`),
KEY `feed_sub_id` (`feed_sub_id`),
KEY `alt_league_name` (`alt_league_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=837
Table "matches" - 125K records
CREATE TABLE IF NOT EXISTS `matches` (
`match_id` int(11) NOT NULL AUTO_INCREMENT,
`date` date NOT NULL,
`feed_match_id` varchar(255) NOT NULL,
`status` varchar(10) NOT NULL,
`time` varchar(10) NOT NULL,
`updated` datetime NOT NULL,
`home_team_id` int(11) NOT NULL,
`away_team_id` int(11) NOT NULL,
`home_ft_score` varchar(11) NOT NULL,
`away_ft_score` varchar(11) NOT NULL,
`home_ht_score` varchar(11) NOT NULL,
`away_ht_score` varchar(11) NOT NULL,
`country_id` int(11) NOT NULL,
`league_id` int(11) NOT NULL,
`feed_league_id` varchar(8) NOT NULL,
`feed_sub_id` varchar(10) NOT NULL,
`stage_id` int(11) NOT NULL,
`week_number` int(11) NOT NULL,
`n_updates` int(11) NOT NULL,
PRIMARY KEY (`match_id`),
KEY `date` (`date`),
KEY `feed_match_id` (`feed_match_id`),
KEY `status` (`status`),
KEY `time` (`time`),
KEY `updated` (`updated`),
KEY `home_ft_score` (`home_ft_score`),
KEY `away_ft_score` (`away_ft_score`),
KEY `home_ht_score` (`home_ht_score`),
KEY `away_ht_score` (`away_ht_score`),
KEY `country_id` (`country_id`),
KEY `league_id` (`league_id`),
KEY `stage_id` (`stage_id`),
KEY `week_number` (`week_number`),
KEY `home_team_id` (`home_team_id`),
KEY `away_team_id` (`away_team_id`),
KEY `feed_league_id` (`feed_league_id`),
KEY `feed_sub_id` (`feed_sub_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=125231 ;
Table "stats" - 250K records
CREATE TABLE IF NOT EXISTS `stats` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`match_id` int(11) NOT NULL,
`stat` varchar(255) NOT NULL,
`equipa` varchar(255) NOT NULL,
`tipo_id` smallint(6) NOT NULL,
PRIMARY KEY (`id`),
KEY `match_id` (`match_id`),
KEY `stat` (`stat`),
KEY `equipa` (`equipa`),
KEY `tipo_id` (`tipo_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=246459 ;
Can someone help me simplifying this query. It takes too long to be executed.
If you need more information, please ask.
Thanks.

Answer 1

I don't even know where to begin. The basic problem is that you are counting in your SELECT clause. This means that for each record in leagues (main query), you are executing 3 other big queries (not to mention all the regex). You're not giving your server a chance.
Move your counting from SELECT to FROM (make them join tables).
UPDATE: Here's what I mean:
SELECT leagues.feed_league_id, Total.cnt
FROM leagues
JOIN countries ON countries.country_id=leagues.country_id
JOIN (SELECT l1.feed_league_id, COUNT(DISTINCT m2.match_id) as cnt
FROM `matches` AS m2
LEFT JOIN stats ON stats.match_id=m2.match_id
LEFT JOIN leagues AS l1 ON m2.feed_league_id=l1.feed_league_id
WHERE stats.tipo='Golos1ª2ª' AND m2.status='FT' AND (m2.home_ht_score*1+ m2.away_ht_score*1) >1 AND m2.home_ht_score REGEXP '[0-9]+' AND m2.away_ht_score REGEXP '[0-9]+'
GROUP BY l1.feed_league_id
) AS Total on Total.feed_league_id = leagues.feed_league_id
GROUP BY leagues.feed_league_id
As you can see this is only for Total, but it's the same for the other two big queries.