Get 100 rows with maximum of 10 rows per group - mysql

Question

I have the following query, and I would like to get 100 items from the database, but host_id is in the urls table many times, and I would like to get a maximum of 10 unique rows from that table per host_id.
select *
from urls
join hosts using(host_id)
where
(
last_run_date is null
or last_run_date <= date_sub(curdate(), interval 30 day)
)
and ignore_url != 1
limit 100
So, I would like:
Maximum Results = 100
Max Rows Per Host = 10
I am not sure what I would need to do to accomplish this task. Is there a way to do this without a subquery?
Hosts Table
CREATE TABLE `hosts` (
`host_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`host` VARCHAR(50) NOT NULL,
`last_fetched` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`ignore_host` TINYINT(1) UNSIGNED NOT NULL,
PRIMARY KEY (`host_id`),
UNIQUE INDEX `host` (`host`)
)
Urls Table
CREATE TABLE `urls` (
`url_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`parent_url_id` INT(10) UNSIGNED NOT NULL,
`scheme` VARCHAR(5) NOT NULL,
`host_id` INT(10) UNSIGNED NOT NULL,
`path` VARCHAR(500) NOT NULL,
`query` VARCHAR(500) NOT NULL,
`date_found` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`last_run_date` DATETIME NULL DEFAULT NULL,
`ignore_url` TINYINT(1) UNSIGNED NOT NULL,
PRIMARY KEY (`url_id`),
UNIQUE INDEX `host_path_query` (`host_id`, `path`, `query`)
)

Answer 1

Thats it (I hope)
I cant test i real. i have no data. pls test it and give me a little ping.
SELECT *
FROM (
SELECT
#nr:=IF(#lasthost = host_id, #nr+1, 1) AS nr,
u.*,
#lasthost:=IF(#lasthost = host_id, #lasthost, host_id) AS lasthost
FROM
urls u,
( SELECT #nr:=4, #lasthost:=-1 ) AS tmp
WHERE (
last_run_date IS NULL
OR last_run_date <= date_sub(curdate(), INTERVAL 30 DAY)
)
AND ignore_url != 1
ORDER BY host_id, last_run_date
) AS t
LEFT JOIN HOSTS USING(host_id)
WHERE t.nr < 11
LIMIT 100;
ok,
first:
I only select the rows with your query, and order it
by the host_id and time
SELECT
u.*
FROM
urls u
( SELECT #nr:=4, #lasthost:=-1 ) AS tmp
WHERE (
last_run_date IS NULL
OR last_run_date <= date_sub(curdate(), INTERVAL 30 DAY)
)
AND ignore_url != 1
ORDER BY host_id, last_run_date
second
I add to variables nr and lasthost and setup it in the select. Now
i count nr each row and reset it to 1 if the host_id is change. So i get a
list of rows numbert from 1 to n for each host_id
SELECT
#nr:=IF(#lasthost = host_id, #nr+1, 1) AS nr,
u.*,
#lasthost:=IF(#lasthost = host_id, #lasthost, host_id) AS lasthost
FROM
urls u,
( SELECT #nr:=4, #lasthost:=-1 ) AS tmp
WHERE (
last_run_date IS NULL
OR last_run_date <= date_sub(curdate(), INTERVAL 30 DAY)
)
AND ignore_url != 1
ORDER BY host_id, last_run_date
third
i put it this query in a new select so i can join your second table and restrict the result only for rows less 11 and also limit the result to 100
SELECT *
FROM (
SELECT
#nr:=IF(#lasthost = host_id, #nr+1, 1) AS nr,
u.*,
#lasthost:=IF(#lasthost = host_id, #lasthost, host_id) AS lasthost
FROM
urls u,
( SELECT #nr:=4, #lasthost:=-1 ) AS tmp
WHERE (
last_run_date IS NULL
OR last_run_date <= date_sub(curdate(), INTERVAL 30 DAY)
)
AND ignore_url != 1
ORDER BY host_id, last_run_date
) AS t
LEFT JOIN HOSTS USING(host_id)
WHERE t.nr < 11
LIMIT 100;
Thats all

Answer 2

So you need a limited JOIN. Another guess:
SELECT * FROM hosts
LEFT JOIN urls ON
urls.host_id = hosts.host_id
WHERE urls.host_id IN
(SELECT host_id FROM urls
LIMIT 0,10)
LIMIT 0,100