Data Structure causing impossible joins - mysql

Tables:
nodes
data_texts
data_profiles
data_locations
data_profiles
data_media
data_products
data_metas
categories
tags
categories_nodes
tags_nodes
This question is a generalized question and is on the back of another question
Explanation:
Each of the "data" tables has a node_id that refers back to the id of the nodes table (hasMany/belongsTo association).
A "Node" can be anything - a TV Show, a Movie, a Person, an Article...etc (all generated via a CMS, so the user can control what type of "Nodes" they want).
When pulling data, I want to be able to query against certain fields. For example if they do a search, I want to be able to pull nodes that have data_texts.title = '%george%' or order by the datetime field in data_locations.
The problem is, when I do a join on all seven data tables (or more), the query has to hit so many combined rows that it just times out (even with a nearly empty database.... total 200 rows across the entire database).
I realize I can determine IF I need a join depending on what I'm doing - but even with five or six joins (once the database gets to 10k+ records), it's going to be horribly slow, if it works at all. Per this question, the query I'm using just doing a join on these tables times out completely.
Each node can have multiple rows of each data type (for multi-language reasons among others).
I'm completely defeated - I'm at the point where I think I need to restructure the entire thing, but don't have the time for that. I've thought about combining all into one table, but aren't sure how....etc
nodes
CREATE TABLE `nodes` (
`id` CHAR(36) NOT NULL,
`name` VARCHAR(100) NOT NULL,
`slug` VARCHAR(100) NOT NULL,
`node_type_id` CHAR(36) NOT NULL,
`site_id` CHAR(36) NOT NULL,
`created` DATETIME NOT NULL,
`modified` DATETIME NOT NULL,
PRIMARY KEY (`id`),
INDEX `nodeTypeId` (`node_type_id`),
INDEX `slug` (`slug`),
INDEX `nodeId` (`id`)
)
COLLATE='latin1_swedish_ci'
ENGINE=MyISAM;
data_texts:
CREATE TABLE `data_texts` (
`id` CHAR(36) NOT NULL,
`title` VARCHAR(250) NULL DEFAULT NULL,
`subtitle` VARCHAR(500) NULL DEFAULT NULL,
`content` LONGTEXT NULL,
`byline` VARCHAR(250) NULL DEFAULT NULL,
`language_id` CHAR(36) NULL DEFAULT NULL,
`foreign_key` CHAR(36) NULL DEFAULT NULL,
`model` VARCHAR(40) NULL DEFAULT NULL,
`node_id` CHAR(36) NULL DEFAULT NULL,
`created` DATETIME NOT NULL,
`modified` DATETIME NOT NULL,
PRIMARY KEY (`id`),
INDEX `nodeId` (`node_id`),
INDEX `languageId_nodeId` (`language_id`, `node_id`),
INDEX `foreignKey_model` (`foreign_key`, `model`)
)
COLLATE='utf8_general_ci'
ENGINE=MyISAM;
data_profiles
CREATE TABLE `data_profiles` (
`id` CHAR(36) NOT NULL,
`name` VARCHAR(80) NULL DEFAULT NULL,
`email_personal` VARCHAR(100) NULL DEFAULT NULL,
`email_business` VARCHAR(100) NULL DEFAULT NULL,
`email_other` VARCHAR(100) NULL DEFAULT NULL,
`title` VARCHAR(100) NULL DEFAULT NULL,
`description` LONGTEXT NULL,
`prefix` VARCHAR(40) NULL DEFAULT NULL,
`phone_home` VARCHAR(40) NULL DEFAULT NULL,
`phone_business` VARCHAR(40) NULL DEFAULT NULL,
`phone_mobile` VARCHAR(40) NULL DEFAULT NULL,
`phone_other` VARCHAR(40) NULL DEFAULT NULL,
`foreign_key` CHAR(36) NULL DEFAULT NULL,
`model` VARCHAR(40) NULL DEFAULT NULL,
`node_id` CHAR(36) NULL DEFAULT NULL,
`language_id` CHAR(36) NULL DEFAULT NULL,
`created` DATETIME NOT NULL,
`modified` DATETIME NOT NULL,
`user_id` CHAR(36) NULL DEFAULT NULL,
PRIMARY KEY (`id`),
INDEX `nodeId` (`node_id`),
INDEX `languageId_nodeId` (`node_id`, `language_id`),
INDEX `foreignKey_model` (`foreign_key`, `model`)
)
COLLATE='latin1_swedish_ci'
ENGINE=MyISAM;
categories
CREATE TABLE `categories` (
`id` CHAR(36) NOT NULL,
`name` VARCHAR(100) NOT NULL,
`node_type_id` CHAR(36) NOT NULL,
`site_id` CHAR(36) NOT NULL,
`slug` VARCHAR(100) NOT NULL,
`created` DATETIME NOT NULL,
`modified` DATETIME NOT NULL,
PRIMARY KEY (`id`),
INDEX `nodeTypeId` (`node_type_id`),
INDEX `slug` (`slug`)
)
COMMENT='Used to categorize nodes'
COLLATE='utf8_general_ci'
ENGINE=MyISAM;
categories_nodes
CREATE TABLE `categories_nodes` (
`id` CHAR(36) NOT NULL,
`category_id` CHAR(36) NOT NULL,
`node_id` CHAR(36) NOT NULL,
PRIMARY KEY (`id`),
INDEX `categoryId_nodeId` (`category_id`, `node_id`)
)
COLLATE='utf8_general_ci'
ENGINE=MyISAM;
node_tags
CREATE TABLE `node_tags` (
`id` CHAR(36) NOT NULL,
`name` VARCHAR(40) NOT NULL,
`site_id` CHAR(36) NOT NULL,
`created` DATETIME NOT NULL,
`modified` DATETIME NOT NULL,
PRIMARY KEY (`id`),
INDEX `siteId` (`site_id`)
)
COLLATE='utf8_general_ci'
ENGINE=MyISAM;
nodes_node_tags
CREATE TABLE `nodes_node_tags` (
`id` CHAR(36) NOT NULL,
`node_id` CHAR(36) NOT NULL,
`node_tag_id` CHAR(36) NOT NULL,
PRIMARY KEY (`id`),
INDEX `node_id_node_tag_id` (`node_id`, `node_tag_id`)
)
COLLATE='utf8_general_ci'
ENGINE=MyISAM;
MySQL:
SELECT `Node`.`id`, `Node`.`name`, `Node`.`slug`, `Node`.`node_type_id`, `Node`.`site_id`, `Node`.`created`, `Node`.`modified`
FROM `mysite`.`nodes` AS `Node`
LEFT JOIN `mysite`.`data_date_times` AS `DataDateTime` ON (`DataDateTime`.`node_id` = `Node`.`id`)
LEFT JOIN `mysite`.`data_locations` AS `DataLocation` ON (`DataLocation`.`node_id` = `Node`.`id`)
LEFT JOIN `mysite`.`data_media` AS `DataMedia` ON (`DataMedia`.`node_id` = `Node`.`id`)
LEFT JOIN `mysite`.`data_metas` AS `DataMeta` ON (`DataMeta`.`node_id` = `Node`.`id`)
LEFT JOIN `mysite`.`data_profiles` AS `DataProfile` ON (`DataProfile`.`node_id` = `Node`.`id`)
LEFT JOIN `mysite`.`data_products` AS `DataProduct` ON (`DataProduct`.`node_id` = `Node`.`id`)
LEFT JOIN `mysite`.`data_texts` AS `DataText` ON (`DataText`.`node_id` = `Node`.`id`)
WHERE 1=1
GROUP BY `Node`.`id`

Firstly, try InnoDB, not MyISAM.
Secondly, remove the group by, see how well it runs then, and how many rows are involved. Shouldn't be that many, but it's interesting.
You don't need the 'nodeId' index on node (as you already have it as a primary key). Again, shouldn't make any difference.
The where clause is irrelevant. You can remove it with no effect one way or another.
Thirdly, well, something is seriously broken.
Have a quick look on how to start profiling (e.g. http://dev.mysql.com/doc/refman/5.0/en/show-profile.html) , and run a profile command to see where all the time is going. Post it here if it doesn't immediately show that something is broken.

I'm unfortunately not in a position where I can do any tests right now. I'll just throw out some ideas. I might be able to do some tests later.
Be suspicious of different collations.
Some of your ids are useless. For example, you should drop the column categories_nodes.id, and put a primary key constraint on {category_id, node_id} instead.
Be suspicious of any design that requires joining all the tables at run time. There are better ways.
Use innodb and foreign key constraints.

Related

Long running Mysql Query on Indexes and sort by clause

I have a very long running MySql query. The query simply joins two tables which are very huge
bizevents - Nearly 34 Million rows
bizevents_actions - Nearly 17 million rows
Here is the query:
select
bizevent0_.id as id1_37_,
bizevent0_.json as json2_37_,
bizevent0_.account_id as account_3_37_,
bizevent0_.createdBy as createdB4_37_,
bizevent0_.createdOn as createdO5_37_,
bizevent0_.description as descript6_37_,
bizevent0_.iconCss as iconCss7_37_,
bizevent0_.modifiedBy as modified8_37_,
bizevent0_.modifiedOn as modified9_37_,
bizevent0_.name as name10_37_,
bizevent0_.version as version11_37_,
bizevent0_.fired as fired12_37_,
bizevent0_.preCreateFired as preCrea13_37_,
bizevent0_.entityRefClazz as entityR14_37_,
bizevent0_.entityRefIdAsStr as entityR15_37_,
bizevent0_.entityRefIdType as entityR16_37_,
bizevent0_.entityRefName as entityR17_37_,
bizevent0_.entityRefType as entityR18_37_,
bizevent0_.entityRefVersion as entityR19_37_
from
BizEvent bizevent0_
left outer join BizEvent_actions actions1_ on
bizevent0_.id = actions1_.BizEvent_id
where
bizevent0_.createdOn >= '1969-12-31 19:00:01.0'
and (actions1_.action <> 'SoftLock'
and actions1_.targetRefClazz = 'com.biznuvo.core.orm.domain.org.EmployeeGroup'
and actions1_.targetRefIdAsStr = '1'
or actions1_.action <> 'SoftLock'
and actions1_.objectRefClazz = 'com.biznuvo.core.orm.domain.org.EmployeeGroup'
and actions1_.objectRefIdAsStr = '1')
order by
bizevent0_.createdOn;
Below are the table definitions -- As you see i have defined the indexes well enough on these two tables on all the search columns plus the sort column. But still my queries are running for very very long time. Appreciate any more ideas either with respective indexing.
-- bizevent definition
CREATE TABLE `bizevent` (
`id` bigint(20) NOT NULL,
`json` longtext,
`account_id` int(11) DEFAULT NULL,
`createdBy` varchar(50) NOT NULL,
`createdon` datetime(3) DEFAULT NULL,
`description` varchar(255) DEFAULT NULL,
`iconCss` varchar(50) DEFAULT NULL,
`modifiedBy` varchar(50) NOT NULL,
`modifiedon` datetime(3) DEFAULT NULL,
`name` varchar(255) NOT NULL,
`version` int(11) NOT NULL,
`fired` bit(1) NOT NULL,
`preCreateFired` bit(1) NOT NULL,
`entityRefClazz` varchar(255) DEFAULT NULL,
`entityRefIdAsStr` varchar(255) DEFAULT NULL,
`entityRefIdType` varchar(25) DEFAULT NULL,
`entityRefName` varchar(255) DEFAULT NULL,
`entityRefType` varchar(50) DEFAULT NULL,
`entityRefVersion` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `IDXk9kxuuprilygwfwddr67xt1pw` (`createdon`),
KEY `IDXsf3ufmeg5t9ok7qkypppuey7y` (`entityRefIdAsStr`),
KEY `IDX5bxv4g72wxmjqshb770lvjcto` (`entityRefClazz`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-- bizevent_actions definition
CREATE TABLE `bizevent_actions` (
`BizEvent_id` bigint(20) NOT NULL,
`action` varchar(255) DEFAULT NULL,
`objectBizType` varchar(255) DEFAULT NULL,
`objectName` varchar(255) DEFAULT NULL,
`objectRefClazz` varchar(255) DEFAULT NULL,
`objectRefIdAsStr` varchar(255) DEFAULT NULL,
`objectRefIdType` int(11) DEFAULT NULL,
`objectRefVersion` int(11) DEFAULT NULL,
`targetBizType` varchar(255) DEFAULT NULL,
`targetName` varchar(255) DEFAULT NULL,
`targetRefClazz` varchar(255) DEFAULT NULL,
`targetRefIdAsStr` varchar(255) DEFAULT NULL,
`targetRefIdType` int(11) DEFAULT NULL,
`targetRefVersion` int(11) DEFAULT NULL,
`embedJson` longtext,
`actions_ORDER` int(11) NOT NULL,
PRIMARY KEY (`BizEvent_id`,`actions_ORDER`),
KEY `IDXa21hhagjogn3lar1bn5obl48gll` (`action`),
KEY `IDX7agsatk8u8qvtj37vhotja0ce77` (`targetRefClazz`),
KEY `IDXa7tktl678kqu3tk8mmkt1mo8lbo` (`targetRefIdAsStr`),
KEY `IDXa22eevu7m820jeb2uekkt42pqeu` (`objectRefClazz`),
KEY `IDXa33ba772tpkl9ig8ptkfhk18ig6` (`objectRefIdAsStr`),
CONSTRAINT `FKr9qjs61id11n48tdn1cdp3wot` FOREIGN KEY (`BizEvent_id`) REFERENCES `bizevent` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;>
By the way we are using Amazon RDS 5.7.33 MySql version. 16 GB RAM and 4 vCPU.
I also did a Explain Extended on the query and below is what it shows. Appreciate any help.
Initially the search of the bizevent_actions didn;t have the indexes defined. I have defined the indexes for them and tried the query but of no use.
One technique that worked for me in a similar situation was abandoning the idea of JOIN completely and switching to queries by PK. More detailed: find out which table in join would give less rows on average if you use only that table and related filter to query; get the primary keys from that table and then query the other one using WHERE pk IN ().
In your case one example would be:
SELECT
bizevent0_.id as id1_37_,
bizevent0_.json as json2_37_,
bizevent0_.account_id as account_3_37_,
...
FROM BizEvent bizevent0_
WHERE
bizevent0_.createdOn >= '1969-12-31 19:00:01.0'
AND bizevent0_.id IN (
SELECT BizEvent_id
FROM BizEvent_actions actions1_
WHERE
actions1_.action <> 'SoftLock'
and actions1_.targetRefClazz = 'com.biznuvo.core.orm.domain.org.EmployeeGroup'
and actions1_.targetRefIdAsStr = '1'
or actions1_.action <> 'SoftLock'
and actions1_.objectRefClazz = 'com.biznuvo.core.orm.domain.org.EmployeeGroup'
and actions1_.objectRefIdAsStr = '1')
ORDER BY
bizevent0_.createdOn;
This assumes that you're not actually willing to select 33+ Mio rows from BizEvent though - your code with LEFT OUTER JOIN would have done exactly this.

MySQL - ordering by column with many different values in big table

Probably through poor database design, the following really simple query is taking ~1.5 minutes to run.
SELECT s.title, t.name AS team_name
FROM stories AS s
JOIN teams AS t ON s.team_id = t.id
WHERE s.pubdate >= "1970-01-01 00:00"
ORDER BY s.hits /* <-- here's the problem */
LIMIT 3 OFFSET 0
The problem is the stories table is fairly big, with ~1.5m rows, and there's a ton of unique values for hits (this column logs the hits to each story.)
Take out the order clause and it resolves almost instantly.
Question: what can I do to optimise for queries like this? Presumably I shouldn't apply an index to hits since direct no look-ups take place on that column.
[UPDATE]
SHOW CREATE TABLE for all tables concerned:
CREATE TABLE stories (
`id` varchar(11) NOT NULL,
`link` text NOT NULL,
`title` varchar(255) CHARACTER SET utf8 NOT NULL,
`description` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
`pubdate` datetime NOT NULL,
`source_id` varchar(11) NOT NULL,
`team_id` varchar(11) NOT NULL,
`hits` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `Unique combo (title + date)` (`title`,`pubdate`),
KEY `team (FK)` (`team_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1
CREATE TABLE teams (
`id` varchar(11) NOT NULL,
`is_live` enum('y') DEFAULT NULL,
`name` varchar(50) NOT NULL,
`short_name` varchar(12) DEFAULT NULL,
`server` varchar(11) DEFAULT NULL,
`url_token` varchar(255) NOT NULL,
`league` varchar(11) NOT NULL,
`away_game_id` varchar(255) DEFAULT NULL,
`digest_list_id` varchar(25) DEFAULT NULL,
`twitter_handle` varchar(255) DEFAULT NULL,
`no_official_news` enum('y') DEFAULT NULL,
`alt_names` varchar(255) DEFAULT NULL,
`no_use_nickname` enum('y') DEFAULT NULL,
`official_hashtag` varchar(30) DEFAULT NULL,
`merge_news_and_fans` enum('y') DEFAULT NULL,
`colour_1` varchar(6) NOT NULL,
`colour_2` varchar(6) DEFAULT NULL,
`colour_3` varchar(6) DEFAULT NULL,
`link_colour_modifier` float DEFAULT NULL,
`alt_link_colour_modifier` float DEFAULT NULL,
`title_shade` enum('dark','light') NOT NULL,
`shirt_style` enum('vert_stripes','horiz_stripes','vert_stripes_thin','horiz_stripes_thin','vert_split','horiz_split') DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `URL token` (`url_token`),
KEY `league (FK)` (`league`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1
Consider removing the filter on pubdate if the user does not need it. It confuses the optimizer.
INDEX(hits, pubdate, title)
will probably help the query the most. It is "covering".
The reason why removing ORDER BY runs fast: Without it, it gives you any 3 rows. With it, and without a useful index, it needs to sort the 1.5M rows to discover the 3 with the least number of hits.
Perhaps you wanted ORDER BY s.hits DESC? -- to get those with the most hits.

MYSQL INNER JOIN is slow with index

this is my simple inner join:
SELECT
SUM(ASSNZ.assenzeDidattiche) AS TotaleAssenze,
SUM(ASSNZ.ore) AS totale_parziale,
FLOOR(((SUM(ASSNZ.assenzeDidattiche) / SUM(ASSNZ.ore)) * 100)) AS andamento,
MAX(ASSNZ.dataLezione) AS ultima_lezione,
ASSNZ.idServizio,
ASSNZ.idUtente
FROM
ciac_corsi_assenze AS ASSNZ
INNER JOIN
ciac_serviziAcquistati_ITA AS ACQ
ON ACQ.idContatto = ASSNZ.idUtente
AND ACQ.idServizio = ASSNZ.idServizio
AND ACQ.stato_allievo <> 'ritirato'
GROUP BY
ASSNZ.idServizio,
ASSNZ.idUtente
table "ASSNZ" has 213886 rows with index "idUtente", "idServizio"
table "ACQ" has 8950 rows with index "idContatto", "idServizio"
ASSNZ table:
CREATE TABLE `ciac_corsi_assenze` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`idUtente` int(11) DEFAULT NULL,
`idServizio` int(11) DEFAULT NULL,
`idCorso` int(11) DEFAULT NULL,
`idCalendario` int(11) DEFAULT NULL,
`modalita` varchar(255) DEFAULT NULL,
`ore` int(11) DEFAULT NULL,
`assenzeDidattiche` float DEFAULT NULL,
`assenzeAmministrative` float DEFAULT NULL,
`dataLezione` date DEFAULT NULL,
`ora_inizio` varchar(8) DEFAULT NULL,
`ora_fine` varchar(8) DEFAULT NULL,
`dataFineStage` date DEFAULT NULL,
`giustificata` varchar(128) DEFAULT NULL,
`motivazione` longtext,
`grouped` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `idUtente` (`idUtente`) USING BTREE,
KEY `idServizio` (`idServizio`) USING BTREE,
KEY `dataLezione` (`dataLezione`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=574582 DEFAULT CHARSET=utf8;
ACQ table:
CREATE TABLE `ciac_serviziacquistati_ita` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`idServizio` int(11) NOT NULL,
`idContatto` int(11) NOT NULL,
`idAzienda` int(11) NOT NULL,
`idSede` int(11) NOT NULL,
`tipoPersona` int(11) NOT NULL,
`num_registro` int(11) NOT NULL,
`codice` varchar(255) CHARACTER SET latin1 DEFAULT NULL,
`dal` date NOT NULL,
`al` date NOT NULL,
`ore` int(11) NOT NULL,
`costoOrario` decimal(10,0) NOT NULL,
`annoFormativo` varchar(128) CHARACTER SET latin1 NOT NULL,
`stato_attuale` int(11) NOT NULL,
`datore_attuale` int(11) NOT NULL,
`stato_allievo` varchar(64) CHARACTER SET latin1 NOT NULL DEFAULT 'corsista',
`data_ritiro` date DEFAULT NULL,
`crediti_formativi` int(11) NOT NULL,
`note` longtext CHARACTER SET latin1 NOT NULL,
`valore_economico` decimal(10,2) NOT NULL,
`dataInserimento` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idServizio` (`idServizio`) USING BTREE,
KEY `idAzienda` (`idAzienda`) USING BTREE,
KEY `idContatto` (`idContatto`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=9542 DEFAULT CHARSET=utf8;
this is my EXPLAIN of the select
Now because the query is slow, during 1.5s / 2.0s??
Something wrong?
UPDATE
added new index (with the John Bollinger's answer) to the table ciac_corsi_assenze:
PRIMARY KEY (`id`),
KEY `dataLezione` (`dataLezione`) USING BTREE,
KEY `test` (`idUtente`,`idServizio`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=574582 DEFAULT CHARSET=utf8;
added new index to the table ciac_serviziAcquistati_ITA:
PRIMARY KEY (`id`),
KEY `idAzienda` (`idAzienda`) USING BTREE,
KEY `test2` (`idContatto`,`idServizio`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=9542 DEFAULT CHARSET=utf8;
New EXPLAIN:
But it's always slow :(
Your tables have separate indexes on various columns of interest, but MySQL will use at most one index per table to perform your query. This particular query would probably be sped by table ciac_corsi_assenze having an index on (idUtente, idServizio) (and such an index would supersede the existing one on (idUtente) alone). That should allow MySQL to avoid sorting the result rows to perform the grouping, and it will help more in performing the join than any of the existing indexes do.
The query would probably be sped further by table ciac_serviziAcquistati_ITA having an index on (idContatto, idServizio), or even on (idContatto, idServizio, ritirato). Either of those would supersede the existing index on just (idContatto).
John went the right direction. However the order of columns in the composite index needs changing.
For the GROUP BY, this order is needed (on ASSNZ):
INDEX(idServizio, idUtente)
(and that should replace KEY(idServizio), but not KEY(idUtente))
Then ACQ needs, in this order:
INDEX(idContatto, idServizio, stato_allievo)
replacing only KEY(idContatto).

MySQL use separate indices for JOIN and GROUP BY

I am trying to execute following query
SELECT
a.sessionID AS `sessionID`,
firstSeen, birthday, gender,
isAnonymous, LanguageCode
FROM transactions AS trx
INNER JOIN actions AS a ON a.sessionID = trx.SessionID
WHERE a.ActionType = 'PURCHASE'
GROUP BY trx.TransactionNumber
Explain provides the following output
1 SIMPLE trx ALL TransactionNumber,SessionID NULL NULL NULL 225036 Using temporary; Using filesort
1 SIMPLE a ref sessionID sessionID 98 infinitiExport.trx.SessionID 1 Using index
The problem is that I am trying to use one field for join and different field for GROUP BY.
How can I tell MySQL to use different indices for same table?
CREATE TABLE `transactions` (
`SessionID` varchar(32) NOT NULL DEFAULT '',
`date` datetime DEFAULT NULL,
`TransactionNumber` varchar(32) NOT NULL DEFAULT '',
`CustomerECommerceTrackID` int(11) DEFAULT NULL,
`SKU` varchar(45) DEFAULT NULL,
`AmountPaid` double DEFAULT NULL,
`Currency` varchar(10) DEFAULT NULL,
`Quantity` int(11) DEFAULT NULL,
`Name` tinytext NOT NULL,
`Category` varchar(45) NOT NULL DEFAULT '',
`customerInfoXML` text,
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
PRIMARY KEY (`id`),
KEY `TransactionNumber` (`TransactionNumber`),
KEY `SessionID` (`SessionID`)
) ENGINE=InnoDB AUTO_INCREMENT=212007 DEFAULT CHARSET=utf8;
CREATE TABLE `actions` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`sessionActionDate` datetime DEFAULT NULL,
`actionURL` varchar(255) DEFAULT NULL,
`sessionID` varchar(32) NOT NULL DEFAULT '',
`ActionType` varchar(64) DEFAULT NULL,
`CustomerID` int(11) DEFAULT NULL,
`IPAddressID` int(11) DEFAULT NULL,
`CustomerDeviceID` int(11) DEFAULT NULL,
`customerInfoXML` text,
PRIMARY KEY (`id`),
KEY `ActionType` (`ActionType`),
KEY `CustomerDeviceID` (`CustomerDeviceID`),
KEY `sessionID` (`sessionID`)
) ENGINE=InnoDB AUTO_INCREMENT=15042833 DEFAULT CHARSET=utf8;
Thanks
EDIT 1: My indexes were broken, I had to add (SessionID, TransactionNumber) index to transactions table, however now, when I try to include trx.customerInfoXML table mysql stops using index
EDIT 2 Another answer does not really solved my problem because it's not standard sql syntax and generally not a good idea to force indices.
For ORM users such syntax is a unattainable luxury.
EDIT 3 I updated my indices and it solved the problem, see EDIT 1

MySQL: Optimizing JOINs to find non-matching records

We have a host management system (let's call it CMDB), and a DNS system, each using different tables. The former syncs to the latter, but manual changes cause them to get out of sync. I would like to craft a query to find aliases in CMDB that do NOT have a matching entry in DNS (either no entry, or the name/IP is different)
Because of the large size of the tables, and the need for this query to run frequently, optimizing the query is very important.
Here's what the tables look like:
cmdb_record: id, ipaddr
cmdb_alias: record_id, host_alias
dns_entry: name, ipaddr
cmdb_alias.record_id is a foreign key from cmdb_record.id, so that one IP address can have multiple aliases.
So far, here's what I've come up with:
SELECT cmdb_alias.host_alias, cmdb_record.ipaddr
FROM cmdb_record
INNER JOIN cmdb_alias ON cmdb_alias.record_id = cmdb_record.id
LEFT JOIN dns_entry
ON dns_entry.ipaddr = cmdb_record.ipaddr
AND dns_entry.name = cmdb_alias.host_alias
WHERE dns_entry.ipaddr IS NULL OR dns_entry.name IS NULL
This seems to work, but takes a very long time to run. Is there a better way to do this? Thanks!
EDIT: As requested, here are the SHOW CREATE TABLEs. There are lots of extra fields that aren't particularly relevant, but included for completeness.
Create Table: CREATE TABLE `cmdb_record` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`ip_version` int(11) DEFAULT NULL,
`ipaddr` varchar(40) DEFAULT NULL,
`ipaddr_numeric` decimal(40,0) DEFAULT NULL,
`block_id` int(11) NOT NULL,
`record_commented` tinyint(1) NOT NULL,
`mod_time` datetime NOT NULL,
`deleted` tinyint(1) NOT NULL,
`deleted_date` datetime DEFAULT NULL,
`record_owner` varchar(50) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `ipaddr` (`ipaddr`),
KEY `cmdb_record_fe30f0f7` (`ipaddr`),
KEY `cmdb_record_2b8b575` (`ipaddr_numeric`),
KEY `cmdb_record_45897ef2` (`block_id`),
CONSTRAINT `block_id_refs_id_ed6ed320` FOREIGN KEY (`block_id`) REFERENCES `cmdb_block` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=104427 DEFAULT CHARSET=latin1
Create Table: CREATE TABLE `cmdb_alias` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`host_alias` varchar(255) COLLATE latin1_general_cs NOT NULL,
`record_id` int(11) NOT NULL,
`record_order` int(11) NOT NULL,
PRIMARY KEY (`id`),
KEY `cmdb_alias_fcffc3bb` (`record_id`),
KEY `alias_lookup` (`host_alias`),
CONSTRAINT `record_id_refs_id_8169fc71` FOREIGN KEY (`record_id`) REFERENCES `cmdb_record` (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=155433 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs
Create Table: CREATE TABLE `dns_entry` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`rec_grp_id` varchar(40) NOT NULL,
`parent_id` int(11) NOT NULL,
`domain_id` int(11) DEFAULT NULL,
`name` varchar(255) DEFAULT NULL,
`type` varchar(6) DEFAULT NULL,
`ipaddr` varchar(255) DEFAULT NULL,
`ttl` int(11) DEFAULT NULL,
`prio` int(11) DEFAULT NULL,
`status` varchar(20) NOT NULL,
`op` varchar(20) NOT NULL,
`mod_time` datetime NOT NULL,
`whodunit` varchar(50) NOT NULL,
`comments` longtext NOT NULL,
PRIMARY KEY (`id`),
KEY `dns_entry_a2431ea` (`domain_id`),
KEY `dns_entry_52094d6e` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=49437 DEFAULT CHARSET=utf8
If you don't have one already, create an index on dns_entry(ipaddr, name). This might be all you need to speed the query.