MySQL Query Optimization for below Query - mysql

Below are the Table Schema and Data in EVENT_DETAILS is 1.76 Million Records
CREATE TABLE EVENT_DETAILS
(
ORG_ID CHAR(32) NOT NULL,
CONFIG_KEY CHAR(32) NOT NULL,
COMPONENT_NAME VARCHAR(512) NOT NULL,
REQUEST_ID CHAR(32) NOT NULL,
UPDATE_DATE INT NOT NULL,
UPDATE_TIME INT NOT NULL,
CLIENT CHAR(16) NOT NULL,
STATUS INT NOT NULL,
EVENT_DATA VARCHAR(1024),
MESSAGE VARCHAR(512),
INSERTED_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
);
CREATE INDEX MYINDEX1 ON EVENT_DETAILS (ORG_ID,CONFIG_KEY,COMPONENT_NAME,UPDATE_DATE);
CREATE INDEX MYINDEX2 ON EVENT_DETAILS (UPDATE_DATE,COMPONENT_NAME);
CREATE TABLE DOMAIN_KEYS (
DOMAIN_KEY char(32) COLLATE utf8_unicode_ci NOT NULL,
DOMAIN_IDENTITY varchar(256) COLLATE utf8_unicode_ci NOT NULL,
DOMAIN_DISPLAY_NAME varchar(256) COLLATE utf8_unicode_ci NOT NULL,
ORG_ID char(32) COLLATE utf8_unicode_ci NOT NULL,
LOCATION_KEY int(11) DEFAULT '0',
DESCRIPTION varchar(4000) COLLATE utf8_unicode_ci DEFAULT NULL,
CATEGORY int(11) NOT NULL DEFAULT '1',
ENVIRONMENT_TYPE int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (DOMAIN_KEY)
);
CREATE TABLE ORGANISATIONS (
ORG_ID char(32) NOT NULL,
ORG_IDENTITY varchar(256) NOT NULL,
ORG_NAME varchar(256) NOT NULL,
ORG_DESC varchar(256) DEFAULT NULL,
PARENT_ORG_ID char(32) NOT NULL,
PRIMARY KEY (ORG_ID)
);
Below is the Query and on average it takes around 90 Seconds to fetch the Required data and Optimizer is not using any Index I created, Data is fetched for last 5 days data which is around 40% of complete data. Please suggest some Optimizations and any Indexes if required
SELECT ED.ORG_ID,
ORG.ORG_IDENTITY,
ORG.ORG_NAME,
ED.CONFIG_KEY,
REPLACE(REPLACE(ED.COMPONENT_NAME,'-CURRENT',''),'-HISTORICAL','') AS COMP_NAME,
DO.DOMAIN_IDENTITY,
DO.DOMAIN_DISPLAY_NAME,
GROUP_CONCAT(ED.STATUS ORDER BY ED.INSERTED_TIME DESC) AS ALL_STATUS
FROM EVENT_DETAILS ED ,
ORGANISATIONS ORG ,
DOMAIN_KEYS DO
WHERE UPDATE_DATE > '20191204'
AND ORG.ORG_ID = ED.ORG_ID
AND DO.DOMAIN_KEY=ED.CONFIG_KEY
AND DO.LOCATION_KEY <> 0
GROUP BY ED.ORG_ID,
ORG.ORG_IDENTITY,
ORG.ORG_NAME,
ED.CONFIG_KEY,
DO.DOMAIN_IDENTITY,
DO.DOMAIN_DISPLAY_NAME,
COMP_NAME
HAVING ALL_STATUS LIKE '0,0,0%';

Related

mysql count each user's number of likes ,number of tweets and number of following and followers account

I'm building a twitter-like app using node.js for fun and I have multiple tables :
users: to store users' data.
tweets: to store tweets.
likes: to store what users liked what tweet.
retweets: to store what users retweeted what tweet.
following: to store what user is following other users.
CREATE TABLE IF NOT EXISTS `following` (
`user_id` varchar(50) NOT NULL,
`followed_id` varchar(50) NOT NULL,
`date_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`,`followed_id`)
)
CREATE TABLE IF NOT EXISTS `likes` (
`user_id` varchar(50) NOT NULL,
`tweet_id` varchar(50) NOT NULL,
`date_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`,`tweet_id`)
)
CREATE TABLE IF NOT EXISTS `retweets` (
`user_id` varchar(50) NOT NULL,
`tweet_id` varchar(50) NOT NULL,
`date_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`,`tweet_id`)
)
CREATE TABLE IF NOT EXISTS `tweets` (
`tweet_id` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
`created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`reply_to_tweet_id` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
`reply_to_user_id` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
`truncated` tinyint(1) NOT NULL,
`author` varchar(30) NOT NULL,
`text` varchar(255) NOT NULL,
`media` varchar(255) NOT NULL,
`entities` json NOT NULL,
PRIMARY KEY (`tweet_id`)
)
CREATE TABLE IF NOT EXISTS `users` (
`user_id` varchar(30) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
`email` varchar(30) NOT NULL,
`username` varchar(30) NOT NULL,
`password` varchar(255) NOT NULL,
`handelname` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
`account_url` varchar(255) NOT NULL,
`bio` varchar(255) NOT NULL,
`profile_pic` varchar(255) NOT NULL,
`cover_pic` varchar(255) NOT NULL,
`protected` tinyint(1) NOT NULL,
`created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`),
UNIQUE KEY `email` (`email`),
UNIQUE KEY `username` (`username`)
)
and I want to create a new view for the users containing there information along with how many tweets
they did like, the number of users they follow, the number of users follow them and how many tweets they have created.
I want this information in one table but I tried many ways and none of them worked fine for me!
what is the proper solution for such a problem?
You may try below query -
SELECT `user_id`
,LI.LI_CNT no_of_tweets_liked
,FL.FL_CNT no_of_followers
,FL2.FL2_CNT no_of_followee
,RT.RT_CNT no_of_retweets
FROM `users` U
JOIN (SELECT `user_id`, COUNT(`tweet_id`) LI_CNT
FROM `likes`
GROUP BY `user_id`) LI ON U.`user_id` = LI.`user_id`
JOIN (SELECT `user_id`, COUNT(`followed_id`) FL_CNT
FROM `following`
GROUP BY `user_id`) FL ON U.`user_id` = FL.`user_id`
JOIN (SELECT `followed_id`, COUNT(`followed_id`) FL2_CNT
FROM `following`
GROUP BY `followed_id`) FL2 ON U.`user_id` = FL2.`followed_id`
JOIN (SELECT `user_id`, COUNT(`tweet_id`) RT_CNT
FROM `retweets`
GROUP BY `user_id`) RT ON U.`user_id` = RT.`user_id`;

Getting matches from multiple tables in MySQL

How can i return the matches of two tables.
Table 1:
CREATE TABLE `lost` (
`id` int(11) NOT NULL,
`firstName` int(100) NOT NULL,
`lastName` varchar(100) NOT NULL,
`country` varchar(2) NOT NULL,
`address` varchar(100) NOT NULL,
`email` varchar(120) NOT NULL,
`color` varchar(32) NOT NULL,
`location` varchar(100) NOT NULL,
`airport` int(11) NOT NULL,
`dateReported` date NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
Table 2:
CREATE TABLE `found` (
`id` int(11) NOT NULL,
`firstName` int(100) NOT NULL,
`lastName` varchar(100) NOT NULL,
`country` varchar(2) NOT NULL,
`address` varchar(100) NOT NULL,
`email` varchar(120) NOT NULL,
`color` varchar(32) NOT NULL,
`location` varchar(100) NOT NULL,
`airport` int(11) NOT NULL,
`dateReported` date NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
Now i want to create a query which selects all fields which have any matches on both tables.
And if possible another row which says how many matching fields it has per row.
Not sure I understood, is this what you want?
SELECT t.*,s.*,
(t.firstName = s.firstName ) +
(t.lastName = s.lastName) +
(t.country = s.country) +
(t.address = s.address) +
..... as how_many_matches
FROM `lost` t
JOIN `found` s
ON(t.id = s.id)
MySQL evaluates Boolean expressions as 1 for TRUE and 0 for FALSE , so can you just sum up the comparisons of the columns.
Note that both of your tables are exactly the same, this is not a recommended design. I advise you to add another column TYPE that will save lost / found and combine the two tables .

how to calculate the number of a table's rows changed(update or insert) in a day on mysql

I have a trade records table and i want to calculate the rows of this table changed including update or insert.
I have check the information_schema in mysql.There is a table called TABLE and it has a column TABLE_ROWS.But i didn't find anything helpful.
well,this table like this
CREATE TABLE pre_actived_apk_record
(
id INT(11) PRIMARY KEY NOT NULL AUTO_INCREMENT,
appid VARCHAR(255) NOT NULL,
channelid VARCHAR(255) NOT NULL,
uid INT(11) NOT NULL,
appname VARCHAR(255) NOT NULL,
packagename VARCHAR(255) NOT NULL,
versioncode VARCHAR(255) NOT NULL,
versionname VARCHAR(255) NOT NULL,
brand VARCHAR(255) NOT NULL,
model VARCHAR(255) NOT NULL,
imei VARCHAR(255) NOT NULL,
sys_version VARCHAR(255) NOT NULL,
mac_address VARCHAR(255) NOT NULL,
imsi VARCHAR(255) NOT NULL,
iccid VARCHAR(255) NOT NULL,
uploadtime INT(11) NOT NULL,
dateline INT(11) NOT NULL,
rule INT(11) NOT NULL,
price INT(11) NOT NULL
);
And i wonder i could use some built-in method to show the result of this table chanaged.

making queries run faster

ive run into a bit of a snag here, it seems that my pages are loading slower and slower the most content is being added to the database.
i have already added indexes on the ID and ID_* fields, that fixed the problem a while ago, but is not working anymore.
i've also run OPTIMIZE on all tables.
here is my query:
select
l.id, l.id_infusionsoft, l.name_first, l.name_last, l.postcode, a.website as a_website, l.website as l_website, l.date_added,
a.id_dealership, a.id_lead, a.id as id_assign, a.date_assigned, a.manual_or_auto, a.assigned_by,
d.name as dealership,
COALESCE(a.date_assigned, l.date_added) AS date_sort
from `leads` as l
left join `assignments` as a on (a.id_lead = l.id)
left join `dealerships` as d on (d.id = a.id_dealership)
order by date_sort desc
here are the table structures:
CREATE TABLE assignments (
id int(11) NOT NULL auto_increment,
id_dealership int(11) NOT NULL,
id_lead int(11) NOT NULL,
date_assigned int(11) NOT NULL,
website varchar(255) NOT NULL default '',
make varchar(255) NOT NULL default '',
model varchar(255) NOT NULL default '',
ip_address varchar(255) NOT NULL default '',
is_reassign varchar(255) NOT NULL default 'no',
manual_or_auto varchar(255) NOT NULL default 'N/A',
assigned_by varchar(255) NOT NULL default 'N/A',
PRIMARY KEY (id),
KEY id_dealership (id_dealership),
KEY id_lead (id_lead),
KEY date_assigned (date_assigned)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=24569 ;
CREATE TABLE dealerships (
id int(11) NOT NULL auto_increment,
province varchar(255) NOT NULL default '',
city varchar(255) NOT NULL default '',
`name` varchar(255) NOT NULL,
email text NOT NULL,
email_subject varchar(255) NOT NULL default 'Car Lead',
`type` varchar(255) NOT NULL,
make varchar(255) NOT NULL,
leads int(11) NOT NULL default '0',
`status` varchar(255) NOT NULL,
low_notif int(11) NOT NULL,
reassign_id int(11) NOT NULL,
reassign_days int(11) NOT NULL,
salesman varchar(255) NOT NULL,
PRIMARY KEY (id)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=176 ;
CREATE TABLE leads (
id int(11) NOT NULL auto_increment,
id_infusionsoft int(11) NOT NULL,
name_first varchar(255) NOT NULL,
name_last varchar(255) NOT NULL,
email varchar(255) NOT NULL,
phone_home varchar(255) NOT NULL,
phone_cell varchar(255) NOT NULL,
phone_work varchar(255) NOT NULL,
postcode varchar(255) NOT NULL,
website varchar(255) NOT NULL,
address varchar(255) NOT NULL,
province varchar(255) NOT NULL,
employer varchar(255) NOT NULL,
city varchar(255) NOT NULL,
rentorown varchar(255) NOT NULL,
emp_months varchar(255) NOT NULL,
emp_years varchar(255) NOT NULL,
sin varchar(255) NOT NULL,
occupation varchar(255) NOT NULL,
monthly_income varchar(255) NOT NULL,
bankruptcy varchar(255) NOT NULL,
tradein varchar(255) NOT NULL,
cosign varchar(255) NOT NULL,
monthly_payment varchar(255) NOT NULL,
residence_years varchar(255) NOT NULL,
residence_months varchar(255) NOT NULL,
birthday varchar(255) NOT NULL,
make varchar(255) NOT NULL,
model varchar(255) NOT NULL,
date_added int(11) NOT NULL,
ip_address varchar(255) NOT NULL default '',
time_to_call varchar(255) NOT NULL,
referrer varchar(255) NOT NULL default '',
`source` varchar(255) NOT NULL,
PRIMARY KEY (id),
KEY id_infusionsoft (id_infusionsoft),
KEY date_added (date_added)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=20905 ;
what can i do to make my query run faster?
EDIT: here is the explain select output:
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE l ALL NULL NULL NULL NULL 15381 Using temporary; Using filesort
1 SIMPLE a ref id_lead id_lead 4 infu.l.id 1
1 SIMPLE d eq_ref PRIMARY PRIMARY 4 infu.a.id_dealership 1
Use InnoDB instead of MyISAM. InnoDB caches both data and indexes, whereas MyISAM only caches indexes. InnoDB has other benefits too.
Do you really need all 15381 leads to be returned by this query? Or are you trying to query only a subset of leads, such as those that have assignments and dealerships?
Make sure you understand the type of join you're using. I suspect you're using left join when you need an inner join. It's probably causing the result set to be larger than it needs to be. Adding overhead for sorting, memory use, etc.
Choose data types more appropriately. Does every string have to be varchar(255)? Are you aware that varchars pad out to their maximum length in memory? You're even using varchar(255) to store an IP address. Also, you're using a signed integer to store dates?
Change the following index:
ALTER TABLE assignments ADD KEY assgn_lead_dealership (id_lead, id_dealership),
DROP KEY id_lead;

n-number of nested inner join

I have a sql-statement which correctly fetches data from one table. However, I need to fetch in the same way from n number of tables named table_n. All of these tables contain a 3-field primary key of projid, docid and revnr. I need to return n as docType as well, to differentiate the tables. The result will be sorted by projid and/or docid.
I tried sorting all the outputs from the different queries in PHP but it was way too slow (at least a few seconds on a 3MB database). I'm convinced MySQL/MSSQL will do it faster.
This is my current query:
SELECT a.* FROM `table_1` a
INNER JOIN (SELECT docid,
Max(revnr) max_val
FROM `table_1`
WHERE ( projid = something )
GROUP BY docid) b
ON a.docid = b.docid
AND a.revnr = b.max_val ORDER BY docid DESC
My current query gets the rows with highest revnr for each docid and projid.
I'm developing on MySQL but I need it to work on MSSQL as well. A general SQL solution would be great.
Thanks!
EDIT: Table schemas of the tables i currently have:
CREATE TABLE IF NOT EXISTS `table_1` (
`projid` int(11) NOT NULL,
`docid` int(11) NOT NULL,
`revnr` int(11) NOT NULL,
`revname` varchar(64) NOT NULL,
`signedOn` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`sign` int(11) NOT NULL,
`ritningsnr` varchar(128) NOT NULL,
`moment` varchar(256) NOT NULL,
`omrade` varchar(256) NOT NULL,
`start` datetime NOT NULL,
`stop` datetime NOT NULL,
`extTodo` int(11) NOT NULL,
PRIMARY KEY (`projid`,`docid`,`revnr`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='egenkontroll';
CREATE TABLE IF NOT EXISTS `table_2` (
`projid` int(11) NOT NULL,
`docid` int(11) NOT NULL,
`revnr` int(11) NOT NULL,
`revname` varchar(64) NOT NULL,
`signedOn` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`sign` int(11) NOT NULL,
`extWater` int(11) NOT NULL,
`extRisk` int(11) NOT NULL,
`extSystem` int(11) NOT NULL,
`extHelp` int(11) NOT NULL,
`extProvtryck` int(11) NOT NULL,
`extDoc` int(11) NOT NULL,
`extEgenkontroll` int(11) NOT NULL COMMENT 'exttabell',
`extOther` int(11) NOT NULL,
`extMontorer` int(11) NOT NULL,
PRIMARY KEY (`projid`,`docid`,`revnr`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='arbetsberedning';
CREATE TABLE IF NOT EXISTS `table_3` (
`projid` int(11) NOT NULL,
`docid` int(11) NOT NULL,
`revnr` int(11) NOT NULL,
`revname` varchar(64) NOT NULL,
`adress` varchar(256) NOT NULL,
`pipesMark` tinyint(1) NOT NULL,
`pipesKulvert` tinyint(1) NOT NULL,
`pipesBasement` tinyint(1) NOT NULL,
`pipesVaning` tinyint(1) NOT NULL,
`pipesIngjutna` tinyint(1) NOT NULL,
`ledningTappvatten` tinyint(1) NOT NULL,
`ledningVarmevatten` tinyint(1) NOT NULL,
`ledningHetvatten` tinyint(1) NOT NULL,
`ledningKylaPrim` tinyint(1) NOT NULL,
`ledningKylaSek` tinyint(1) NOT NULL,
`ledningGas` tinyint(1) NOT NULL,
`ledningLuft` tinyint(1) NOT NULL,
`ledningAvlopp` tinyint(1) NOT NULL,
`ledningOther` varchar(512) NOT NULL,
`materialGjutjarn` tinyint(1) NOT NULL,
`materialSteel` tinyint(1) NOT NULL,
`materialKoppar` tinyint(1) NOT NULL,
`materialPlast` tinyint(1) NOT NULL,
`materialRostfritt` tinyint(1) NOT NULL,
`materialOther` varchar(512) NOT NULL,
`omfattningLength` int(11) NOT NULL COMMENT 'meter',
`omfattningDimension` varchar(16) NOT NULL,
`omfattningRitningnr` varchar(128) NOT NULL,
`doneWithPump` tinyint(1) NOT NULL,
`doneWithVattenledning` tinyint(1) NOT NULL,
`doneWithKompressor` tinyint(1) NOT NULL,
`doneWithTathetsprovare` tinyint(1) NOT NULL,
`tryckmedieVatten` tinyint(1) NOT NULL,
`tryckmedieLuft` tinyint(1) NOT NULL,
`tryckmedieOther` varchar(128) NOT NULL,
`manometerDiameter` int(11) NOT NULL COMMENT 'mm',
`manometerGradering` int(11) NOT NULL COMMENT 'kPa',
`manometerReadPressure` int(11) NOT NULL,
`manometerTid` int(11) NOT NULL COMMENT 'sekunder',
`testedOn` datetime NOT NULL,
`testedBy` varchar(128) NOT NULL COMMENT '"id_" + (userid)',
`comments` varchar(1024) NOT NULL,
`commentsBy` varchar(128) NOT NULL COMMENT '"id_" + (userid)',
`signedOn` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`sign` int(11) NOT NULL,
PRIMARY KEY (`projid`,`docid`,`revnr`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
The fields I need are projid, docid, revnr, revname, signedOn, sign and are present in all current and future tables.