Updating a stored JSON in Mysql Database - mysql

I am trying to update a Stored JSON in my database and i am unable to run the following update query. I have copied below a select statement and an update statement
SELECT
`core_animal_event`. `animal_id` AS `MilkingEvent_animalID`,
JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."62"')) AS `MilkingEvent_milkCompositeLitres`,
coalesce(JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."68"')) +
JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."61"')) +
JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."59"')))
FROM `core_animal_event` WHERE (`core_animal_event`. `event_type` = 2) AND (`core_animal_event`. `country_id` = '10');
UPDATE `core_animal`
SET
JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."62"')) =
coalesce(JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."68"')) +
JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."61"')) +
JSON_UNQUOTE(JSON_EXTRACT(`core_animal_event`.`additional_attributes`, '$."59"')))
WHERE (`core_animal_event`. `event_type` = 2) AND (`core_animal_event`. `country_id` = '10')
The following is the sample data from which we have a stored JSON column called additional_attribute and animal_id which is unique
# animal_id, additional_attributes
'2576', '{\"59\": null, \"61\": null, \"62\": null, \"63\": null, \"64\": null, \"65\": null, \"66\": null, \"67\": null, \"68\": null, \"69\": \"1\", \"70\": \"2\", \"71\": \"1\", \"72\": null, \"73\": \"2\", \"74\": \"1\", \"75\": null, \"76\": null, \"77\": [\"1\"], \"78\": \"32\", \"79\": \"70\", \"80\": \"4\", \"81\": null, \"82\": null, \"83\": null, \"84\": \"Mkiwa\", \"85\": \"19280\", \"86\": \"2405\", \"87\": \"TNZ000192802405\", \"88\": \"Brownwhite\", \"89\": \"1565789020239.jpg\", \"90\": \"1565789049469.jpg\", \"96\": null, \"97\": null, \"98\": null, \"99\": \"1\", \"100\": null, \"101\": null, \"102\": null, \"103\": null, \"104\": null, \"105\": null, \"106\": null, \"107\": null, \"108\": null, \"109\": null, \"110\": null, \"111\": null, \"112\": null, \"113\": null, \"114\": null, \"115\": null, \"116\": null, \"117\": null, \"118\": null, \"119\": null, \"120\": null, \"121\": null, \"122\": null, \"123\": null, \"124\": null, \"125\": null, \"126\": null, \"127\": null, \"128\": null, \"129\": null, \"130\": null, \"131\": null, \"132\": null, \"133\": null, \"134\": null, \"135\": null, \"136\": null, \"137\": null, \"138\": null, \"139\": null, \"141\": null, \"142\": null, \"143\": null, \"144\": null, \"145\": null}'
The following is an example of a create statement
CREATE TABLE `core_animal_event` (
`id` int NOT NULL AUTO_INCREMENT,
`animal_id` int NOT NULL,
`event_type` int NOT NULL,
`additional_attributes` json DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `org_id` (`country_id`),
KEY `animal_id` (`animal_id`),
KEY `event_type` (`event_type`),
CONSTRAINT `core_animal_event_ibfk_1` FOREIGN KEY (`animal_id`) REFERENCES `core_animal` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB AUTO_INCREMENT=941817 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci ROW_FORMAT=COMPACT;

Escape characters are not necessary here. I even think they may be the source of the problem.
Here's my code for the tests :
CREATE TABLE `core_animal_event` (
`animal_id` int(11) NOT NULL AUTO_INCREMENT,
`additional_attributes` json DEFAULT NULL,
`event_type` int(11) NOT NULL,
`country_id` int(11) NOT NULL,
PRIMARY KEY (`animal_id`)
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1
INSERT INTO core_animal_event
(animal_id, additional_attributes, event_type, country_id)
VALUES(1, '{ "59": null, "61": null, "62": null, "63": null, "64": null, "65": null, "66": null, "67": null, "68": null, "69": "1", "70": "2", "71": "1", "72": null, "73": "2", "74": "1", "75": null, "76": null, "77": ["1"], "78": "32", "79": "70", "80": "4", "81": null, "82": null, "83": null, "84": "Mkiwa", "85": "19280", "86": "2405", "87": "TNZ000192802405", "88": "Brownwhite", "89": "1565789020239.jpg", "90": "1565789049469.jpg", "96": null, "97": null, "98": null, "99": "1", "100": null, "101": null, "102": null, "103": null, "104": null, "105": null, "106": null, "107": null, "108": null, "109": null, "110": null, "111": null, "112": null, "113": null, "114": null, "115": null, "116": null, "117": null, "118": null, "119": null, "120": null, "121": null, "122": null, "123": null, "124": null, "125": null, "126": null, "127": null, "128": null, "129": null, "130": null, "131": null, "132": null, "133": null, "134": null, "135": null, "136": null, "137": null, "138": null, "139": null, "141": null, "142": null, "143": null, "144": null, "145": null}', 2, 10);
Without escape characters \, your SELECT query work.
For your update, here's an example :
UPDATE core_animal_event
SET additional_attributes = json_set(additional_attributes, '$."62"',
COALESCE(JSON_UNQUOTE(JSON_EXTRACT(additional_attributes, '$."68"')) +
JSON_UNQUOTE(JSON_EXTRACT(additional_attributes, '$."61"')) +
JSON_UNQUOTE(JSON_EXTRACT(additional_attributes, '$."59"')))
)
WHERE (event_type = 2) AND (country_id = '10');
//-------
EDIT:
Be careful when using JSON_EXTRACT and COALESCE function, in case all values are null, the returned value is 0, not a null value.
EDIT 2:
Akina is right, your COALESCE function is no good... As you do, it's an addition (but maybe that's what you want...)
EDIT 3:
If you want to use COALESCE here's an example:
SELECT animal_id AS colID, JSON_UNQUOTE(JSON_EXTRACT(additional_attributes, '$."62"')) AS col62,
COALESCE (
IF(JSON_TYPE(JSON_EXTRACT(additional_attributes, '$."68"'))='NULL', null, JSON_EXTRACT(additional_attributes, '$."68"')),
IF(JSON_TYPE(JSON_EXTRACT(additional_attributes, '$."59"'))='NULL', null, JSON_EXTRACT(additional_attributes, '$."59"')),
IF(JSON_TYPE(JSON_EXTRACT(additional_attributes, '$."61"'))='NULL', null, JSON_EXTRACT(additional_attributes, '$."61"'))
) AS result_of_coalesce
FROM core_animal_event WHERE (event_type = 2) AND (country_id = '10');

Related

prevent django test runner from creating stale table

I have mariadb database that used to have CHARSET utf8 COLLATE utf8_general_ci config but now CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci. All tables have the same CHARSET and COLLATE as those of the database.
When I run ./manage.py test, stacktrace looks like this:
....
django.db.utils.OperationalError: (1118, 'Row size too large (> 8126). Changing some columns to TEXT or BLOB may help. In current row format, BLOB prefix of 0 bytes is stored inline.')
I managed to find out what the troubling table is, and the sql query looks like the following. Note that I changed names of table and fields for security:
CREATE TABLE `troubling_table`
(
`id` INTEGER auto_increment NOT NULL PRIMARY KEY,
`no_tax` VARCHAR(20) NOT NULL,
`cd_pc` VARCHAR(7) NOT NULL,
`cd_wdept` VARCHAR(12) NOT NULL,
`id_write` VARCHAR(20) NULL,
`cd_docu` VARCHAR(10) NULL,
`dt_acct` VARCHAR(8) NULL,
`st_docu` VARCHAR(3) NULL,
`tp_drcr` VARCHAR(3) NULL,
`cd_acct` VARCHAR(20) NULL,
`amt` NUMERIC(19, 4) NULL,
`cd_partner` VARCHAR(20) NULL,
`nm_partner` VARCHAR(50) NULL,
`tp_job` VARCHAR(40) NULL,
`cls_job` VARCHAR(40) NULL,
`ads_hd` VARCHAR(400) NULL,
`nm_ceo` VARCHAR(40) NULL,
`dt_start` VARCHAR(8) NULL,
`dt_end` VARCHAR(8) NULL,
`am_taxstd` NUMERIC(19, 4) NULL,
`am_addtax` NUMERIC(19, 4) NULL,
`tp_tax` VARCHAR(10) NULL,
`no_company` VARCHAR(20) NULL,
`dts_insert` VARCHAR(20) NULL,
`id_insert` VARCHAR(20) NULL,
`dts_update` VARCHAR(20) NULL,
`id_update` VARCHAR(20) NULL,
`nm_note` VARCHAR(100) NULL,
`cd_bizarea` VARCHAR(12) NULL,
`cd_dept` VARCHAR(12) NULL,
`cd_cc` VARCHAR(12) NULL,
`cd_pjt` VARCHAR(20) NULL,
`cd_fund` VARCHAR(20) NULL,
`cd_budget` VARCHAR(20) NULL,
`no_cash` VARCHAR(20) NULL,
`st_mutual` VARCHAR(3) NULL,
`cd_card` VARCHAR(20) NULL,
`no_deposit` VARCHAR(20) NULL,
`cd_bank` VARCHAR(20) NULL,
`ucd_mng1` VARCHAR(20) NULL,
`ucd_mng2` VARCHAR(20) NULL,
`ucd_mng3` VARCHAR(20) NULL,
`ucd_mng4` VARCHAR(20) NULL,
`ucd_mng5` VARCHAR(20) NULL,
`cd_employ` VARCHAR(20) NULL,
`cd_mng` VARCHAR(20) NULL,
`no_bdocu` VARCHAR(20) NULL,
`no_bdoline` NUMERIC(4, 0) NULL,
`tp_docu` VARCHAR(3) NULL,
`no_acct` NUMERIC(5, 0) NULL,
`tp_trade` VARCHAR(10) NULL,
`no_check` VARCHAR(20) NULL,
`no_check1` VARCHAR(20) NULL,
`cd_exch` VARCHAR(10) NULL,
`rt_exch` NUMERIC(10, 4) NULL,
`cd_trade` VARCHAR(10) NULL,
`no_check2` VARCHAR(50) NULL,
`no_check3` VARCHAR(50) NULL,
`no_check4` VARCHAR(100) NULL,
`tp_cross` VARCHAR(1) NULL,
`erp_cd` VARCHAR(50) NULL,
`am_ex` NUMERIC(19, 4) NULL,
`tp_export` VARCHAR(1) NULL,
`no_to` VARCHAR(20) NULL,
`dt_shipping` VARCHAR(8) NULL,
`tp_gubun` VARCHAR(3) NULL,
`no_invoice` VARCHAR(20) NULL,
`no_item` VARCHAR(20) NULL,
`md_tax1` VARCHAR(4) NULL,
`nm_item1` VARCHAR(50) NULL,
`nm_size1` VARCHAR(20) NULL,
`qt_tax1` NUMERIC(17, 4) NULL,
`am_prc1` NUMERIC(19, 4) NULL,
`am_supply1` NUMERIC(19, 4) NULL,
`am_tax1` NUMERIC(19, 4) NULL,
`nm_note1` VARCHAR(20) NULL,
`cd_bizplan` VARCHAR(20) NULL,
`cd_bgacct` VARCHAR(10) NULL,
`cd_mngd1` VARCHAR(20) NULL,
`nm_mngd1` VARCHAR(100) NULL,
`cd_mngd2` VARCHAR(20) NULL,
`nm_mngd2` VARCHAR(100) NULL,
`cd_mngd3` VARCHAR(20) NULL,
`nm_mngd3` VARCHAR(100) NULL,
`cd_mngd4` VARCHAR(20) NULL,
`nm_mngd4` VARCHAR(100) NULL,
`cd_mngd5` VARCHAR(20) NULL,
`nm_mngd5` VARCHAR(100) NULL,
`cd_mngd6` VARCHAR(20) NULL,
`nm_mngd6` VARCHAR(100) NULL,
`cd_mngd7` VARCHAR(20) NULL,
`nm_mngd7` VARCHAR(100) NULL,
`cd_mngd8` VARCHAR(20) NULL,
`nm_mngd8` VARCHAR(100) NULL,
`yn_iss` VARCHAR(1) NULL,
`final_status` VARCHAR(2) NULL,
`no_bill` VARCHAR(24) NULL,
`tp_bill` VARCHAR(1) NULL,
`tp_record` VARCHAR(1) NULL,
`tp_etcacct` VARCHAR(1) NULL,
`st_gware` VARCHAR(3) NULL,
`sell_dam_nm` VARCHAR(30) NULL,
`sell_dam_email` VARCHAR(50) NULL,
`sell_dam_mobil` VARCHAR(20) NULL,
`nm_pumm` VARCHAR(100) NULL,
`jeonjasend15_yn` VARCHAR(1) NULL,
`dt_write` VARCHAR(8) NULL,
`st_tax` VARCHAR(1) NULL,
`md_tax2` VARCHAR(4) NULL,
`nm_item2` VARCHAR(50) NULL,
`nm_size2` VARCHAR(20) NULL,
`qt_tax2` NUMERIC(17, 4) NULL,
`am_prc2` NUMERIC(19, 4) NULL,
`am_supply2` NUMERIC(19, 4) NULL,
`am_tax2` NUMERIC(19, 4) NULL,
`nm_note2` VARCHAR(20) NULL,
`md_tax3` VARCHAR(4) NULL,
`nm_item3` VARCHAR(50) NULL,
`nm_size3` VARCHAR(20) NULL,
`qt_tax3` NUMERIC(17, 4) NULL,
`am_prc3` NUMERIC(19, 4) NULL,
`am_supply3` NUMERIC(19, 4) NULL,
`am_tax3` NUMERIC(19, 4) NULL,
`nm_note3` VARCHAR(20) NULL,
`md_tax4` VARCHAR(4) NULL,
`nm_item4` VARCHAR(50) NULL,
`nm_size4` VARCHAR(20) NULL,
`qt_tax4` NUMERIC(17, 4) NULL,
`am_prc4` NUMERIC(19, 4) NULL,
`am_supply4` NUMERIC(19, 4) NULL,
`am_tax4` NUMERIC(19, 4) NULL,
`nm_note4` VARCHAR(20) NULL,
`no_asset` VARCHAR(20) NULL,
`nm_bigo` VARCHAR(100) NULL,
`nm_ptr` VARCHAR(20) NULL,
`ex_hp` VARCHAR(15) NULL,
`ex_emil` VARCHAR(100) NULL,
`no_biztax` VARCHAR(8) NULL,
`yn_import` VARCHAR(1) NULL,
`ref_no_docu` VARCHAR(20) NULL,
`cd_fx` VARCHAR(2) NULL,
`fx_bill` VARCHAR(20) NULL,
`no_iss` VARCHAR(24) NULL,
`file_attach` VARCHAR(100) NULL,
`tp_evidence` VARCHAR(4) NULL,
`st_bizbox` VARCHAR(1) NULL,
`tp_input` VARCHAR(30) NULL,
`sell_dam_tel` VARCHAR(20) NULL,
`no_car` VARCHAR(20) NULL,
`no_carbody` VARCHAR(17) NULL,
`dec_lease` VARCHAR(100) NULL,
`no_tdocu` VARCHAR(20) NULL,
`no_tdoline` NUMERIC(4, 0) NULL,
`cd_bizcar` VARCHAR(20) NULL,
`cd_taxacct` VARCHAR(10) NULL,
`yn_fixasset` VARCHAR(1) NULL
)
So if I run this query in sql editor, the error looks the same as that of django. This error didn't happend when I created database with the now gone pair of characterset and collate. But when I run the test it raises error. Different charset may be one of the reaons.
So I deleted the model and apply migrations, since that model is no longer in use. So, stale.
But even after that model is erased, django test runner still seems to bother to create that stale table.
Does django test runner go through every migraion files from start? Is that why I can't run test against model that used to be created with old charset and collate?
How can I prevent django test runner from creating stale, no longer existed, table because old table conflicts in charset and collate with new table, without changing db charset and collate?

Use correlated subquery over multiple columns

DB-Fiddle
CREATE TABLE logistics (
id int primary key,
campaign VARCHAR(255),
event_type VARCHAR (255),
date_offered VARCHAR (255),
date_ordered VARCHAR (255),
date_delivered VARCHAR (255),
date_recorded VARCHAR (255),
date_completed VARCHAR (255),
quantity_offered VARCHAR(255),
quantity_ordered VARCHAR(255),
quantity_delivered VARCHAR(255),
quantity_recorded VARCHAR(255),
quantity_completed VARCHAR(255)
);
INSERT INTO logistics
(id, campaign, event_type,
date_offered, date_ordered,
date_delivered, date_recorded, date_completed,
quantity_offered, quantity_ordered,
quantity_delivered, quantity_recorded, quantity_completed
)
VALUES
("1", "C001", "offered", "2019-04-10", NULL, NULL, NULL, NULL, "500", NULL, NULL, NULL, NULL),
("2", "C001", "ordered", NULL, "2019-04-16", NULL, NULL, NULL, NULL, "450", NULL, NULL, NULL),
("3", "C001", "stored", NULL, NULL, "2019-04-18", NULL, NULL, NULL, NULL, "465", NULL, NULL),
("4", "C001", "stored", NULL, NULL, NULL, "2019-04-20", NULL, NULL, NULL, NULL, "440", NULL),
("5", "C001", "stored", NULL, NULL, NULL, NULL, "2019-04-22", NULL, NULL, NULL, NULL, "445"),
("6", "C002", "offered", "2019-08-15", NULL, NULL, NULL, NULL, "600", NULL, NULL, NULL, NULL),
("7", "C002", "ordered", NULL, "2019-09-03", NULL, NULL, NULL, NULL, "700", NULL, NULL, NULL),
("8", "C002", "stored", NULL, NULL, "2019-09-05", NULL, NULL, NULL, NULL, "690", NULL, NULL),
("9", "C002", "stored", NULL, NULL, NULL, "2019-09-08", NULL, NULL, NULL, NULL, "692", NULL),
("10", "C003", "offered", "2019-10-24", NULL, NULL, NULL, NULL, "300", NULL, NULL, NULL, NULL),
("11", "C003", "ordered", NULL, "2019-10-28", NULL, NULL, NULL, NULL, "250", NULL, NULL, NULL),
("12", "C003", "stored", NULL, NULL, "2019-10-31", NULL, NULL, NULL, NULL, "320", NULL, NULL),
("13", "C004", "offered", "2019-11-05", NULL, NULL, NULL, NULL, "800", NULL, NULL, NULL, NULL),
("14", "C004", "ordered", NULL, "2019-11-14", NULL, NULL, NULL, NULL, "870", NULL, NULL, NULL),
("15", "C004", "stored", NULL, NULL, "2019-11-16", NULL, NULL, NULL, NULL, "740", NULL, NULL),
("16", "C005", "offered", "2019-12-17", NULL, NULL, NULL, NULL, "240", NULL, NULL, NULL, NULL),
("17", "C005", "ordered", NULL, "2020-01-12", NULL, NULL, NULL, NULL, "250", NULL, NULL, NULL),
("18", "C005", "stored", NULL, NULL, "2020-01-16", NULL, NULL, NULL, NULL, "226", NULL, NULL),
("19", "C006", "offered", "2020-01-09", NULL, NULL, NULL, NULL, "100", NULL, NULL, NULL, NULL),
("20", "C006", "ordered", NULL, "2020-01-23", NULL, NULL, NULL, NULL, "105", NULL, NULL, NULL),
("21", "C007", "offered", "2020-02-17", NULL, NULL, NULL, NULL, "900", NULL, NULL, NULL, NULL),
("22", "C008", "offered", "2020-02-25", NULL, NULL, NULL, NULL, "400", NULL, NULL, NULL, NULL);
The table above displays the purchasing process of different campaigns.
The first three process steps are defined by the event_types offered, ordered and stored.
Once a campaign has reached the process step stored the process continues with different dates called date_completed, date_recorded and date_delivered. Basically, those dates are sub_events from the event_type stored.
The hierarchy of those events is like this:
event_type = stored > ordered > offered
date = date_completed > date_recorded > date_delivered
Now, I want to extract the campaigns based on their highest event_type or - in case they already have reached the event_type stored - based on their highest date according to the defined hierarchy. The result should look like this:
campaign event_type date quantity
C001 stored 2019-04-22 445
C002 stored 2019-09-08 692
C003 stored 2019-10-31 320
C004 stored 2019-11-16 740
C005 stored 2020-01-16 226
C006 ordered 2020-01-23 105
C007 offered 2020-02-17 900
C008 offered 2020-02-25 400
With reference to this question I tried to modify the query like this:
SELECT
campaign,
event_type,
coalesce(date_offered, date_ordered) as main_event_date,
coalesce(date_delivered, date_recorded, date_completed) as sub_event_date,
coalesce(quantity_offered, quantity_ordered) as main_event_quantity,
coalesce(quantity_delivered, quantity_recorded, quantity_completed) as sub_event_quantity
FROM logistics lg
WHERE lg.id = (SELECT lg2.id
FROM logistics lg2
WHERE lg2.campaign = lg.campaign
ORDER BY field(lg2.event_type, 'stored', 'ordered', 'offered')
field(lg2.sub_event_date, 'date_completed', 'date_recorded', 'date_delivered')
LIMIT 1
);
However, I don't know how I should you use the field function once the campaign is in the event_type stored because there is no sub_event column for the dates?
You can use a subquery and group te data by campaign:
SELECT campaign,MAX(event_type) AS event_type,
CASE WHEN MAX(sub_event_date)>MAX(main_event_date) THEN MAX(sub_event_date) ELSE MAX(main_event_date) END AS event_date,
CASE WHEN MAX(sub_event_date)>MAX(main_event_date) THEN MAX(sub_event_quantity) ELSE MAX(main_event_quantity) END AS event_quantity
FROM (
SELECT
campaign,
event_type,
coalesce(date_offered, date_ordered) as main_event_date,
coalesce(date_delivered, date_recorded, date_completed) as sub_event_date,
coalesce(quantity_offered, quantity_ordered) as main_event_quantity,
coalesce(quantity_delivered, quantity_recorded, quantity_completed) as sub_event_quantity
FROM logistics lg) l
GROUP BY campaign
DB-Fiddle
The solution is as the following:
Step 1: Create an aggregated_sub_table called AGR_logistics which adds an additional column called sub_event_type to the original table:
CREATE TABLE AGR_logistics
SELECT
id,
campaign,
event_type,
(CASE
WHEN event_type = "stored" AND quantity_delivered IS NOT NULL THEN "delivered"
WHEN event_type = "stored" AND quantity_recorded IS NOT NULL THEN "recorded"
WHEN event_type = "stored" AND quantity_completed IS NOT NULL THEN "completed"
ELSE NULL END) AS sub_event_type,
date_offered,
date_ordered,
date_delivered,
date_recorded,
date_completed,
quantity_offered,
quantity_ordered,
quantity_delivered,
quantity_recorded,
quantity_completed
FROM logistics;
Step 2: Run a correleated subquery as described here over AGR_logistics:
SELECT
id,
campaign,
event_type,
sub_event_type,
coalesce(date_offered, date_ordered, date_delivered, date_recorded, date_completed) as event_date,
coalesce(quantity_offered, quantity_ordered, quantity_delivered, quantity_recorded, quantity_completed) as event_quantity
FROM AGR_logistics AGR_01
WHERE AGR_01.id = (SELECT AGR_02.id
FROM AGR_logistics AGR_02
WHERE AGR_02.campaign = AGR_01.campaign
ORDER BY
field(AGR_02.event_type, 'stored', 'ordered', 'offered'),
field(AGR_02.sub_event_type, 'completed', 'recorded', 'delivered')
LIMIT 1
);

Simple query very slow due to order by

Please could anyone help with the following query? (180352 rows)
SELECT COUNT(p.stock_id) AS num_products,
p.master_photo, p.product_photo, p.stock_id, p.master, p.title, p.price, p.stock_level, p.on_order, p.location, p.supplier, p.category, p.sub_category, p.reorder
FROM products AS p
WHERE p.sub_category != 'Subscriptions'
GROUP BY p.master
ORDER BY p.stock_id ASC
LIMIT 0, 20
It's running at 6 seconds.
When I remove the order by it run's at 0.0023 seconds.
And also the same when I remove the group by.
The stock_id (unique) and sub_category are indexed.
I can't think of another way to approach a query like this as it is vital that I group by the master to get the count of product variations and also vital that they can be ordered (not necessarily by stock_id but that's the default).
Thank you
As requested by e4c5 below is the result of the explain with the order by
id: 1
select_type: SIMPLE
table: p
type: range
possible_keys: sub_category
key: sub_category
key_len: 52
ref: NULL
rows: 181691
Extra: Using where; Using temporary; Using filesort
and then without the order by
id: 1
select_type: SIMPLE
table: p
type: index
possible_keys: sub_category
key: master
key_len: 52
ref: NULL
rows: 21
Extra: Using where
and then below is the create table
CREATE TABLE IF NOT EXISTS `products` (
`stock_id` varchar(50) NOT NULL,
`conv_stock_id` varchar(100) NOT NULL,
`conv_quantity` decimal(10,2) NOT NULL,
`master` varchar(50) NOT NULL,
`master_photo` varchar(255) NOT NULL,
`free_guide_photo` varchar(255) NOT NULL,
`product_var_photo` varchar(255) NOT NULL,
`master_title` varchar(255) NOT NULL,
`master_slug` varchar(255) NOT NULL,
`master_page_title` varchar(255) NOT NULL,
`product_photo` varchar(255) NOT NULL,
`original_product_photo` varchar(255) NOT NULL,
`title` varchar(255) NOT NULL,
`orig_title` varchar(255) NOT NULL,
`page_title` varchar(255) NOT NULL,
`description` longtext NOT NULL,
`slug` varchar(255) NOT NULL,
`custom_url` varchar(255) NOT NULL,
`location` varchar(255) NOT NULL,
`supplier` varchar(50) NOT NULL,
`supplier_stock_id` varchar(50) NOT NULL,
`supplier_discount` int(11) NOT NULL,
`category` varchar(50) NOT NULL,
`sub_category` varchar(50) NOT NULL,
`cost_price` decimal(10,2) NOT NULL,
`discount_cost_price` decimal(10,2) NOT NULL,
`price` decimal(10,2) NOT NULL,
`sale_price` decimal(10,2) NOT NULL,
`sale_price_startdate` date NOT NULL,
`sale_price_enddate` date NOT NULL,
`orig_price_trail` int(3) NOT NULL,
`price_trail` varchar(50) NOT NULL,
`price_rule` int(1) NOT NULL,
`pack_size` int(11) NOT NULL,
`parcel_size` int(1) NOT NULL,
`packaging_rule` int(11) NOT NULL,
`cut_tear` int(1) NOT NULL,
`oversized_parcel` int(1) NOT NULL,
`print_label` int(1) NOT NULL,
`stock_level` decimal(10,1) NOT NULL,
`stock_level_group` varchar(50) NOT NULL,
`stock_level_increment` decimal(10,2) NOT NULL,
`stock_check_date` datetime NOT NULL,
`reorder` int(1) NOT NULL,
`reorder_level` decimal(10,1) NOT NULL,
`reorder_quantity` decimal(10,1) NOT NULL,
`reorder_attempts` int(1) NOT NULL,
`unit_size` decimal(10,1) NOT NULL,
`on_order` decimal(10,1) NOT NULL,
`date_ordered` datetime NOT NULL,
`back_order` decimal(10,1) NOT NULL,
`uom` decimal(10,1) NOT NULL,
`uom_value` varchar(100) NOT NULL,
`stock_estimate` int(1) NOT NULL,
`due_date` datetime NOT NULL,
`quantity` varchar(255) NOT NULL,
`colour` varchar(255) NOT NULL,
`colour_family` varchar(255) NOT NULL,
`type` varchar(255) NOT NULL,
`style` varchar(255) NOT NULL,
`pattern` varchar(255) NOT NULL,
`shape` varchar(255) NOT NULL,
`design` varchar(255) NOT NULL,
`fibre` varchar(255) NOT NULL,
`material` varchar(255) NOT NULL,
`pattern_for` varchar(255) NOT NULL,
`difficulty` varchar(255) NOT NULL,
`fabric_count` varchar(255) NOT NULL,
`yarn_thickness` varchar(255) NOT NULL,
`suggested_needle_size` varchar(255) NOT NULL,
`tension` varchar(255) NOT NULL,
`collections` varchar(255) NOT NULL,
`product_features` varchar(255) NOT NULL,
`size` varchar(255) NOT NULL,
`actual_size` varchar(255) NOT NULL,
`length` varchar(255) NOT NULL,
`width` varchar(255) NOT NULL,
`weight` varchar(255) NOT NULL,
`weight_gsm` varchar(255) NOT NULL,
`brand` varchar(255) NOT NULL,
`designer` varchar(255) NOT NULL,
`composition` varchar(255) NOT NULL,
`washing_instructions` varchar(255) NOT NULL,
`matching_thread` varchar(50) NOT NULL,
`sample` varchar(50) NOT NULL,
`fat_quarter` varchar(50) NOT NULL,
`barcode` varchar(13) NOT NULL,
`list_international` int(1) NOT NULL,
`token` varchar(50) NOT NULL,
`create_sample` int(1) NOT NULL,
`create_fatquarter` int(1) NOT NULL,
`create_listing_type` int(1) NOT NULL,
`create_listing_size` int(11) NOT NULL,
`create_listing_price` decimal(10,2) NOT NULL,
`create_listing_price_rule` int(11) NOT NULL,
`create_listing_sale_price` decimal(10,2) NOT NULL,
`create_listing_parcelsize` int(1) NOT NULL,
`create_listing_barcode` varchar(13) NOT NULL,
`auto_listing` int(1) NOT NULL,
`custom_bridal` int(1) NOT NULL,
`pickwave_assign` int(1) NOT NULL,
`kit_product` int(11) NOT NULL,
`fatquarter_product` int(1) NOT NULL,
`sample_product` int(1) NOT NULL,
`grouped_product` int(1) NOT NULL,
`grouped_product_quantity` decimal(10,1) NOT NULL,
`multiple_product` int(1) NOT NULL,
`freepost_product` int(1) NOT NULL,
`status` int(1) NOT NULL,
`update_stock_level` int(1) NOT NULL,
`force_product_photo` int(1) NOT NULL,
`created_master_photo` int(1) NOT NULL,
`force_master_photo` int(1) NOT NULL,
`created_free_guide_photo` int(1) NOT NULL,
`force_free_guide_photo` int(1) NOT NULL,
`created_product_var_photo` int(1) NOT NULL,
`force_product_var_photo` int(1) NOT NULL,
`force_additional_photo` int(1) NOT NULL,
`created_price_levelling` int(1) NOT NULL,
`created_grouped_product` int(1) NOT NULL,
`updated_stock_level` int(1) NOT NULL,
`create_multiple_listing` int(1) NOT NULL,
`create_freepost_listing` int(1) NOT NULL,
`create_freeguide_info` int(1) NOT NULL,
`created_by` int(11) NOT NULL,
`date_created` datetime NOT NULL,
UNIQUE KEY `stock_id` (`stock_id`),
KEY `token` (`token`),
KEY `title` (`title`),
KEY `stock_level_group` (`stock_level_group`),
KEY `sub_category` (`sub_category`),
KEY `stock_level` (`stock_level`),
KEY `category` (`category`),
KEY `conv_stock_id` (`conv_stock_id`),
KEY `conv_quantity` (`conv_quantity`),
KEY `created_price_levelling` (`created_price_levelling`),
KEY `master` (`master`),
KEY `colour` (`colour`),
KEY `auto_listing` (`auto_listing`),
KEY `multiple_product` (`multiple_product`),
KEY `status` (`status`),
KEY `ebay_master` (`ebay_master`),
KEY `parcel_size` (`parcel_size`),
KEY `grouped_product` (`grouped_product`),
KEY `sample_product` (`sample_product`),
KEY `fatquarter_product` (`fatquarter_product`),
KEY `created_grouped_product` (`created_grouped_product`),
KEY `price` (`price`),
KEY `freepost_product` (`freepost_product`),
KEY `master_title` (`master_title`),
KEY `c_sub_category_master` (`sub_category`,`master`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
You haven't provided the output from explain, however based on your query it would seem that ORDER BY forces a full table scan. That would make the query very slow.
When you don't use the ORDER BY, the db reads the results for the first 20 master values (there maybe quite a few of them) and groups them together and returns the result.
When you order by stock_id the whole table needs to be looked at to find which masters are associated with the lowest values stock_ids
It maybe possible to improve performance with a composite index on sub_category,master however a conclusion cannot be made unless you share your SHOW CREATE TABLES, EXPLAIN output.
Update
Based on your CREATE TABLE statements, I see that your database isn't normalized. For example Why do I get the feeling that the following columns should in a table of their own?
supplier varchar(50) NOT NULL,
supplier_stock_id varchar(50) NOT NULL,
supplier_discount int(11) NOT NULL,
You should only have a supplier_stock_id in your products table (foreign key to the suppliers table). There are similar sets of columns which really should be moved out.
When you do so you can create leaner and meaner indexes on this table. But that's not all the table becomes narrower. Which in turn means the worst case scenario of a full table scan actually becomes a lot faster.
I also noticed that the table does not have a primary key. Which is a big no-no. The stock_id if it's numeric should be primary key. If it's not numeric it might stil be the best candidate for primary key but this is something you need to decide.
Try adding an Index on stock_id in the products table... that should help.

SQL INSERT statement conflicted with the CHECK constraint

I am getting this error
InsertRecord(InsertRow): HResult of -2147217873 (80040e2f)
Error Source: Microsoft OLE DB Provider for SQL Server Error
Description : The INSERT statement conflicted with the CHECK constraint CK_ContactName. The conflict occurred in database EPIC.7.5_GR_SM_ETLTRAINING; table dbo.ContactName.
The Statements
CREATE TABLE [dbo].[ContactName](
[UniqContactName] [dbo].[DM_UNIQID] IDENTITY(65536,1) NOT NULL,
[UniqFixedContactName] [dbo].[DM_UNIQID] NOT NULL,
[UniqEntity] [dbo].[DM_UNIQID] NOT NULL,
[LkPrefix] [dbo].[DM_PREFIX] NOT NULL,
[FullName] [dbo].[DM_NAME] NOT NULL,
[FirstName] [varchar](30) NOT NULL,
[MiddleName] [varchar](16) NOT NULL,
[LastName] [varchar](30) NOT NULL,
[LkSuffix] [dbo].[DM_SUFFIX] NOT NULL,
[DescriptionOf] [dbo].[DM_DESC_050] NOT NULL,
[Title] [dbo].[DM_TITLE] NOT NULL,
[Department] [dbo].[DM_DEPARTMENT] NOT NULL,
[UniqContactAddressMain] [dbo].[DM_UNIQID] NOT NULL,
[UniqContactAddressEmployer] [dbo].[DM_UNIQID] NOT NULL,
[UniqContactNumberMain] [dbo].[DM_UNIQID] NOT NULL,
[UniqContactNumberEmailMain] [dbo].[DM_UNIQID] NOT NULL,
[ContactMethodCode] [char](1) NOT NULL,
[InformalHeading] [varchar](50) NOT NULL,
[FormalHeading] [varchar](50) NOT NULL,
[BirthDate] [dbo].[DM_DATE] NULL,
[GenderCode] [char](1) NOT NULL,
[SSN] [char](9) NOT NULL,
[MaritalStatusCode] [char](1) NOT NULL,
[RelationToInsuredCode] [char](2) NOT NULL,
[LkLanguage] [dbo].[DM_LANGUAGE] NOT NULL,
[Comments] [dbo].[DM_COMMENT] NOT NULL,
[BillingDeliveryCode] [char](1) NOT NULL,
[ServicingDeliveryCode] [char](1) NOT NULL,
[MarketingDeliveryCode] [char](1) NOT NULL,
[CategoryCode] [char](1) NOT NULL,
[EmployerName] [dbo].[DM_NAME] NOT NULL,
[LkOccupation] [dbo].[DM_OCCUPATION] NOT NULL,
[HiredDate] [dbo].[DM_DATE] NULL,
[YearsEmployed] [smallint] NULL,
[YearsPriorEmployer] [smallint] NULL,
[FEIN] [char](9) NOT NULL,
[DUNSNumber] [char](9) NOT NULL,
[CdNAICSCode] [char](6) NOT NULL,
[CdSICCode] [char](8) NOT NULL,
[BusinessTypeCode] [char](2) NOT NULL,
[BusinessTypeOtherDesc] [varchar](50) NOT NULL,
[NumberEmployees] [smallint] NULL,
[NumberMembersManagers] [int] NULL,
[BusinessStartedDate] [datetime] NULL,
[NatureOfBusinessCode] [char](2) NOT NULL,
[NatureOfBusinessOtherDesc] [varchar](50) NOT NULL,
[CreditBureauNameCode] [char](5) NOT NULL,
[CreditBureauNameOtherDesc] [varchar](50) NOT NULL,
[CreditBureauIDNumber] [varchar](30) NOT NULL,
[DriverLicenseNumber] [varchar](25) NOT NULL,
[LicensedState] [char](4) NOT NULL,
[LicensedDate] [datetime] NULL,
[LicensedMADate] [dbo].[DM_DATE] NULL,
[DriverTypeCode] [char](1) NOT NULL,
[GoodStudentCode] [char](1) NOT NULL,
[DriverTrainingCode] [char](1) NOT NULL,
[AccidentPreventionCourseDate] [dbo].[DM_DATE] NULL,
[CommercialExperienceBeganDate] [dbo].[DM_DATE] NULL,
[MatchClientNameOf] [smallint] NOT NULL,
[InsertedByCode] [dbo].[DM_INSERTUPDATEBYCODE] NOT NULL,
[InsertedDate] [dbo].[DM_DATETIME] NOT NULL,
[UpdatedByCode] [dbo].[DM_INSERTUPDATEBYCODE] NOT NULL,
[UpdatedDate] [dbo].[DM_DATETIME] NULL,
[Flags] [dbo].[DM_FLAGS] NOT NULL,
[ts] [datetime] NOT NULL,
[SIN] [char](9) NOT NULL,
[BusinessNumber] [varchar](30) NOT NULL,
[BusinessIDNumber] [varchar](30) NOT NULL,
[IBCCode] [char](6) NOT NULL,
CONSTRAINT [PK_ContactName] PRIMARY KEY NONCLUSTERED ALTER TABLE [dbo].[ContactName]
WITH CHECK ADD CONSTRAINT [CK_ContactName]
CHECK ((
[UniqContactName]=(-1)
AND [UniqEntity]=(-1)
OR [UniqContactName]>(-1)
AND [UniqEntity]>(-1)
))
GO
I have checked contactName db and all the rows from uniqueContactName all have a value above 65000, then for UniqEntity they all are above 65000 as well.
Does anyone have an idea why this would fail?

MySql How to minimum value one row

I will send an email to my client when approaching the measurement date with php. Something measurement valid one year, something valid 3 years, etc... therefore one company's data one row. But I would like write, that how many days valid his measurement.
I hope yours understand my problem. Thanks in advance! Adam
I have a table with structure:
CREATE TABLE IF NOT EXISTS `meresek` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`COMP_ID` varchar(4) NOT NULL COMMENT 'What a company ID',
`Comp_V` int(11) NOT NULL DEFAULT '1' COMMENT 'What size number of sites',
`MERES_1` date DEFAULT NULL,
`MERES_END_1` date DEFAULT NULL,
`EV_MERES_2` date DEFAULT NULL,
`MERES_END_2` date DEFAULT NULL,
`MERES_3` date DEFAULT NULL,
`MERES_END_3` date DEFAULT NULL,
`MERES_4` date DEFAULT NULL,
`MERES_END_4` date DEFAULT NULL,
.
.
.
.
`MERES_END_8` date DEFAULT NULL,
PRIMARY KEY (`ID`),
UNIQUE KEY `COMP_ID` (`COMP_ID`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=70;
INSERT INTO `meresek` ( `ID`, `COMP_ID`, `COMP_V`,`MERES_1`
`MERES_END_1`,
`EV_MERES_2`,
`MERES_END_2`,
`MERES_3`,
`MERES_END_3`,
.
.
.
`MERES_END_8) VALUES
(1, 'X1', 1, NULL, NULL, '2011-03-07', '2011-03-31', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
(2, 'X2', 1, NULL, NULL, '2010-12-19', '2019-12-31', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
(3, 'Y11', 1, NULL, NULL, '2011-05-12', '2014-12-31', NULL, NULL, NULL, NULL, '2011-05-12', '2020-05-12', '2011-05-12', '2020-05-12', NULL, NULL, NULL, NULL),
(4, 'Y12', 1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, '2012-02-02', '2013-02-02', '2012-02-02', '2013-02-02')
select least(
coalesce(col_1,col_2....col_8),
coalesce(col_2,col_1....col_8),
coalesce(col_3,col_1....col_8),
coalesce(col_4,col_1....col_8),
coalesce(col_5,col_1....col_8),
coalesce(col_6,col_1....col_8),
coalesce(col_7,col_1....col_8),
coalesce(col_8,col_1....col_1)) where id = '35'