Related
I have to get some columns as is and some columns from a query as JSON document. The as is column names are known to me but rest are dynamic columns so there are not known beforehand.
Below is the query like
select col1,col2,col3,
sum(col4) as col4,
sum(col5) as col5
from my_table
group by col1,col2,col3;
here col4,col5 names are unknown to me as they are been fetched dynamically.
Suppost my_table data looks like
The expected result is like below
I tried
select JSON_OBJECT(*) from
(
select col1,col2,col3,
sum(col4) as col4,
sum(col5) as col5
from my_table
group by col1,col2,col3
);
But obviously it does not yield expected output.
I'm on 19c DB version 19.17
Any help or suggestion would be great help!
It's kinda hacky, but you could:
Use json_object(*) to convert the whole row to json
Pass the result of this json_transform*, which you can use to remove unwanted attributes
So you could do something like:
with rws as (
select mod ( level, 2 ) col1, mod ( level, 3 ) col2,
level col3, level col4
from dual
connect by level <= 10
), grps as (
select col1,col2,
sum(col3) as col3,
sum(col4) as col4
from rws
group by col1,col2
)
select col1,col2,
json_transform (
json_object(*),
remove '$.COL1',
remove '$.COL2'
) json_data
from grps;
COL1 COL2 JSON_DATA
---------- ---------- ------------------------------
1 1 {"COL3":8,"COL4":8}
0 2 {"COL3":10,"COL4":10}
1 0 {"COL3":12,"COL4":12}
0 1 {"COL3":14,"COL4":14}
1 2 {"COL3":5,"COL4":5}
0 0 {"COL3":6,"COL4":6}
json_transform is a 21c feature that's been backported to 19c in 19.10.
You may achieve this by using Polymorphic Table Function available since 18c.
Define the function that will project only specific columns and serialize others into JSON. An implementation is below.
PTF package (function implementation).
create package pkg_jsonify as
/*Package to implement PTF.
Functions below implement the API
described in the DBMS_TF package*/
function describe(
tab in out dbms_tf.table_t,
keep_cols in dbms_tf.columns_t
) return dbms_tf.describe_t
;
procedure fetch_rows;
end pkg_jsonify;
/
create package body pkg_jsonify as
function describe(
tab in out dbms_tf.table_t,
keep_cols in dbms_tf.columns_t
) return dbms_tf.describe_t
as
add_cols dbms_tf.columns_new_t;
new_col_cnt pls_integer := 0;
begin
for i in 1..tab.column.count loop
/*Initially remove column from the output*/
tab.column(i).pass_through := FALSE;
/*and keep it in the row processing (to be available for serialization*/
tab.column(i).for_read := TRUE;
for j in 1..keep_cols.count loop
/*If column is in a projection list, then remove it
from processing and pass it as is*/
if tab.column(i).description.name = keep_cols(j)
then
tab.column(i).pass_through := TRUE;
/*Skip column in the row processing (JSON serialization)*/
tab.column(i).for_read := FALSE;
end if;
end loop;
end loop;
/*Define new output column*/
add_cols := dbms_tf.columns_new_t(
1 => dbms_tf.column_metadata_t(
name => 'JSON_DOC_DATA',
type => dbms_tf.type_clob
)
);
/*Return the list of new cols*/
return dbms_tf.describe_t(
new_columns => add_cols
);
end;
procedure fetch_rows
/*Process rowset and serialize cols*/
as
rowset dbms_tf.row_set_t;
num_rows pls_integer;
new_col dbms_tf.tab_clob_t;
begin
/*Get rows*/
dbms_tf.get_row_set(
rowset => rowset,
row_count => num_rows
);
for rn in 1..num_rows loop
/*Calculate new column value in the same row*/
new_col(rn) := dbms_tf.row_to_char(
rowset => rowset,
rid => rn,
format => dbms_tf.FORMAT_JSON
);
end loop;
/*Put column to output*/
dbms_tf.put_col(
columnid => 1,
collection => new_col
);
end;
end pkg_jsonify;
/
PTF function definition based on the package.
create function f_cols_to_json(tab in table, cols in columns)
/*Function to serialize into JSON using PTF*/
return table pipelined
row polymorphic using pkg_jsonify;
/
Demo.
create table sample_tab
as
select
trunc(level/10) as id
, mod(level, 3) as id2
, level as val1
, level * level as val2
from dual
connect by level < 40
with prep as (
select
id
, id2
, sum(val1) as val1_sum
, max(val2) as val2_max
from sample_tab
group by
id
, id2
)
select *
from table(f_cols_to_json(prep, columns(id, id2)))
ID
ID2
JSON_DOC_DATA
0
1
{"VAL1_SUM":12, "VAL2_MAX":49}
0
2
{"VAL1_SUM":15, "VAL2_MAX":64}
0
0
{"VAL1_SUM":18, "VAL2_MAX":81}
1
1
{"VAL1_SUM":58, "VAL2_MAX":361}
1
2
{"VAL1_SUM":42, "VAL2_MAX":289}
1
0
{"VAL1_SUM":45, "VAL2_MAX":324}
2
2
{"VAL1_SUM":98, "VAL2_MAX":841}
2
0
{"VAL1_SUM":72, "VAL2_MAX":729}
2
1
{"VAL1_SUM":75, "VAL2_MAX":784}
3
0
{"VAL1_SUM":138, "VAL2_MAX":1521}
3
1
{"VAL1_SUM":102, "VAL2_MAX":1369}
3
2
{"VAL1_SUM":105, "VAL2_MAX":1444}
with prep as (
select
id
, id2
, sum(val1) as val1_sum
, max(val2) as val2_max
from sample_tab
group by
id
, id2
)
select *
from table(f_cols_to_json(prep, columns(id)))
ID
JSON_DOC_DATA
0
{"ID2":1, "VAL1_SUM":12, "VAL2_MAX":49}
0
{"ID2":2, "VAL1_SUM":15, "VAL2_MAX":64}
0
{"ID2":0, "VAL1_SUM":18, "VAL2_MAX":81}
1
{"ID2":1, "VAL1_SUM":58, "VAL2_MAX":361}
1
{"ID2":2, "VAL1_SUM":42, "VAL2_MAX":289}
1
{"ID2":0, "VAL1_SUM":45, "VAL2_MAX":324}
2
{"ID2":2, "VAL1_SUM":98, "VAL2_MAX":841}
2
{"ID2":0, "VAL1_SUM":72, "VAL2_MAX":729}
2
{"ID2":1, "VAL1_SUM":75, "VAL2_MAX":784}
3
{"ID2":0, "VAL1_SUM":138, "VAL2_MAX":1521}
3
{"ID2":1, "VAL1_SUM":102, "VAL2_MAX":1369}
3
{"ID2":2, "VAL1_SUM":105, "VAL2_MAX":1444}
fiddle
How can i update data using replace or regex-like method from
id | jdata
---------------
01 | {"name1":["number","2"]}
02 | {"val1":["number","12"],"val2":["number","22"]}
to
id | jdata
---------------
01 | {"name1":2 }
02 | {"val1": 12,"val2":22 }
I need to make a proper json entry for numbers and replace an array with a number from that array. Column "jdata" can have any number of similar attributes from the example. Something similar to this would do:
UPDATE table SET jdata = REPLACE(jdata, '["number","%d"]', %d);
Two ways:
The long, more clumsy way, using JSON_ARRAY:
UPDATE table1,
(
SELECT
id,
JSON_EXTRACT(jdata, "$.name1[0]") as A,
JSON_EXTRACT(jdata, "$.name1[1]") as B,
JSON_EXTRACT(jdata, "$.val1[0]") as C,
JSON_EXTRACT(jdata, "$.val1[1]") as D,
JSON_EXTRACT(jdata, "$.val2[0]") as E,
JSON_EXTRACT(jdata, "$.val2[1]") as F
FROM table1
) x
SET jdata = CASE WHEN table1.id=1 THEN JSON_ARRAY("name1",x.B)
ELSE JSON_ARRAY("val1",x.D,"val2",F) END
WHERE x.id=table1.id;
Or using JSON_REPLACE:
update table1
set jdata = JSON_REPLACE(jdata, "$.name1",JSON_EXTRACT(jdata,"$.name1[1]"))
where id=1;
update table1
set jdata = JSON_REPLACE(jdata, "$.val1",JSON_EXTRACT(jdata,"$.val1[1]"),
"$.val2",JSON_EXTRACT(jdata,"$.val2[1]"))
where id=2;
see: DBFIDDLE for both options
EDIT: To get more depth in the query, you can start with below, and create a new JSON message from this stuff without the number:
WITH RECURSIVE cte1 as (
select 0 as x
union all
select x+1 from cte1 where x<10
)
select
id,
x,
JSON_UNQUOTE(JSON_EXTRACT(JSON_KEYS(jdata),CONCAT("$[",x,"]"))) j,
JSON_EXTRACT(jdata,CONCAT("$.",JSON_UNQUOTE(JSON_EXTRACT(JSON_KEYS(jdata),CONCAT("$[",x,"]"))))) v,
JSON_UNQUOTE(JSON_EXTRACT(jdata,CONCAT("$.",JSON_UNQUOTE(JSON_EXTRACT(JSON_KEYS(jdata),CONCAT("$[",x,"]"))),"[0]"))) v1,
JSON_UNQUOTE(JSON_EXTRACT(jdata,CONCAT("$.",JSON_UNQUOTE(JSON_EXTRACT(JSON_KEYS(jdata),CONCAT("$[",x,"]"))),"[1]"))) v2
from table1
cross join cte1
where x<JSON_DEPTH(jdata)
and not JSON_EXTRACT(JSON_KEYS(jdata),CONCAT("$[",x,"]")) is null
order by id,x;
output:
id
x
j
v
v1
v2
1
0
name1
["number", "2"]
number
2
2
0
val1
["number", "12"]
number
12
2
1
val2
["number", "22"]
number
22
This should take care of JSON message which also contains values like val3, val4, etc, until a maximum depth which is now fixed to 10 in cte1.
EDIT2: When it is just needed to remove the "number" from the JSON message, you can also repeat this UPDATE until all "number" tags are removed (you can repeat this in a stored procedure, I am not going to write the stored procedure for you 😉)
update
table1,
( WITH RECURSIVE cte1 as (
select 0 as x
union all
select x+1 from cte1 where x<10
) select * from cte1 )x
set jdata = JSON_REMOVE(table1.jdata, CONCAT("$.",JSON_UNQUOTE(JSON_EXTRACT(JSON_KEYS(jdata),CONCAT("$[",x,"]"))),"[0]"))
where JSON_UNQUOTE(JSON_EXTRACT(jdata,CONCAT("$.",JSON_UNQUOTE(JSON_EXTRACT(JSON_KEYS(jdata),CONCAT("$[",x,"]"))),"[0]"))) = "number"
An example, where I do run the update 2 times, is in this DBFIDDLE
Let say I have a table with 3 variable: var1, var2, var3 and another table of reference.
Now I want to create anothe variable: var4; var4 is computed from var1, var2 and var3 and the reference table.
I could create a macro to do that (for each line), but I dont know how to pass the value from the macro (a table) into a result. I wonder how can we do it by creating a function to get a value, not a macro?
I understand that if I can create a function like that (this function will include data step and proc summary), thing will be easy like:
var4 = myfunction(var1, var2, var3).
Update:
The table reference in my case is:
age range1 range2 range3
1 10 1 8
2 0 4 1
3 4 6 1
4 6 5 2
5 10 5 6
So I want my function to get like:
var4 = myfunction(var1, var2, var3), for example var1 = 2, var2 = 2, var3 = 5:
Take the sum of range2 (correspond to var1 = 2); from line 2 to line 5 (corrspond to var2= 2 and var3 = 5) and the result will be: 4 + 6 + 5 + 5.
Thanks in advance.
One solution could be to first modify the reference table to a long format with one row per age per range number, and with the accumulated range in a new variable. The sum from var2 to var3 then becomes the difference between two accumulated values, which is easier to compute in a double join:
* Define input data with parameters;
data have;
input var1 var2 var3;
datalines;
1 1 3
2 2 5
3 1 4
;
run;
* Define reference table;
data ref_table;
input age range1 range2 range3;
datalines;
1 10 1 8
2 0 4 1
3 4 6 1
4 6 5 2
5 10 5 6
;
run;
* Modify reference table to long format with accumulated ranges;
data mod_ref_table;
set ref_table;
* Calculated accumulated values of each range;
acc_range1 + range1;
acc_range2 + range2;
acc_range3 + range3;
* Output the accumulated values for each range;
range_no = 1;
acc_range = acc_range1;
output;
range_no = 2;
acc_range = acc_range2;
output;
range_no = 3;
acc_range = acc_range3;
output;
keep age range_no acc_range;
run;
* Calculate output;
proc sql;
create table want as
select a.*
,case
when a.var2 = 1 then c.acc_range
else c.acc_range - b.acc_range
end as val4
from have as a
left join mod_ref_table as b
on a.var1 = b.range_no and a.var2-1 = b.age
left join mod_ref_table as c
on a.var1 = c.range_no and a.var3 = c.age
;
quit;
Macro returning a value
OK the solution can be found here.
Anyway thanks for your support.
My recursion give me complete structure
For example:
(This example is only for KOMPARTNR=49807, but I have more KOMPARTNR where are more than 500 results.)
How real items structure looks:
49807(level 1) -> 50208(level 2) -> 520008(level 3)
49807(level 1) -> 52344(level 2)
49807(level 1) -> 20308(level 2) -> 51005(level 3)
Output looks:
KOMPARTNR ARTIKEL_NR level
49807 50208 1
49807 52344 1
49807 20308 1
50208 520008 2
520008 530000 3
52344 54500 2
20308 51005 2
51005 53250 3
I want the last records from the Structure. From the example above, only 530000, 54500 and 53250 will stand out. But it must be variable. Another KOMPARTNR have another result, that's why I wrote too "How real structure looks".
What I only want from this example:
KOMPARTNR ARTIKEL_NR level
520008 530000 3
52344 54500 2
51005 53250 3
Here is my recursion:
WITH n(KOMPARTNR, ARTIKEL_NR, level) AS
(SELECT SMSTLPOS.KOMPARTNR, SMSTLPOS.ARTNR, 1 AS level
FROM SMSTLPOS
WHERE
SMSTLPOS.KOMPARTNR='49807'
UNION ALL
SELECT SMSTLPOS1.KOMPARTNR, SMSTLPOS1.ARTNR, n.level+1
FROM SMSTLPOS as SMSTLPOS1, n
WHERE n.ARTIKEL_NR = SMSTLPOS1.KOMPARTNR
)
SELECT * FROM n
How can I get last records from the Structure?
You are almost there ...
WITH n(KOMPARTNR, ARTIKEL_NR, level) AS
(SELECT SMSTLPOS.KOMPARTNR, SMSTLPOS.ARTNR, 1 AS level
FROM SMSTLPOS
WHERE
SMSTLPOS.KOMPARTNR='49807'
UNION ALL
SELECT SMSTLPOS1.KOMPARTNR, SMSTLPOS1.ARTNR, n.level+1
FROM SMSTLPOS as SMSTLPOS1, n
WHERE n.ARTIKEL_NR = SMSTLPOS1.KOMPARTNR
)
SELECT * FROM n WHERE ARTIKEL_NR NOT IN (SELECT TOP (
(SELECT COUNT(*) FROM n) - 2
) ARTIKEL_NR
FROM n
)
And even simpler
WITH n(KOMPARTNR, ARTIKEL_NR, level) AS
(SELECT SMSTLPOS.KOMPARTNR, SMSTLPOS.ARTNR, 1 AS level
FROM SMSTLPOS
WHERE
SMSTLPOS.KOMPARTNR='49807'
UNION ALL
SELECT SMSTLPOS1.KOMPARTNR, SMSTLPOS1.ARTNR, n.level+1
FROM SMSTLPOS as SMSTLPOS1, n
WHERE n.ARTIKEL_NR = SMSTLPOS1.KOMPARTNR
)
SELECT * FROM n EXCEPT SELECT TOP (( SELECT COUNT(ARTIKEL_NR) FROM n) -2) * FROM n
I've been struggling for about 2 hours on one query now. Help? :(
I have a table like this:
id name lft rgt
35 Top level board 1 16
37 2nd level board 3 6 15
38 2nd level board 2 4 5
39 2nd level board 1 2 3
40 3rd level board 1 13 14
41 3rd level board 2 9 12
42 3rd level board 3 7 8
43 4th level board 1 10 11
It is stored in the structure recommended in this tutorial. What I want to do is select a forum board and all sub forums ONE level below the selected forum board (no lower). Ideally, the query would get the selected forum's level while only being passed the board's ID, then it would select that forum, and all it's immediate children.
So, I would hopefully end up with:
id name lft rgt
35 Top level board 1 16
37 2nd level board 3 6 15
38 2nd level board 2 4 5
39 2nd level board 1 2 3
Or
id name lft rgt
37 2nd level board 3 6 15
40 3rd level board 1 13 14
41 3rd level board 2 9 12
42 3rd level board 3 7 8
The top rows here are the parent forums, the others sub forums. Also, I'd like something where a depth value is given, where the depth is relative to the selected parent form. For example, taking the last table as some working data, we would have:
id name lft rgt depth
37 2nd level board 3 6 15 0
40 3rd level board 1 13 14 1
41 3rd level board 2 9 12 1
42 3rd level board 3 7 8 1
Or
id name lft rgt depth
35 Top level board 1 16 0
37 2nd level board 3 6 15 1
38 2nd level board 2 4 5 1
39 2nd level board 1 2 3 1
I hope you get my drift here.
Can anyone help with this? It's really getting me annoyed now :(
James
The easiest way for you to do it - just add a column where you keep the depth.
Otherwise the query will be very inefficient - you will have to get a the whole hierarchy, sorted by left number (that will put very first child be first), join it to itself to make sure that for each next node left number is equal to previous node right number + 1
In general, nested intervals algorithm is nice, but has a serious disadvantage - if you add something to tree, a lot of recalculations required.
A nice alternative for this is Tropashko Nested intervals algorithm with continued fractions - just google for it. And getting a single level below the parent with this algorithm is done very naturally. Also, given a child, you can calculate all numbers for all its parents without hitting a database.
One more thing to consider is that relational databases really are not the most optimal and natural way to store hierarchical data. A structure like you have here - a binary tree, essentially - would be much easier to represent with an XML blob that you can persist, or store as an object in an object-oriented database.
I prefer the adjacency list approach myself. The following example uses a non-recursive stored procedure to return a tree/subtree which I then transform into an XML DOM but you could do whatever you like with the resultset. Remember it's a single call from PHP to MySQL and adjacency lists are much easier to manage.
full script here : http://pastie.org/1294143
PHP
<?php
header("Content-type: text/xml");
$conn = new mysqli("localhost", "foo_dbo", "pass", "foo_db", 3306);
// one non-recursive db call to get the tree
$result = $conn->query(sprintf("call department_hier(%d,%d)", 2,1));
$xml = new DomDocument;
$xpath = new DOMXpath($xml);
$dept = $xml->createElement("department");
$xml->appendChild($dept);
// loop and build the DOM
while($row = $result->fetch_assoc()){
$staff = $xml->createElement("staff");
// foreach($row as $col => $val) $staff->setAttribute($col, $val);
$staff->setAttribute("staff_id", $row["staff_id"]);
$staff->setAttribute("name", $row["name"]);
$staff->setAttribute("parent_staff_id", $row["parent_staff_id"]);
if(is_null($row["parent_staff_id"])){
$dept->setAttribute("dept_id", $row["dept_id"]);
$dept->setAttribute("department_name", $row["department_name"]);
$dept->appendChild($staff);
}
else{
$qry = sprintf("//*[#staff_id = '%d']", $row["parent_staff_id"]);
$parent = $xpath->query($qry)->item(0);
if(!is_null($parent)) $parent->appendChild($staff);
}
}
$result->close();
$conn->close();
echo $xml->saveXML();
?>
XML Output
<department dept_id="2" department_name="Mathematics">
<staff staff_id="1" name="f00" parent_staff_id="">
<staff staff_id="5" name="gamma" parent_staff_id="1"/>
<staff staff_id="6" name="delta" parent_staff_id="1">
<staff staff_id="7" name="zeta" parent_staff_id="6">
<staff staff_id="2" name="bar" parent_staff_id="7"/>
<staff staff_id="8" name="theta" parent_staff_id="7"/>
</staff>
</staff>
</staff>
</department>
SQL Stuff
-- TABLES
drop table if exists staff;
create table staff
(
staff_id smallint unsigned not null auto_increment primary key,
name varchar(255) not null
)
engine = innodb;
drop table if exists departments;
create table departments
(
dept_id tinyint unsigned not null auto_increment primary key,
name varchar(255) unique not null
)
engine = innodb;
drop table if exists department_staff;
create table department_staff
(
dept_id tinyint unsigned not null,
staff_id smallint unsigned not null,
parent_staff_id smallint unsigned null,
primary key (dept_id, staff_id),
key (staff_id),
key (parent_staff_id)
)
engine = innodb;
-- STORED PROCEDURES
drop procedure if exists department_hier;
delimiter #
create procedure department_hier
(
in p_dept_id tinyint unsigned,
in p_staff_id smallint unsigned
)
begin
declare v_done tinyint unsigned default 0;
declare v_dpth smallint unsigned default 0;
create temporary table hier(
dept_id tinyint unsigned,
parent_staff_id smallint unsigned,
staff_id smallint unsigned,
depth smallint unsigned
)engine = memory;
insert into hier select dept_id, parent_staff_id, staff_id, v_dpth from department_staff
where dept_id = p_dept_id and staff_id = p_staff_id;
/* http://dev.mysql.com/doc/refman/5.0/en/temporary-table-problems.html */
create temporary table tmp engine=memory select * from hier;
while not v_done do
if exists( select 1 from department_staff e
inner join hier on e.dept_id = hier.dept_id and e.parent_staff_id = hier.staff_id and hier.depth = v_dpth) then
insert into hier select e.dept_id, e.parent_staff_id, e.staff_id, v_dpth + 1 from department_staff e
inner join tmp on e.dept_id = tmp.dept_id and e.parent_staff_id = tmp.staff_id and tmp.depth = v_dpth;
set v_dpth = v_dpth + 1;
truncate table tmp;
insert into tmp select * from hier where depth = v_dpth;
else
set v_done = 1;
end if;
end while;
select
hier.dept_id,
d.name as department_name,
s.staff_id,
s.name,
p.staff_id as parent_staff_id,
p.name as parent_name,
hier.depth
from
hier
inner join departments d on hier.dept_id = d.dept_id
inner join staff s on hier.staff_id = s.staff_id
left outer join staff p on hier.parent_staff_id = p.staff_id;
drop temporary table if exists hier;
drop temporary table if exists tmp;
end #
delimiter ;
-- TEST DATA
insert into staff (name) values
('f00'),('bar'),('alpha'),('beta'),('gamma'),('delta'),('zeta'),('theta');
insert into departments (name) values
('Computing'),('Mathematics'),('English'),('Engineering'),('Law'),('Music');
insert into department_staff (dept_id, staff_id, parent_staff_id) values
(1,1,null),
(1,2,1),
(1,3,1),
(1,4,3),
(1,7,4),
(2,1,null),
(2,5,1),
(2,6,1),
(2,7,6),
(2,8,7),
(2,2,7);
-- TESTING (call this sproc from your php)
call department_hier(1,1);
call department_hier(2,1);