Importing JSON data into SQL Server; losing decimal precision and scale - json

I'm importing an array of JSON documents, like the following:
{
"_id": {
"$oid": "6092e24247f3024d478902c2"
},
"startSearch": "2021-05-05T18:20:50.132Z",
"Inst_ID": 856,
"hostname": "localhost",
"searchpath": "/Account/Main/_LookupOptimize",
"Params": {
"AccountStatusDisplay": [
"Active",
"Out For Repo",
"Repossessed"
]
},
"endSearch": "2021-05-05T18:20:53.993Z",
"elapsedTime": 3.861,
"totalRecordsFound": 1988
}
I'm importing them into a local instance of SQL Server 2019 like so:
Declare #JSON nvarchar(max)
SELECT #JSON=BulkColumn
FROM OPENROWSET (BULK 'C:\temp\QueueSearch.JSON', SINGLE_CLOB) import
INSERT INTO myJSON
SELECT *
FROM OPENJSON (#JSON)
WITH
(
[_id] nvarchar(max),
[startSearch] datetime,
[Inst_ID] int,
[hostname] nvarchar(80),
[searchpath] nvarchar(50),
[Params] nvarchar(max),
[endSearch] datetime,
[elapsedTime] NUMERIC(10,4),
[totalRecordsFound] int
)
My problem is that the elapsedTime values are being rounded off to integers. (The _id values aren't getting imported either, but first things first.) I'm probably missing something really simple, but how can I preserve the accuracy of my elapsedTime values? Thanks in advance!

Verify the data type of elapsedTime is numeric(10,4) and explicitly map the columns in INSERT to make sure that you're inserting elapsedTime from the JSON into the elapsedTime column.
INSERT INTO myJSON([_id],startSearch,Inst_ID,hostname,searchpath,Params,endSearch,elapsedTime,totalRecordsFound)
SELECT *
FROM OPENJSON (#JSON)
WITH
(
[_id] nvarchar(max),
[startSearch] datetime,
[Inst_ID] int,
[hostname] nvarchar(80),
[searchpath] nvarchar(50),
[Params] nvarchar(max),
[endSearch] datetime,
[elapsedTime] NUMERIC(10,4),
[totalRecordsFound] int
)

Related

How to Parse JSON Arrays to SQL SERVER 2016

I'm looking to create a table of data (see "Output Table") from the below JSON. I can't seem to get to the "final mile". Can someone show me the way to properly handle the parsing of the arrays into the desired output table?
Thanks! Russ
Desired Output Table
Output I am getting
declare #json nvarchar(max) =
'{
"totalCount": 1,
"nextPageKey": null,
"result": [
{
"metricId": "builtin:host.cpu.usage",
"data": [
{
"dimensions": [
"HOST-CCCC3F95D7CE56"
],
"dimensionMap": {
"dt.entity.host": "HOST-CCCC3F95D7CE56"
},
"timestamps": [
1612634400000,
1612645200000,
1612656000000
],
"values": [
0.37900028935185187,
0.3709309895833333,
0.5088975694444444
]
}
]
}
]
}'
Select TableA.totalCount, TableResult.metricId,
TableDim.*
FROM OPENJSON(#json)
WITH(
totalCount int,
result NVARCHAR(MAX) as JSON
) as TableA
CROSS APPLY OPENJSON(TableA.result)
WITH(
metricId VARCHAR(100),
data NVARCHAR(MAX) as JSON
)TableResult
CROSS APPLY OPENJSON(TableResult.data)
WITH(
dimensions NVARCHAR(MAX) as JSON,
timestamps NVARCHAR(MAX) as JSON,
[values] NVARCHAR(MAX) as JSON
)TableDim
Hard to know what the correlation is between timestamps and values, but I've assumed it's by array index.
Is dimensions always a single value in the array? I have assumed so.
You then need to use OPENJSON (and no WITH block) to break out the two arrays, and join them together on index, which is supplied in key:
Select TableA.totalCount, TableResult.metricId,
TableDim.dimensions, Vals.*
FROM OPENJSON(#json)
WITH(
totalCount int,
result NVARCHAR(MAX) as JSON
) as TableA
CROSS APPLY OPENJSON(TableA.result)
WITH(
metricId VARCHAR(100),
data NVARCHAR(MAX) as JSON
)TableResult
CROSS APPLY OPENJSON(TableResult.data)
WITH(
dimensions NVARCHAR(MAX) '$.dimensions[0]',
timestamps NVARCHAR(MAX) as JSON,
[values] NVARCHAR(MAX) as JSON
)TableDim
CROSS APPLY (
SELECT tm.value AS Timestamp, v.value AS Value
FROM OPENJSON(TableDim.timestamps) tm
JOIN OPENJSON(TableDim.[values]) v ON v.[key] = tm.[key]
) Vals

Parsing with OPENJSON

I am trying to take the response from a GET call and load it into SQL Server via OPENJSON, but I'm having trouble parsing the response.
DECLARE #json NVARCHAR(MAX);
SET #json = N'{
"LookupServiceType": "GetAssetValues",
"Items": [
{
"id": "19676",
"value": "{\"AssetCode\":\"TDAACC\",\"Symbol\":null,\"Issue\":\"ACCOUNT #49\",\"Issuer\":\"TD AMERITRADE\"}"
},
{
"id": "19677",
"value": "{\"AssetCode\":\"RE100\",\"Symbol\":null,\"Issue\":\"APN: 057\",\"Issuer\":\"SAN ANTONIO TX 78212\"}"
},
{
"id": "19908",
"value": "{\"AssetCode\":\"NALIPO\",\"Symbol\":null,\"Issue\":\"POLICY # L0472\",\"Issuer\":\"NATIONWIDE LIFE\"}"
}
]
}';
I've tried many variations, but I just can't get it right. Here are a few things I've tried. Any suggestions would be appreciated.
SELECT *
FROM OPENJSON(#json, '$.Items')
WITH (
Items int '$.id',
value NVARCHAR(MAX) '$.value',
AssetCode NVARCHAR(50) '$.value.AssetCode',
Symbol NVARCHAR(50) '$.value.Symbol',
Issue NVARCHAR(50) '$.value.Issue',
Issuer NVARCHAR(50) '$.value.Issuer'
);
SELECT id,AssetCode,Symbol,Issue, Issuer
FROM OPENJSON(#json)
WITH (
Items NVARCHAR(MAX) '$.Items' AS JSON
)
OUTER APPLY OPENJSON(Items)
WITH (
id INT 'strict $.id',
value NVARCHAR(MAX) '$.value' AS JSON
)
OUTER APPLY OPENJSON(value)
WITH (
AssetCode NVARCHAR(50) '$.AssetCode',
Symbol NVARCHAR(50) '$.Symbol',
Issue NVARCHAR(50) '$.Issue',
Issuer NVARCHAR(50) '$.Issuer'
);
I think you're looking for this. There weren't any changes needed to the JSON
select j1.LookupServiceType, j2.id, j3.AssetCode, j3.Symbol, j3.Issue, j3.Issuer
from
OPENJSON(#json) WITH (LookupServiceType nvarchar(4000),
Items NVARCHAR(MAX) '$.Items' AS JSON) j1
outer APPLY
OPENJSON(Items) WITH (id INT 'strict $.id',
value NVARCHAR(MAX) '$.value' /*AS JSON*/) j2
outer apply
OPENJSON([value]) WITH (AssetCode NVARCHAR(50) '$.AssetCode',
Symbol NVARCHAR(50) '$.Symbol',
Issue NVARCHAR(50) '$.Issue',
Issuer NVARCHAR(50) '$.Issuer') j3;
Results
LookupServiceType id AssetCode Symbol Issue Issuer
GetAssetValues 19676 TDAACC NULL ACCOUNT #49 TD AMERITRADE
GetAssetValues 19677 RE100 NULL APN: 057 SAN ANTONIO TX 78212
GetAssetValues 19908 NALIPO NULL POLICY # L0472 NATIONWIDE LIFE
The issue is the format of your JSON string, you have superfluous escapes and "".
Your first query works fine once the JSON string is cleaned up:
DECLARE #json NVARCHAR(MAX);
SET #json = N'{
"LookupServiceType": "GetAssetValues",
"Items": [
{
"id": "19676",
"value": {"AssetCode":"TDAACC","Symbol":null,"Issue":"ACCOUNT #49","Issuer":"TD AMERITRADE"}
},
{
"id": "19677",
"value": {"AssetCode":"RE100","Symbol":null,"Issue":"APN: 057","Issuer":"SAN ANTONIO TX 78212"}
},
{
"id": "19908",
"value": {"AssetCode":"NALIPO","Symbol":null,"Issue":"POLICY # L0472","Issuer":"NATIONWIDE LIFE"}
}
]
}';
SELECT *
FROM OPENJSON(#json, '$.Items')
WITH (
Items int '$.id',
AssetCode NVARCHAR(50) '$.value.AssetCode',
Symbol NVARCHAR(50) '$.value.Symbol',
Issue NVARCHAR(50) '$.value.Issue',
Issuer NVARCHAR(50) '$.value.Issuer'
);
Results:
Items AssetCode Symbol Issue Issuer
19676 TDAACC NULL ACCOUNT #49 TD AMERITRADE
19677 RE100 NULL APN: 057 SAN ANTONIO TX 78212
19908 NALIPO NULL POLICY # L0472 NATIONWIDE LIFE
Alternatively if you don't wish to fix your JSON you could use the following statement:
SELECT *
FROM OPENJSON(REPLACE(REPLACE(Replace(#json, '\', ''), '"{', '{'), '}"', '}'), '$.Items')
WITH (
Items int '$.id',
value NVARCHAR(MAX) '$.value',
AssetCode NVARCHAR(50) '$.value.AssetCode',
Symbol NVARCHAR(50) '$.value.Symbol',
Issue NVARCHAR(50) '$.value.Issue',
Issuer NVARCHAR(50) '$.value.Issuer'
);

Receiving 'NULL' return from OPENJSON() when reading JSON data

I need to pull the data from a seperate JSON File into my SQL table however , I cant seem to come right with a result.
It keeps returning 'NULL' and I dont quite understand why - I suspect it is the JSON path listed in the OPENJSON() but cant seem to come right.
JSON
{
"isMoneyClient": false,
"showPower": true,
"removeGamePlay": true,
"visualTweaks": {
"0": {
"value": true,
"name": "Clock"
},
"1": {
"value": true,
"name": "CopperIcon"
}
}
}
SQL
DECLARE #JSON VARCHAR(MAX)
SELECT #JSON = BulkColumn
FROM OPENROWSET
(BULK 'C:\config.json', SINGLE_CLOB)
AS A
UPDATE dbo.CommonBaseline
SET CommonBaseline.Config_isMoneyClient = isMoneyClient ,
CommonBaseline.Config_showPower = showPower ,
CommonBaseline.Config_removeGamePlay = removeGamePlay
FROM OPENJSON (#JSON)
WITH (
isMoneyClient Varchar (50),
showPower Varchar (50),
removeGamePlay Varchar (50)
)
AS REQUESTED I HAVE BELOW THE COMMON BASELINE SCHEME
CREATE TABLE CommonBaseline (
ServerID int NOT NULL PRIMARY KEY,
Config_isMoneyClient varchar(255) ,
Config_showPower varchar(255) ,
Config_removeGamePlay varchar(255)
);
Update:
This is an attempt to improve this answer. When you miss a column path definition in OPENJSON() call using WITH clause, matches between the keys in JSON expression and the column names are case sensitive. This is another possible reason for unexpected NULL results, because you don't use column paths in your OPENJSON() call.
Example:
DECLARE #json nvarchar(max) = N'{
"isMoneyClient": false,
"showPower": true,
"removeGamePlay": true,
"visualTweaks": {
"0": {
"value": true,
"name": "Clock"
},
"1": {
"value": true,
"name": "CopperIcon"
}
}
}'
-- NULL results.
-- The column names (IsMoneyClient) and
-- the keys in JSON expression (isMoneyClient) don't match.
SELECT *
FROM OPENJSON (#json) WITH (
IsMoneyClient Varchar (50),
ShowPower Varchar (50),
RemoveGamePlay Varchar (50)
)
-- Expected results. The correct column paths are used
SELECT *
FROM OPENJSON (#json) WITH (
IsMoneyClient Varchar(50) '$.isMoneyClient',
ShowPower Varchar(50) '$.showPower',
RemoveGamePlay Varchar(50) '$.removeGamePlay'
)
Original answer:
One possible explanation for your unexpected result is the fact, that in some cases, even if your JSON content is invalid, OPENJSON() reads only part of this content. I'm able to reproduce this with the next example:
Statement:
-- JSON
-- Valid JSON is '[{"name": "A"},{"name": "B"}]'
DECLARE #json nvarchar(max) = N'
{"name": "A"},
{"name": "B"}
'
-- Read JSON content
SELECT * FROM OPENJSON(#json, '$')
SELECT * FROM OPENJSON(#json, '$') WITH (
[name] nvarchar(100) '$.name'
)
Output (OPENJSON() reads only {"name": "A"} part of the JSON input):
----------------
key value type
----------------
name A 1
----
name
----
A
One solution here is to check your JSON content with ISJSON:
IF ISJSON(#json) = 1 PRINT 'Valid JSON' ELSE PRINT 'Not valid JSON';
If it is possible, try to fix the input JSON:
Statement:
-- JSON
-- Valid JSON is '[{"name": "A"},{"name": "B"}]'
DECLARE #json nvarchar(max) = N'
{"name": "A"},
{"name": "B"}
'
-- Read JSON content
SELECT * FROM OPENJSON(CONCAT('[', #json, ']'), '$') WITH (
[name] nvarchar(100) '$.name'
)
Output:
----
name
----
A
B
Please check your table schema of CommonBaseline. Datatype of columns Config_isMoneyClient, Config_showPower, Config_removeGamePlay
As per below scenario it is working fine
DECLARE #JSON VARCHAR(MAX)
SELECT #JSON = BulkColumn
FROM OPENROWSET
(BULK 'e:\config.json', SINGLE_CLOB)
AS A
or
SELECT #JSON = '{
"isMoneyClient": false,
"showPower": true,
"removeGamePlay": true,
"visualTweaks": {
"0": {
"value": true,
"name": "Clock"
},
"1": {
"value": true,
"name": "CopperIcon"
}
}
}'
declare #CommonBaseline as Table
( id int,
Config_isMoneyClient bit,
Config_showPower bit,
Config_removeGamePlay bit )
insert into #CommonBaseline ( id ) values ( 1 ) ---- please check your table CommonBaseline as atleast 1 record to update the same
UPDATE #CommonBaseline
SET Config_isMoneyClient = isMoneyClient ,
Config_showPower = showPower ,
Config_removeGamePlay = removeGamePlay
FROM OPENJSON (#JSON)
WITH (
isMoneyClient Varchar (50),
showPower Varchar (50),
removeGamePlay Varchar (50)
)
select * from #CommonBaseline
Note: Please check you already have your row in CommonBaseline. Since you are using update.

Using T-SQL to retrieve results from Json file and not iterating multiple sub-objects

Following MS documentation, I can get a simple example of loading json file to SQL results. The problems occur when I have more than one sub-object. This code will traverse all elements if at root level. Because I have 2 objects under "Purchase" I have to explicitly reference them. Is there an easier way to return results for all sub-objects? In this case I would like two rows of Order info.
Also have to hard code the filename to OPENROWSET instead of using (#file). Any ideas on syntax to pass in a variable for file?
Code
USE TempDB
DECLARE #json AS NVARCHAR(MAX)
DECLARE #file AS NVARCHAR(MAX)
SET #file = 'c:\temp\test.json';
SELECT #json = BulkColumn FROM OPENROWSET (BULK 'c:\temp\test2.json', SINGLE_CLOB) AS j
SELECT *
FROM OPENJSON ( #json )
WITH (
Number varchar(200) '$.Purchase[0].Order.Number' ,
Date datetime '$.Purchase[0].Order.Date',
Customer varchar(200) '$.Purchase[0].AccountNumber',
Quantity int '$.Purchase[0].Item.Quantity'
)
File contents:
{
"Purchase": [
{
"Order": {
"Number": "SO43659",
"Date": "2011-05-31T00:00:00"
},
"AccountNumber": "AW29825",
"Item": {
"Price": 2024.9940,
"Quantity": 1
}
},
{
"Order": {
"Number": "SO43661",
"Date": "2011-06-01T00:00:00"
},
"AccountNumber": "AW73565",
"Item": {
"Price": 2024.9940,
"Quantity": 3
}
}
]
}
Reference:
https://learn.microsoft.com/en-us/sql/relational-databases/json/convert-json-data-to-rows-and-columns-with-openjson-sql-server?view=sql-server-2017#option-2---openjson-output-with-an-explicit-structure
Thanks,
Bill
To get both rows, you need to use the second argument of the OPENJSON function, like this:
SELECT *
FROM OPENJSON ( #json,'$.Purchase' )
WITH (
Number varchar(200) '$.Order.Number' ,
Date datetime '$.Order.Date',
Customer varchar(200) '$.AccountNumber',
Quantity int '$.Item.Quantity'
)
This way you are telling SQL Server that you want all the nodes under the '$.Purchase' path (and it finds two rows). Without that, you would get all the nodes under root (and it finds just one row, the Purchase node).

Converting JSON to table in SQL Server 2016

I'm working on a Web project where the client application communicates with the DB via JSONs.
The initial implementation took place with SQL Server 2012 (NO JSON support and hence we implemented a Stored Function that handled the parsing) and now we are moving to 2016 (YES JSON support).
So far, we are reducing processing time by a significant factor (in some cases, over 200 times faster!).
There are some interactions that contain arrays that need to be converted into tables. To achieve that, the OPENJSON function does ALMOST what we need.
In some of these (array-based) cases, records within the arrays have one or more fields that are also OBJECTS (in this particular case, also arrays), for instance:
[{
"Formal_Round_Method": "Floor",
"Public_Round_Method": "Closest",
"Formal_Precision": "3",
"Public_Precision": "3",
"Formal_Significant_Digits": "3",
"Public_Significant_Digits": "3",
"General_Comment": [{
"Timestamp": "2018-07-16 09:19",
"From": "1",
"Type": "Routine_Report",
"Body": "[To + Media + What]: Comment 1",
"$$hashKey": "object:1848"
}, {
"Timestamp": "2018-07-16 09:19",
"From": "1",
"Type": "User_Comment",
"Body": "[]: Comment 2",
"$$hashKey": "object:1857"
}, {
"Timestamp": "2018-07-16 09:19",
"From": "1",
"Type": "Routine_Report",
"Body": "[To + Media + What]: Comment 3",
"$$hashKey": "object:1862"
}]
}, {
"Formal_Round_Method": "Floor",
"Public_Round_Method": "Closest",
"Formal_Precision": "3",
"Public_Precision": "3",
"Formal_Significant_Digits": "3",
"Public_Significant_Digits": "3",
"General_Comment": []
}]
Here, General_Comment is also an array.
When running the command:
SELECT *
FROM OPENJSON(#_l_Table_Data)
WITH ( Formal_Round_Method NVARCHAR(16) '$.Formal_Round_Method' ,
Public_Round_Method NVARCHAR(16) '$.Public_Round_Method' ,
Formal_Precision INT '$.Formal_Precision' ,
Public_Precision INT '$.Public_Precision' ,
Formal_Significant_Digits INT '$.Formal_Significant_Digits' ,
Public_Significant_Digits INT '$.Public_Significant_Digits' ,
General_Comment NVARCHAR(4000) '$.General_Comment'
) ;
[#_l_Table_Data is a variable holding the JSON string]
we are getting the column General_Comment = NULL even though the is data in there (at least in the first element of the array).
I guess that I should be using a different syntax for those columns that may contain OBJECTS and not SIMPLE VALUES, but I have no idea what that syntax should be.
I found a Microsoft page that actually solves the problem.
Here is how the query should look like:
SELECT *
FROM OPENJSON(#_l_Table_Data)
WITH ( Formal_Round_Method NVARCHAR(16) '$.Formal_Round_Method' ,
Public_Round_Method NVARCHAR(16) '$.Public_Round_Method' ,
Formal_Precision INT '$.Formal_Precision' ,
Public_Precision INT '$.Public_Precision' ,
Formal_Significant_Digits INT '$.Formal_Significant_Digits' ,
Public_Significant_Digits INT '$.Public_Significant_Digits' ,
General_Comment NVARCHAR(MAX) '$.General_Comment' AS JSON
) ;
So, you need to add AS JSON at the end of the column definition and (God knows why) the type MUST be NVARCHAR(MAX).
Very simple indeed!!!
Create Function ParseJson:
Create or Alter FUNCTION [dbo].[ParseJson] (#JSON NVARCHAR(MAX))
RETURNS #Unwrapped TABLE
(
[id] INT IDENTITY, --just used to get a unique reference to each json item
[level] INT, --the hierarchy level
[key] NVARCHAR(100), --the key or name of the item
[Value] NVARCHAR(MAX),--the value, if it is a null, int,binary,numeric or string
type INT, --0 TO 5, the JSON type, null, numeric, string, binary, array or object
SQLDatatype sysname, --whatever the datatype can be parsed to
parent INT, --the ID of the parent
[path] NVARCHAR(4000) --the path as used by OpenJSON
)
AS begin
INSERT INTO #Unwrapped ([level], [key], Value, type, SQLDatatype, parent,
[path])
VALUES
(0, --the level
NULL, --the key,
#json, --the value,
CASE WHEN Left(ltrim(#json),1)='[' THEN 4 ELSE 5 END, --the type
'json', --SQLDataType,
0 , --no parent
'$' --base path
);
DECLARE #ii INT = 0,--the level
#Rowcount INT = -1; --the number of rows from the previous iteration
WHILE #Rowcount <> 0 --while we are still finding levels
BEGIN
INSERT INTO #Unwrapped ([level], [key], Value, type, SQLDatatype, parent,
[path])
SELECT [level] + 1 AS [level], new.[Key] AS [key],
new.[Value] AS [value], new.[Type] AS [type],
-- SQL Prompt formatting off
/* in order to determine the datatype of a json value, the best approach is to a determine
the datatype that can be parsed. It JSON, an array of objects can contain attributes that arent
consistent either in their name or value. */
CASE
WHEN new.Type = 0 THEN 'bit null'
WHEN new.[type] IN (1,2) then COALESCE(
CASE WHEN TRY_CONVERT(INT,new.[value]) IS NOT NULL THEN 'int' END,
CASE WHEN TRY_CONVERT(NUMERIC(14,4),new.[value]) IS NOT NULL THEN 'numeric' END,
CASE WHEN TRY_CONVERT(FLOAT,new.[value]) IS NOT NULL THEN 'float' END,
CASE WHEN TRY_CONVERT(MONEY,new.[value]) IS NOT NULL THEN 'money' END,
CASE WHEN TRY_CONVERT(DateTime,new.[value],126) IS NOT NULL THEN 'Datetime2' END,
CASE WHEN TRY_CONVERT(Datetime,new.[value],127) IS NOT NULL THEN 'Datetime2' END,
'nvarchar')
WHEN new.Type = 3 THEN 'bit'
WHEN new.Type = 5 THEN 'object' ELSE 'array' END AS SQLDatatype,
old.[id],
old.[path] + CASE WHEN old.type = 5 THEN '.' + new.[Key]
ELSE '[' + new.[Key] COLLATE DATABASE_DEFAULT + ']' END AS path
-- SQL Prompt formatting on
FROM #Unwrapped old
CROSS APPLY OpenJson(old.[Value]) new
WHERE old.[level] = #ii AND old.type IN (4, 5);
SELECT #Rowcount = ##RowCount;
SELECT #ii = #ii + 1;
END;
return
END
For Usage:
select * from ParseJson(jsonString)