postgres to get match count when json value matches condition - json

I have a column in my db entitled supers with a JSON element inside.
Below is its structure:
[
{
"name": "Trample",
"category": "evergreen"
},
{
"name": "Spell",
"category": "keyword"
},
{
"name": "Trample token",
"category": "keyword"
},
{
"name": "Cast",
"category": "keyword"
},
{
"name": "Cost",
"category": "keyword"
},
{
"name": "Total",
"category": "keyword"
},
{
"name": "Power",
"category": "keyword"
},
{
"name": "Creature token",
"category": "keyword"
},
{
"name": "Control",
"category": "keyword"
},
{
"name": "Less",
"category": "keyword"
},
{
"name": "Elder",
"category": "cardType"
},
{
"name": "Dinosaur",
"category": "cardType"
},
{
"name": "Creature",
"category": "cardType"
},
{
"name": "Legendary",
"category": "cardType"
},
{
"name": "Cost x Less",
"category": "super"
},
]
I have the following query to get all rows that have similar values:
select name
from all_cards
where exists (select 1
from jsonb_array_elements(supers) f(x)
where x->>'category' = 'keyword'
and x->>'name' in ('Spell', 'Trample token', 'Cast', 'Cost', 'Total', 'Power', 'Creature token', 'Control', 'Less')
having count(*)>=8);
This query works to get all the names where count(*) >=8 but how do I get that count(*) out as a value, so I can sort the results based on that number?
I've tried jsonb_array_length, count, sum, nothing seems to be able to give me the answer I'm looking for.

You can do a cross join to get the count:
select ac.name, k.num_keywords
from all_cards ac
cross join lateral (
select count(*) as num_keywords
from jsonb_array_elements(ac.supers) f(x)
where x->>'category' = 'keyword'
and x->>'name' in ('Spell', 'Trample token', 'Cast', 'Cost', 'Total', 'Power', 'Creature token', 'Control', 'Less')
) k
where k.num_keywords >= 8

Related

How to Append to an Array in BigQuery or: How I Learned to Handle Optional Responses in Multiple Choice Typeform Payload

Working with Typeform Surveys
Here is the survey design for question 1:
Here is the survey design for question 2:
Here is the survey design for question 3:
Response Data
I took the survey twice.
On the first submission:
(favorite color) red, blue, Other: yellow
(quest) To seek the holy grail
(another question) what is your name
On the second submission:
(favorite color) red, Other: orange
(quest) Other: something else
(another question) What... is the air-speed velocity of an unladen swallow?
Here is a sample JSON file, which I saved as option.json:
{ "event_id": "EVENT_ID_1", "event_type": "form_response", "form_response": { "submitted_at": "2022-07-12T22:51:01Z", "token": "TOKEN_1", "calculated": null, "answers": [{ "value": { "boolean": null, "file_url": null, "url": null, "phone_number": null, "email": null, "field": { "ref": "QUESTION_REF_1", "id": "QUESTION_ID_1", "type": "multiple_choice" }, "text": null, "number": null, "choices": { "labels": [{ "value": "red" }, { "value": "blue" }], "other": "yellow" }, "type": "choices", "date": null, "choice": null } }, { "value": { "boolean": null, "file_url": null, "url": null, "phone_number": null, "email": null, "field": { "ref": "QUESTION_REF_2", "id": "QUESTION_ID_2", "type": "multiple_choice" }, "text": null, "number": null, "choices": { "labels": [{ "value": "To seek the holy grail" }], "other": null }, "type": "choices", "date": null, "choice": null } }, { "value": { "boolean": null, "file_url": null, "url": null, "phone_number": null, "email": null, "field": { "ref": "QUESTION_REF_3", "id": "QUESTION_ID_3", "type": "short_text" }, "text": "what is your name", "number": null, "choices": null, "type": "text", "date": null, "choice": null } }], "form_id": "FORM_ID", "variables": [], "definition": { "id": "FORM_ID", "title": "Test Multiple Choice Other", "fields": [{ "value": { "ref": "QUESTION_REF_1", "id": "QUESTION_ID_1", "type": "multiple_choice", "title": "What is your favorite color?", "choices": [{ "value": { "id": "CHOICE_ID_1", "label": "red" } }, { "value": { "id": "CHOICE_ID_2", "label": "blue" } }], "allow_multiple_selections": "true", "allow_other_choice": "true" } }, { "value": { "ref": "QUESTION_REF_2", "id": "QUESTION_ID_2", "type": "multiple_choice", "title": "What is your quest?", "choices": [{ "value": { "id": "CHOICE_ID_3", "label": "To seek the holy grail" } }], "allow_multiple_selections": "true", "allow_other_choice": "true" } }, { "value": { "ref": "QUESTION_REF_3", "id": "QUESTION_ID_3", "type": "short_text", "title": "What else could I ask?", "choices": [], "allow_multiple_selections": null, "allow_other_choice": null } }] }, "hidden": null, "landed_at": "2022-07-12T22:49:34Z" }, "_sdc_received_at": "2022-07-12T22:51:33.248Z", "_sdc_sequence": "1657666262148", "_sdc_batched_at": "2022-07-12T22:57:17.785Z", "_sdc_table_version": "0" }
{ "event_id": "EVENT_ID_2", "event_type": "form_response", "form_response": { "submitted_at": "2022-07-12T22:53:08Z", "token": "TOKEN_2", "calculated": null, "answers": [{ "value": { "boolean": null, "file_url": null, "url": null, "phone_number": null, "email": null, "field": { "ref": "QUESTION_REF_1", "id": "QUESTION_ID_1", "type": "multiple_choice" }, "text": null, "number": null, "choices": { "labels": [{ "value": "red" }], "other": "orange" }, "type": "choices", "date": null, "choice": null } }, { "value": { "boolean": null, "file_url": null, "url": null, "phone_number": null, "email": null, "field": { "ref": "QUESTION_REF_2", "id": "QUESTION_ID_2", "type": "multiple_choice" }, "text": null, "number": null, "choices": { "labels": [], "other": "something else" }, "type": "choices", "date": null, "choice": null } }, { "value": { "boolean": null, "file_url": null, "url": null, "phone_number": null, "email": null, "field": { "ref": "QUESTION_REF_3", "id": "QUESTION_ID_3", "type": "short_text" }, "text": "What... is the air-speed velocity of an unladen swallow?", "number": null, "choices": null, "type": "text", "date": null, "choice": null } }], "form_id": "FORM_ID", "variables": [], "definition": { "id": "FORM_ID", "title": "Test Multiple Choice Other", "fields": [{ "value": { "ref": "QUESTION_REF_1", "id": "QUESTION_ID_1", "type": "multiple_choice", "title": "What is your favorite color?", "choices": [{ "value": { "id": "CHOICE_ID_1", "label": "red" } }, { "value": { "id": "CHOICE_ID_2", "label": "blue" } }], "allow_multiple_selections": "true", "allow_other_choice": "true" } }, { "value": { "ref": "QUESTION_REF_2", "id": "QUESTION_ID_2", "type": "multiple_choice", "title": "What is your quest?", "choices": [{ "value": { "id": "CHOICE_ID_3", "label": "To seek the holy grail" } }], "allow_multiple_selections": "true", "allow_other_choice": "true" } }, { "value": { "ref": "QUESTION_REF_3", "id": "QUESTION_ID_3", "type": "short_text", "title": "What else could I ask?", "choices": [], "allow_multiple_selections": null, "allow_other_choice": null } }] }, "hidden": null, "landed_at": "2022-07-12T22:51:27Z" }, "_sdc_received_at": "2022-07-12T22:53:38.604Z", "_sdc_sequence": "1657666388772", "_sdc_batched_at": "2022-07-12T22:57:17.818Z", "_sdc_table_version": "0" }
Then I created a table in BigQuery using that option.json file:
I am trying to flatten this in a way that I can easily share it with others.
sequence for multiple choice responses: one issue is that I need to keep track of how many options someone selects for each multiple choice question.
another problem is that I need to be able to handle the "other" option, which does not get saved in the same way as a regular multiple choice option.
What I have so far works fine as long as no one selects "optional" in the multiple choice types:
SELECT
tf.event_id,
a.value.field.id AS question_id,
f.value.title AS question_title,
a.value.type AS question_type,
CASE
WHEN a.value.type = 'choices' THEN c.value
WHEN a.value.type = 'text' THEN a.value.text
END AS value,
IF( a.value.type = 'choices' , choices_index+1 , 1 ) AS sequence,
COALESCE( ARRAY_LENGTH( a.value.choices.labels ) , 1 ) AS sequence_end,
FROM `<PROJECT_ID>.<DATASET_ID>.option` AS tf
LEFT JOIN UNNEST( tf.form_response.answers ) AS a
LEFT JOIN UNNEST( a.value.choices.labels ) AS c WITH OFFSET AS choices_index
INNER JOIN UNNEST( form_response.definition.fields ) f
ON a.value.field.id = f.value.id
Result:
Result in JSON:
[{
"event_id": "EVENT_ID_1",
"question_id": "QUESTION_ID_1",
"question_title": "What is your favorite color?",
"question_type": "choices",
"value": "red",
"sequence": "1",
"sequence_end": "2"
}, {
"event_id": "EVENT_ID_1",
"question_id": "QUESTION_ID_1",
"question_title": "What is your favorite color?",
"question_type": "choices",
"value": "blue",
"sequence": "2",
"sequence_end": "2"
}, {
"event_id": "EVENT_ID_1",
"question_id": "QUESTION_ID_2",
"question_title": "What is your quest?",
"question_type": "choices",
"value": "To seek the holy grail",
"sequence": "1",
"sequence_end": "1"
}, {
"event_id": "EVENT_ID_1",
"question_id": "QUESTION_ID_3",
"question_title": "What else could I ask?",
"question_type": "text",
"value": "what is your name",
"sequence": "1",
"sequence_end": "1"
}, {
"event_id": "EVENT_ID_2",
"question_id": "QUESTION_ID_1",
"question_title": "What is your favorite color?",
"question_type": "choices",
"value": "red",
"sequence": "1",
"sequence_end": "1"
}, {
"event_id": "EVENT_ID_2",
"question_id": "QUESTION_ID_2",
"question_title": "What is your quest?",
"question_type": "choices",
"value": null,
"sequence": null,
"sequence_end": "0"
}, {
"event_id": "EVENT_ID_2",
"question_id": "QUESTION_ID_3",
"question_title": "What else could I ask?",
"question_type": "text",
"value": "What... is the air-speed velocity of an unladen swallow?",
"sequence": "1",
"sequence_end": "1"
}]
I want the result to append the "optional" responses to the end of the list of the multiple-choice selections, which would mean my total query result should have 9 rows.
Specifically:
EVENT_ID_1, QUESTION_ID_1 should have 3 rows:
red (sequence: 1, sequence_end: 3)
blue (sequence: 2, sequence_end: 3)
yellow (sequence: 3, sequence_end: 3)
and
EVENT_ID_2, QUESTION_ID_1 should have 2 rows:
red (sequence: 1, sequence_end: 2)
orange (sequence: 2, sequence_end: 2)
and
EVENT_ID_3, QUESTION_ID_2 should have 1 non-null row:
something else (sequence: 1, sequence_end: 1)
I need your brilliance. Could you please help?
Basic SQL approach can be:
Find all the normal response values.
Find all the other response values.
Combine them together using union all
Below is an example from your shared data:
with titles as (
select distinct f.value.id question_id,f.value.title as question_title
from `TABLE_NAME` tf
,UNNEST( form_response.definition.fields ) f
)
,response as(
 
select tf.event_id
,a.value.field.id as question_id
,a.value.type AS question_type
,CASE
WHEN a.value.type = 'choices' THEN c.value
WHEN a.value.type = 'text' THEN a.value.text
END AS value
from `TABLE_NAME` tf
,UNNEST( tf.form_response.answers ) AS a
left join UNNEST( a.value.choices.labels ) AS c
 
)
,optionals as(
select distinct tf.event_id
,a.value.field.id as question_id
,a.value.type AS question_type
,a.value.choices.other value
from `TABLE_NAME` tf
,UNNEST( tf.form_response.answers ) AS a
where a.value.choices.other is not null
 
)
select t.question_title
,f.*
,row_number() over (partition by event_id,f.question_id,question_type ) sequence
,count(*) over (partition by event_id,f.question_id,question_type) sequence_end
from (
select * from response where value is not null
union all
select * from optionals
)f
left join titles t on f.question_id = t.question_id
order by event_id,question_id
Above is just an example, your query also works fine all you have to do is get another result set which will contain only optional value and merge both results together.
N.B Use Temp tables instead of CTE.

Search an XML attrbiute and Group it

I need to search for an XML Attribute Value, group it and form a corresponding JSON element.
Now for example: Label is my AttributeID and I have 2 attributes with Label.
(In realtime - it can be 'n' number of times.)
Need to group 2 Labels as a single JSON element.
VendorClass, VendorDivision and VendorDept should all come inside Vendor Json attribute.
VendorClass attributeValue should be mapped to classCode inside Vendor and other 2 codes should be null.
Likewise, VendorDept attributeValue should be mapped to DeptCode., VedorDivision attrbiuteValue mapped to DivisionCode.
O/P JSON (mapping):
type - should always be in upper characters - mapped to attributeID,
name - in lower character - mapped to "attributeId_attributeValue",
id/code - is attribute_value
Below is my i/p XML
<CurrentRule Currency="USD"
CurrentStatus="ACTIVE"
Priority="0"
RuleCategory="Current"
RuleType="COMBINATION"
>
<CurrentRuleTargetAttributeValueList/>
<CurrentRuleAttributeValueList>
<CurrentRuleAttributeValue
TriggerAttributeID="Label"
TriggerAttributeValue="10"/>
<CurrentRuleAttributeValue
TriggerAttributeID="Label"
TriggerAttributeValue="1003"/>
<CurrentRuleAttributeValue
TriggerAttributeID="ABCDCode"
TriggerAttributeValue="AC"/>
<CurrentRuleAttributeValue
TriggerAttributeID="ABCDCode"
TriggerAttributeValue="FD"/>
<CurrentRuleAttributeValue
TriggerAttributeID="VendorClass"
TriggerAttributeValue="00N"/>
<CurrentRuleAttributeValue
TriggerAttributeID="VendorDept"
TriggerAttributeValue="100"/>
<CurrentRuleAttributeValue
TriggerAttributeID="VendorDivision"
TriggerAttributeValue="10"/>
<CurrentRuleAttributeValue
TriggerAttributeID="VendorMarket"
TriggerAttributeValue="QVC"/>
<CurrentRuleAttributeValue
TriggerAttributeID="PriceCode"
TriggerAttributeValue="FP"/>
<CurrentRuleAttributeValue
TriggerAttributeID="ProductNumber"
TriggerAttributeValue="A0000"/>
<CurrentRuleAttributeValue
TriggerAttributeID="Trader"
TriggerAttributeValue="1010"/>
<CurrentRuleAttributeValue
TriggerAttributeID="Trader"
TriggerAttributeValue="1046"/>
</CurrentRuleAttributeValueList>
</CurrentRule>
Expected JSON o/p:
{
"header": {
"type": "ORDER",
"name": "merch-attribute-example",
"createUser": "admin"
},
"filters": {
"Labels": [
{
"type": "LABEL",
"name": "Label_10",
"LabelId": 10
},
{
"type": "LABEL",
"name": "Label_1003",
"LabelId": 1003
}
],
"vendor": [
{
"type": "VENDOR",
"name": "vendorclass_00n",
"divisionCode": null,
"departmentCode": null,
"classCode": "00N"
},
{
"type": "VENDOR",
"name": "vendordept_100",
"divisionCode": null,
"departmentCode": "100",
"classCode": null
},
{
"type": "VENDOR",
"name": "vendordivision_10",
"divisionCode": "10",
"departmentCode": null,
"classCode": null
}
],
"abcd": [
{
"type": "ABCD",
"name": "abcdcode_ac",
"abcdCode": "AC"
},
{
"type": "ABCD",
"name": "abcdcode_fd",
"abcdCode": "FD"
}
],
"priceCodes": [
{
"type": "PRICE_CODE",
"name": "pricecode_fp",
"code": "FP"
}
],
"products": [
{
"type": "PRODUCT",
"name": "productnumber_a0000",
"productNumbers": [
"A0000"
]
}
],
"traders": [
{
"type": "TRADER",
"name": "trader_1010",
"vendorCode": "1010"
},
{
"type": "TRADER",
"name": "trader_1046",
"vendorCode": "1046"
}
]
}
}
enter code here
Based on the rules described this is the best approximation. I'll leave to you how to decide if it is an id or code and the rest of the output that is not clearly explained.
%dw 2.0
output application/json
---
{
header: {
"type": "ORDER",
"name": "merch-attribute-example",
"createUser": "admin"
},
filters: payload.CurrentRule.CurrentRuleAttributeValueList
mapObject ((value, key, index) ->
{
attr: key.#
}
)
pluck (($$):$)
groupBy ((item, index) -> item.attr.TriggerAttributeID)
mapObject ((value1, key1, index1) ->
(key1): value1 map
{
"type": upper(key1),
name: lower(key1) ++ "_" ++ $.attr.TriggerAttributeValue,
id: $.attr.TriggerAttributeValue
}
)
}
Output:
{
"header": {
"type": "ORDER",
"name": "merch-attribute-example",
"createUser": "admin"
},
"filters": {
"Label": [
{
"type": "LABEL",
"name": "label_10",
"id": "10"
},
{
"type": "LABEL",
"name": "label_1003",
"id": "1003"
}
],
"ABCDCode": [
{
"type": "ABCDCODE",
"name": "abcdcode_AC",
"id": "AC"
},
{
"type": "ABCDCODE",
"name": "abcdcode_FD",
"id": "FD"
}
],
"VendorClass": [
{
"type": "VENDORCLASS",
"name": "vendorclass_00N",
"id": "00N"
}
],
"VendorDept": [
{
"type": "VENDORDEPT",
"name": "vendordept_100",
"id": "100"
}
],
"VendorDivision": [
{
"type": "VENDORDIVISION",
"name": "vendordivision_10",
"id": "10"
}
],
"VendorMarket": [
{
"type": "VENDORMARKET",
"name": "vendormarket_QVC",
"id": "QVC"
}
],
"PriceCode": [
{
"type": "PRICECODE",
"name": "pricecode_FP",
"id": "FP"
}
],
"ProductNumber": [
{
"type": "PRODUCTNUMBER",
"name": "productnumber_A0000",
"id": "A0000"
}
],
"Trader": [
{
"type": "TRADER",
"name": "trader_1010",
"id": "1010"
},
{
"type": "TRADER",
"name": "trader_1046",
"id": "1046"
}
]
}
}

How do I add time-key to properties in geoJSON during SELECT from Postgres

I have a PHP-script to SELECT data from Postgres in geoJSON-format. That works fine. This is the SQL-code.
SELECT jsonb_build_object(
'type', 'FeatureCollection',
'features', json_agg(features.feature)
)
FROM (SELECT jsonb_build_object(
'type', 'Feature',
'id', id,
'geometry', st_AsGeojson(st_SetSrid(st_MakePoint(split_part(to_jsonb(row)->'data'->'location'->>'value', ',', 2)::double precision, split_part(to_jsonb(row)->'data'->'location'->>'value', ',', 1)::double precision), 4326))::json,
'properties', to_jsonb(row) - 'id'
) AS feature
FROM (SELECT * FROM cs_fiets_json WHERE last_updated > '$sel_f') row) features;
I need to include a time-key with value into the properties (to be able to use TimeDimension in Leaflet) and I am stuck at that.
The time-value I need is somewhere deep in the row.
Played with json_set and json_insert. But don't know if, how and where to use that.
So I need "time": "2022-03-02T14:32:37.00Z" directly under "properties".
Thanks in advance!
{
"id": 1581083302,
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [
5.0545584,
52.1574455
]
},
"properties": {
"data": {
"id": "359215101322999",
"voc": {
"type": "Number",
"value": 42,
"metadata": {
"dateCreated": {
"type": "DateTime",
"value": "2021-11-25T10:49:10.00Z"
},
"dateModified": {
"type": "DateTime",
"value": "2022-03-02T14:32:37.00Z"
}
}
},
"pm10": {
"type": "Number",
"value": 14,
"metadata": {
"dateCreated": {
"type": "DateTime",
"value": "2021-11-25T10:49:10.00Z"
},
"dateModified": {
"type": "DateTime",
"value": "2022-03-02T14:32:37.00Z"
}
}
},

Azure Cost Management API does not allow me to select columns

I tried to use the Azure Cost Management - Query Usage API to get details (certain columns) on all costs for a given subscription. The body I use for the request is
{
"type": "Usage",
"timeframe": " BillingMonthToDate ",
"dataset": {
"granularity": "Daily",
"configuration": {
"columns": [
"MeterCategory",
"CostInBillingCurrency",
"ResourceGroup"
]
}
}
But the response I get back is this:
{
"id": "xxxx",
"name": "xxxx",
"type": "Microsoft.CostManagement/query",
"location": null,
"sku": null,
"eTag": null,
"properties": {
"nextLink": null,
"columns": [
{
"name": "UsageDate",
"type": "Number"
},
{
"name": "Currency",
"type": "String"
} ],
"rows": [
[
20201101,
"EUR"
],
[
20201102,
"EUR"
],
[
20201103,
"EUR"
],
...
]
}
The JSON continues listing all the dates with the currency.
When I use the dataset.aggregation or dataset.grouping clauses in the JSON, I do get costs returned in my JSON but then I don't get the detailed column information that I want. And of course it is not possible to combine these 2 clauses with the dataset.columns clause. Anyone have any idea what I'm doing wrong?
I found a solution without using the dataset.columns clause (which might just be a faulty clause?). By grouping the data according tot the columns I want, I can also get the data for those column values:
{
"type": "Usage",
"timeframe": "BillingMonthToDate",
"dataset": {
"granularity": "Daily",
"aggregation": {
"totalCost": {
"name": "PreTaxCost",
"function": "Sum"
}
},
"grouping": [
{
"type": "Dimension",
"name": "SubscriptionName"
},
{
"type": "Dimension",
"name": "ResourceGroupName"
}
,
{
"type": "Dimension",
"name": "meterSubCategory"
}
,
{
"type": "Dimension",
"name": "MeterCategory"
}
]
}

Azure Data Factory Copy Activity

I have been working on this for a couple days and cannot get past this error. I have 2 activities in this pipeline. The first activity copies data from an ODBC connection to an Azure database, which is successful. The 2nd activity transfers the data from Azure table to another Azure table and keeps failing.
The error message is:
Copy activity met invalid parameters: 'UnknownParameterName', Detailed message: An item with the same key has already been added..
I do not see any invalid parameters or unknown parameter names. I have rewritten this multiple times using their add activity code template and by myself, but do not receive any errors when deploying on when it is running. Below is the JSON pipeline code.
Only the 2nd activity is receiving an error.
Thanks.
Source Data set
{
"name": "AnalyticsDB-SHIPUPS_06shp-01src_AZ-915PM",
"properties": {
"structure": [
{
"name": "UPSD_BOL",
"type": "String"
},
{
"name": "UPSD_ORDN",
"type": "String"
}
],
"published": false,
"type": "AzureSqlTable",
"linkedServiceName": "Source-SQLAzure",
"typeProperties": {},
"availability": {
"frequency": "Day",
"interval": 1,
"offset": "04:15:00"
},
"external": true,
"policy": {}
}
}
Destination Data set
{
"name": "AnalyticsDB-SHIPUPS_06shp-02dst_AZ-915PM",
"properties": {
"structure": [
{
"name": "SHIP_SYS_TRACK_NUM",
"type": "String"
},
{
"name": "SHIP_TRACK_NUM",
"type": "String"
}
],
"published": false,
"type": "AzureSqlTable",
"linkedServiceName": "Destination-Azure-AnalyticsDB",
"typeProperties": {
"tableName": "[olcm].[SHIP_Tracking]"
},
"availability": {
"frequency": "Day",
"interval": 1,
"offset": "04:15:00"
},
"external": false,
"policy": {}
}
}
Pipeline
{
"name": "SHIPUPS_FC_COPY-915PM",
"properties": {
"description": "copy shipments ",
"activities": [
{
"type": "Copy",
"typeProperties": {
"source": {
"type": "RelationalSource",
"query": "$$Text.Format('SELECT COMPANY, UPSD_ORDN, UPSD_BOL FROM \"orupsd - UPS interface Dtl\" WHERE COMPANY = \\'01\\'', WindowStart, WindowEnd)"
},
"sink": {
"type": "SqlSink",
"sqlWriterCleanupScript": "$$Text.Format('delete imp_fc.SHIP_UPS_IntDtl_Tracking', WindowStart, WindowEnd)",
"writeBatchSize": 0,
"writeBatchTimeout": "00:00:00"
},
"translator": {
"type": "TabularTranslator",
"columnMappings": "COMPANY:COMPANY, UPSD_ORDN:UPSD_ORDN, UPSD_BOL:UPSD_BOL"
}
},
"inputs": [
{
"name": "AnalyticsDB-SHIPUPS_03shp-01src_FC-915PM"
}
],
"outputs": [
{
"name": "AnalyticsDB-SHIPUPS_03shp-02dst_AZ-915PM"
}
],
"policy": {
"timeout": "1.00:00:00",
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"style": "StartOfInterval",
"retry": 3,
"longRetry": 0,
"longRetryInterval": "00:00:00"
},
"scheduler": {
"frequency": "Day",
"interval": 1,
"offset": "04:15:00"
},
"name": "915PM-SHIPUPS-fc-copy->[imp_fc]_[SHIP_UPS_IntDtl_Tracking]"
},
{
"type": "Copy",
"typeProperties": {
"source": {
"type": "SqlSource",
"sqlReaderQuery": "$$Text.Format('select distinct ups.UPSD_BOL, ups.UPSD_BOL from imp_fc.SHIP_UPS_IntDtl_Tracking ups LEFT JOIN olcm.SHIP_Tracking st ON ups.UPSD_BOL = st.SHIP_SYS_TRACK_NUM WHERE st.SHIP_SYS_TRACK_NUM IS NULL', WindowStart, WindowEnd)"
},
"sink": {
"type": "SqlSink",
"writeBatchSize": 0,
"writeBatchTimeout": "00:00:00"
},
"translator": {
"type": "TabularTranslator",
"columnMappings": "UPSD_BOL:SHIP_SYS_TRACK_NUM, UPSD_BOL:SHIP_TRACK_NUM"
}
},
"inputs": [
{
"name": "AnalyticsDB-SHIPUPS_06shp-01src_AZ-915PM"
}
],
"outputs": [
{
"name": "AnalyticsDB-SHIPUPS_06shp-02dst_AZ-915PM"
}
],
"policy": {
"timeout": "1.00:00:00",
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"style": "StartOfInterval",
"retry": 3,
"longRetryInterval": "00:00:00"
},
"scheduler": {
"frequency": "Day",
"interval": 1,
"offset": "04:15:00"
},
"name": "915PM-SHIPUPS-AZ-update->[olcm]_[SHIP_Tracking]"
}
],
"start": "2017-08-22T03:00:00Z",
"end": "2099-12-31T08:00:00Z",
"isPaused": false,
"hubName": "adf-tm-prod-01_hub",
"pipelineMode": "Scheduled"
}
}
Have you seen this link?
They get the same error message and suggest using AzureTableSink instead of SqlSink
"sink": {
"type": "AzureTableSink",
"writeBatchSize": 0,
"writeBatchTimeout": "00:00:00"
}
It would make sense for you too since your 2nd copy activity is Azure to Azure
It could be a red herring but I'm pretty sure "tableName" is a require entry in the typeProperties for a sqlSource. Yours is missing this for the input dataset. Appreciate you have a join in the sqlReaderQuery so probably best to put a dummy (but real) table name in there.
Btw, not clear why you are using $$Text.Format and WindowStart/WindowEnd on your queries if you're not transposing these values into the query; you could just put the query between double quotes.