Related
How can I sort the given JSON object with property count. I want to sort the entire sub-object. The higher the count value should come on the top an so on.
{
"Resource": [
{
"details": [
{
"value": "3.70"
},
{
"value": "3.09"
}
],
"work": {
"count": 1
}
},
{
"details": [
{
"value": "4"
},
{
"value": "5"
}
],
"work": {
"count": 2
},
{
"details": [
{
"value": "5"
},
{
"value": "5"
}
],
"work": "null"
}
]
}
You can try this example to sort your data:
data = {
"data": {
"Resource": [
{
"details": [{"value": "3.70"}, {"value": "3.09"}],
"work": {"count": 1},
},
{"details": [{"value": "4"}, {"value": "5"}], "work": {"count": 2}},
]
}
}
# sort by 'work'/'count'
data["data"]["Resource"] = sorted(
data["data"]["Resource"], key=lambda r: r["work"]["count"]
)
# sort by 'details'/'value'
for r in data["data"]["Resource"]:
r["details"] = sorted(r["details"], key=lambda k: float(k["value"]))
# pretty print:
import json
print(json.dumps(data, indent=4))
Prints:
{
"data": {
"Resource": [
{
"details": [
{
"value": "3.09"
},
{
"value": "3.70"
}
],
"work": {
"count": 1
}
},
{
"details": [
{
"value": "4"
},
{
"value": "5"
}
],
"work": {
"count": 2
}
}
]
}
}
I have simple SELECT QUERY which takes more than 3 seconds to get the result. Actual result count is 15211
Query:
select meta().id assetId, modelAndPart.partNumberID,assetLocation.id locationId from ic_v10_mammoet where type = 'asset' and modelAndPart IS NOT null and tenantId='439'
EXPLAIN:
{
"requestID": "cda5ed1b-efaf-4c5b-bb67-81f0a3542324",
"clientContextID": "13583a95-04cc-4722-90dd-9642068f9ea0",
"signature": "json",
"results": [
{
"plan": {
"#operator": "Sequence",
"~children": [
{
"#operator": "IndexScan",
"index": "type_idx",
"index_id": "f1d17cd15ab5feb6",
"keyspace": "ic_v10_mammoet",
"namespace": "default",
"spans": [
{
"Range": {
"High": [
"\"asset\""
],
"Inclusion": 3,
"Low": [
"\"asset\""
]
}
}
],
"using": "gsi"
},
{
"#operator": "Fetch",
"keyspace": "ic_v10_mammoet",
"namespace": "default"
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "Filter",
"condition": "((((`ic_v10_mammoet`.`type`) = \"asset\") and ((`ic_v10_mammoet`.`modelAndPart`) is not null)) and ((`ic_v10_mammoet`.`tenantId`) = \"439\"))"
},
{
"#operator": "InitialProject",
"result_terms": [
{
"as": "assetId",
"expr": "(meta(`ic_v10_mammoet`).`id`)"
},
{
"expr": "((`ic_v10_mammoet`.`modelAndPart`).`partNumberID`)"
},
{
"as": "locationId",
"expr": "((`ic_v10_mammoet`.`assetLocation`).`id`)"
}
]
},
{
"#operator": "FinalProject"
}
]
}
}
]
},
"text": "select meta().id assetId, modelAndPart.partNumberID,assetLocation.id locationId from ic_v10_mammoet where type = 'asset' and modelAndPart IS NOT null and tenantId='439'"
}
],
"status": "success",
"metrics": {
"elapsedTime": "15.0015ms",
"executionTime": "15.0015ms",
"resultCount": 1,
"resultSize": 3009
}
}
Yes, you forgot to ask a question. You can use this index.
CREATE INDEX idx_assets ON somedata
( tenantId, modelAndPart.partNumberID, assetLocation.id )
WHERE type = 'asset';
I'm working on a project where we use Couchbase 4.1 and we are trying to use N1QL to query for documents. Problem is that it seems very slow even though I have created indexees. The query takes around ~2 seconds with ~11000 documents.
The query:
SELECT name, displayName, imageId, childCategories FROM `bd-couchbase` WHERE assortment = 'CategoryAssortmentOne' AND categoryPath = 'category-displayname/subcategory-displayName' AND displayName IS NOT MISSING
My document is looking like this:
{
"parentName": "8442",
"categoryPath": "category-displayname/subcategory-displayName",
"lastUpdated": "2016-05-31T11:02:03.5129252+02:00",
"childCategories": [
{
"name": "0041",
"displayName": "Category 1",
"imageId": "0041"
},
{
"name": "0042",
"displayName": "Category 2",
"imageId": "0042"
},
{
"name": "0043",
"displayName": "Category 3",
"imageId": "0043"
},
{
"name": "0044",
"displayName": "Category 4",
"imageId": "0044"
},
{
"name": "0045",
"displayName": "Category 5",
"imageId": "0045"
},
{
"name": "0046",
"displayName": "Category 6",
"imageId": "0046"
}
],
"assortment": "CategoryAssortmentOne",
"name": "0040",
"displayName": "MyCategory",
"imageId": "0040"
}
I have the following index:
CREATE INDEX `category_idx` ON `bd-couchbase`((meta().`id`),`name`,`displayName`,`imageId`,`categoryPath`,`childCategories`,`assortment`) USING GSI;
When I execute the explain I can see it uses the #primary index and then doing a fetch (which I guess is the slow part of the query). But when I have created my index is not then supposed to use that?
The result of my explain:
{
"requestID": "da1946f3-5cc8-4d1e-a05b-06789aa6be92",
"signature": "json",
"results": [
{
"#operator": "Sequence",
"~children": [
{
"#operator": "PrimaryScan",
"index": "#primary",
"keyspace": "my-couchbase",
"namespace": "default",
"using": "gsi"
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "Fetch",
"keyspace": "my-couchbase",
"namespace": "default"
},
{
"#operator": "Filter",
"condition": "((((`my-couchbase`.`assortment`) =
\"CategoryAssortmentOne\") and ((`my-couchbase`.`categoryPath`) = \"category-displayname/subcategory-displayName\")) and ((`my-couchbase`.`displayName`) is not missing))"
},
{
"#operator": "InitialProject",
"result_terms": [
{
"expr": "(`my-couchbase`.`name`)"
},
{
"expr": "(`my-couchbase`.`displayName`)"
},
{
"expr": "(`my-couchbase`.`imageId`)"
},
{
"expr": "(`my-couchbase`.`childCategorie
s`)"
}
]
},
{
"#operator": "FinalProject"
}
]
}
}
]
}
],
"status": "success",
"metrics": {
"elapsedTime": "13.6696ms",
"executionTime": "13.6696ms",
"resultCount": 1,
"resultSize": 2089
}
}
Any suggestions?
Thanks in advance.
the query service didn't seem to be able to match your index with the query. any particular reason you included the meta.id in the index?
try redefining the index to cover only the fields used in your WHERE clause: assortment, categoryPath and displayName, and see if it gets mentioned in the EXPLAIN after that.
CREATE INDEX category_idx ON `bd-couchbase`(assortment, categoryPath, displayName, imageId, childCategories, name);
I'm querying my database using N1QL this way:
SELECT sum(l.lo_revenue) as revenue, o.d_year, p.p_brand1 from (SELECT p_brand1, lineorder from part where p_brand1='MFGR#2221') as p UNNEST p.lineorder l UNNEST l.supplier s UNNEST l.orderdate o
where s.s_region='EUROPE'
group by o.d_year, p.p_brand1
order by o.d_year, p.p_brand1
limit 5;
But it returns revenue: null
If I query without order by it gives me the right answer but unsorted, like this:
SELECT sum(l.lo_revenue) as revenue, o.d_year, p.p_brand1 from (SELECT p_brand1, lineorder from part where p_brand1='MFGR#2221') as p UNNEST p.lineorder l UNNEST l.supplier s UNNEST l.orderdate o
where s.s_region='EUROPE'
group by o.d_year, p.p_brand1
order by o.d_year, p.p_brand1
limit 5;
I can query this way too, having the right answer, sorted:
SELECT SUM(l.lo_revenue), o.d_year, p.p_brand1
from part p UNNEST p.lineorder l UNNEST l.supplier s UNNEST l.orderdate o
where p.p_brand1='MFGR#2221' and s.s_region='EUROPE'
group by o.d_year, p.p_brand1
order by o.d_year, p.p_brand1;
But it takes a lot of time and I would like to filter the documents before UNNEST, insted of UNNEST all documents. Is it a bug, or I'm doing something wrong?
My data is like this:
{
"p_partkey": 1,
"p_name": "lace spring",
"p_mfgr": "MFGR#1",
"p_category": "MFGR#11",
"p_brand1": "MFGR#1121",
"p_color": "goldenrod",
"p_type": "PROMO BURNISHED COPPER",
"p_size": 7,
"p_container": "JUMBO PKG",
"lineorder": [{
"lo_orderkey": 504065,
"lo_linenumber": 6,
"lo_custkey": 14704,
"lo_partkey": 1,
"lo_suppkey": 557,
"lo_orderdate": 19920603,
"lo_orderpriority": "5-LOW",
"lo_shippriority": "0",
"lo_quantity": 49,
"lo_extendedprice": 4414900,
"lo_ordtotalprice": 26849571,
"lo_discount": 3,
"lo_revenue": 4282453,
"lo_supplycost": 54060,
"lo_tax": 0,
"lo_commitdate": 19920712,
"lo_shipmode": "RAIL",
"orderdate": [{
"d_datekey": 19920603,
"d_date": "June 3, 1992",
"d_dayofweek": "Thursday",
"d_month": "June",
"d_year": 1992,
"d_yearmonthnum": 199206,
"d_yearmonth": "jun\/92",
"d_daynuminweek": 5,
"d_daynuminmonth": 3,
"d_daynuminyear": 155,
"d_monthnuminyear": 6,
"d_weeknuminyear": 23,
"d_sellingseason": "Summer",
"d_lastdayinweekfl": false,
"d_lastdayinmonthfl": true,
"d_holidayfl": false,
"d_weekdayfl": true
}],
"commitdate": [{
"d_datekey": 19920712,
"d_date": "July 12, 1992",
"d_dayofweek": "Monday",
"d_month": "July",
"d_year": 1992,
"d_yearmonthnum": 199207,
"d_yearmonth": "jul\/92",
"d_daynuminweek": 2,
"d_daynuminmonth": 12,
"d_daynuminyear": 194,
"d_monthnuminyear": 7,
"d_weeknuminyear": 28,
"d_sellingseason": "Summer",
"d_lastdayinweekfl": false,
"d_lastdayinmonthfl": true,
"d_holidayfl": false,
"d_weekdayfl": true
}],
"customer": [{
"c_custkey": 14704,
"c_name": "Customer#000014704",
"c_address": "uZaxFV8o9IGgayUEWtPU1Xmw",
"c_city": "JORDAN 5",
"c_nation": "JORDAN",
"c_region": "MIDDLE EAST",
"c_phone": "23-688-772-4209",
"c_mktsegment": "BUILDING"
}],
"supplier": [{
"s_suppkey": 557,
"s_name": "Supplier#000000557",
"s_address": "jj0wUYh9K3fG5Jh",
"s_city": "CANADA 5",
"s_nation": "CANADA",
"s_region": "AMERICA",
"s_phone": "13-390-153-6699"
}]
}, {
"lo_orderkey": ...}
This is the output of the explain, it shows the order by function, but it returns null:
{
"requestID": "654ee29a-69b9-437a-9f0e-242ac936b4f7",
"signature": "json",
"results": [
{
"#operator": "Sequence",
"~children": [
{
"#operator": "Sequence",
"~children": [
{
"#operator": "Sequence",
"~children": [
{
"#operator": "PrimaryScan",
"index": "#primary",
"keyspace": "part",
"namespace": "default",
"using": "gsi"
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "Fetch",
"keyspace": "part",
"namespace": "default"
},
{
"#operator": "Filter",
"condition": "((`part`.`p_brand1`) = \"MFGR#1121\")"
},
{
"#operator": "InitialProject",
"result_terms": [
{
"expr": "(`part`.`p_brand1`)"
},
{
"expr": "(`part`.`lineorder`)"
}
]
}
]
}
}
]
},
{
"#operator": "Alias",
"as": "p"
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "Unnest",
"as": "l",
"expr": "(`p`.`lineorder`)"
},
{
"#operator": "Unnest",
"as": "s",
"expr": "(`l`.`supplier`)"
},
{
"#operator": "Unnest",
"as": "o",
"expr": "(`l`.`orderdate`)"
},
{
"#operator": "Filter",
"condition": "((`s`.`s_region`) = \"AMERICA\")"
},
{
"#operator": "InitialGroup",
"aggregates": [
"sum((`l`.`lo_revenue`))"
],
"group_keys": [
"(`o`.`d_year`)",
"(`p`.`p_brand1`)"
]
}
]
}
},
{
"#operator": "IntermediateGroup",
"aggregates": [
"sum((`l`.`lo_revenue`))"
],
"group_keys": [
"(`o`.`d_year`)",
"(`p`.`p_brand1`)"
]
},
{
"#operator": "FinalGroup",
"aggregates": [
"sum((`l`.`lo_revenue`))"
],
"group_keys": [
"(`o`.`d_year`)",
"(`p`.`p_brand1`)"
]
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "InitialProject",
"result_terms": [
{
"as": "revenue",
"expr": "sum((`l`.`lo_revenue`))"
},
{
"as": "year",
"expr": "(`o`.`d_year`)"
},
{
"as": "p_brand1",
"expr": "(`p`.`p_brand1`)"
}
]
}
]
}
}
]
},
{
"#operator": "Order",
"sort_terms": [
{
"expr": "(`o`.`d_year`)"
},
{
"expr": "(`p`.`p_brand1`)"
}
]
},
{
"#operator": "FinalProject"
}
]
}
],
"status": "success",
"metrics": {
"elapsedTime": "8.921246ms",
"executionTime": "8.838345ms",
"resultCount": 1,
"resultSize": 6915
}
}
Follows the EXPLAIN without order by:
cbq> EXPLAIN
> {CT SUM(l.lo_revenue) as revenue, o.d_year as year, p.p_brand1 as p_brand1
> {1121') as p UNNEST p.lineorder l UNNEST l.supplier s UNNEST l.orderdate o
> where s.s_region='AMERICA'
> group by o.d_year, p.p_brand1
> ;
{
"requestID": "160cbad9-1d32-4d67-9c94-1623bba27d51",
"signature": "json",
"results": [
{
"#operator": "Sequence",
"~children": [
{
"#operator": "Sequence",
"~children": [
{
"#operator": "PrimaryScan",
"index": "#primary",
"keyspace": "part",
"namespace": "default",
"using": "gsi"
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "Fetch",
"keyspace": "part",
"namespace": "default"
},
{
"#operator": "Filter",
"condition": "((`part`.`p_brand1`) = \"MFGR#1121\")"
},
{
"#operator": "InitialProject",
"result_terms": [
{
"expr": "(`part`.`p_brand1`)"
},
{
"expr": "(`part`.`lineorder`)"
}
]
},
{
"#operator": "FinalProject"
}
]
}
}
]
},
{
"#operator": "Alias",
"as": "p"
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "Unnest",
"as": "l",
"expr": "(`p`.`lineorder`)"
},
{
"#operator": "Unnest",
"as": "s",
"expr": "(`l`.`supplier`)"
},
{
"#operator": "Unnest",
"as": "o",
"expr": "(`l`.`orderdate`)"
},
{
"#operator": "Filter",
"condition": "((`s`.`s_region`) = \"AMERICA\")"
},
{
"#operator": "InitialGroup",
"aggregates": [
"sum((`l`.`lo_revenue`))"
],
"group_keys": [
"(`o`.`d_year`)",
"(`p`.`p_brand1`)"
]
}
]
}
},
{
"#operator": "IntermediateGroup",
"aggregates": [
"sum((`l`.`lo_revenue`))"
],
"group_keys": [
"(`o`.`d_year`)",
"(`p`.`p_brand1`)"
]
},
{
"#operator": "FinalGroup",
"aggregates": [
"sum((`l`.`lo_revenue`))"
],
"group_keys": [
"(`o`.`d_year`)",
"(`p`.`p_brand1`)"
]
},
{
"#operator": "Parallel",
"~child": {
"#operator": "Sequence",
"~children": [
{
"#operator": "InitialProject",
"result_terms": [
{
"as": "revenue",
"expr": "sum((`l`.`lo_revenue`))"
},
{
"as": "year",
"expr": "(`o`.`d_year`)"
},
{
"as": "p_brand1",
"expr": "(`p`.`p_brand1`)"
}
]
},
{
"#operator": "FinalProject"
}
]
}
}
]
}
],
"status": "success",
"metrics": {
"elapsedTime": "10.661133ms",
"executionTime": "10.575926ms",
"resultCount": 1,
"resultSize": 5600
}
}
FinalProject is missing on sub query. Fix will be included in upcoming 4.5 Beta.
I am now struggling for hours but I cannot get my feed import to run properly.
The strange is, that I can import data for the normal fields in Drupal like "Title" or "Language" or "User". But for my custom fields like "field_produkt_beschreibung", no entries are made in Drupal. Even if I supply a Standard Value for the field:
The standard value is not saved in the node.
{
"entity": "node",
"settings": {
"uniq_path": "SKU",
"preprocess": "",
"feed": {
"protect_on_invalid_source": 0,
"protect_on_fewer_items": "0"
},
"processor": {
"name": "default",
"class": "FeedImportProcessor",
"options": {
"items_count": "0",
"skip_imported": 0,
"updates_only": 0,
"reset_cache": "0",
"throw_exception": 1,
"max_reported_errors": "100",
"break_on_undefined_filter": 0,
"skip_defined_functions_check": 0,
"uniq_callback": "",
"after_save": "",
"before_combine": "",
"after_combine": "",
"before_create": "",
"before_save": ""
}
},
"reader": {
"name": "sql",
"class": "SQLFIReader",
"options": {
"dsn": "mysql:dbname=d****7;host=localhost",
"user": "d******7",
"pass": "*****",
"query": "SELECT * FROM mytable",
"params": ""
}
},
"hashes": {
"name": "sql",
"class": "FeedImportSQLHashes",
"options": {
"ttl": 0,
"insert_chunk": 300,
"update_chunk": 300,
"group": "produkte_h_lter"
}
},
"filter": {
"name": "default",
"class": "FeedImportMultiFilter",
"options": {
"param": "[field]",
"include": ""
}
},
"fields": {
"title": {
"field": "title",
"column": false,
"paths": [
"Produktname"
],
"default_action": 0,
"default_value": "Standardwert",
"update_mode": 0,
"filters": {
"\u00fc": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ü",
"\u00fc",
"1"
]
},
"\u00e4": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ä",
"\u00e4",
"1"
]
},
"\u00d6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"Ö",
"\u00d6",
"1"
]
},
"\u00f6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ö",
"\u00f6",
"1"
]
},
"\u00df": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ß",
"\u00df",
"1"
]
},
"R": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"®",
" ",
"1"
]
}
},
"prefilters": []
},
"field_produkt_hersteller_raw": {
"field": "field_produkt_hersteller_raw",
"column": true,
"paths": [
"//Hersteller"
],
"default_action": 0,
"default_value": "kein Hersteller",
"update_mode": 0,
"filters": [],
"prefilters": []
},
"field_produkt_link": {
"field": "field_produkt_link",
"column": true,
"paths": [
"Link"
],
"default_action": 0,
"default_value": "Standard",
"update_mode": 0,
"filters": [],
"prefilters": []
},
"field_produkt_artikelnummer": {
"field": "field_produkt_artikelnummer",
"column": true,
"paths": [
"Artikelnummer"
],
"default_action": 0,
"default_value": "Standardwert",
"update_mode": 0,
"filters": [],
"prefilters": []
},
"field_produkt_beschreibung": {
"field": "field_produkt_beschreibung",
"column": true,
"paths": [
"Beschreibung"
],
"default_action": 0,
"default_value": "Text",
"update_mode": 0,
"filters": {
"\u00fc": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ü",
"\u00fc",
"1"
]
},
"\u00e4": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ä",
"\u00e4",
"1"
]
},
"\u00d6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"Ö",
"\u00d6",
"1"
]
},
"\u00f6": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ö",
"\u00f6",
"1"
]
},
"\u00df": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"ß",
"\u00df",
"1"
]
},
"R": {
"function": "FeedImportFilter::replace",
"params": [
"[field]",
"®",
" ",
"1"
]
}
},
"prefilters": []
}
},
"static_fields": {
"type": "produkt",
"uid": "3",
"language": "de",
"field_produkt_shop": {
"tid": "20"
},
"title": "Test"
},
"functions": []
}
}
I really would appreciate a hint, about what might be wrong with my setup. Thank you very much,
Simon
It was missing filters and umlaute (ä, ö...) in the text fields. It is working now