Simplify Couchdb JSON response - json

I'm storing location data in Couchdb, and am looking for a way to get an array of just the values, instead of key: value for every record. For example:
The current response
{"total rows": 250, "offset": 0, "rows":[
{"id": "ec5de6de2cf7bcac9a2a2a76de5738e4", "key": "user1", "value": {"city": "San Francisco", "address":"1001 Bayhill Dr"},
{"id": "ec5de6de2cf7bcac9a2a2a76de573ae4","key": "user1", "value": {"city": "Palo Alto", "address":"583 Waverley St"}
... (etc).
]}
I only really need:
[{"city": "San Francisco", "address":"1001 Bayhill Dr"},
{"city": "Palo Alto", "address":"583 Waverley St"},
...]
The reason for all this is to minimize the amount of bandwidth that a JSON response consumes.
I can't seem to find a way to transform the view into a simple array. Any suggestions?
Thanks.

You can use _show and _list functions, they take either a document or a view (respectively) and can send back a transformed response in whatever format you need. (in this case, JSON)
Update: I ran a simple test with the data you provided here on my own CouchDB. Here's the list function I ended up writing. Customize it to fit your needs. :)
function (head, req) {
// specify that we're providing a JSON response
provides('json', function() {
// create an array for our result set
var results = [];
while (row = getRow()) {
results.push({
city: row.value.city,
address: row.value.address
});
}
// make sure to stringify the results :)
send(JSON.stringify(results));
});
}

Related

get by id from local json http

I have fake users.json file and I can http.get to list the array of json.
Since I want to get the particular user by id and haven't stored the data in the database, instead just use the fake json data.
[
{
"id": "cb55524d-1454-4b12-92a8-0437e8e6ede7",
"name": "john",
"age": "25",
"country": "germany"
},
{
"id": "ab55524d-1454-4b12-92a8-0437e8e6ede8",
"name": "tom",
"age": "28",
"country": "canada"
}
]
I can do this stuff if the data is stored in the database, but not sure how to proceed with the fake json data.
Any help is appreciated.
Thanks
If you need the json as raw data, for just fake data, You can simply require it and use it as object..
const JsonObj = require('path/to/file.json')
console.log(JsonObj[0].id) // <-- cb55524d-1454-4b12-92a8-0437e8e6ede7
Plus, if you need more dynamic solution, there is a good JSON-server you can easily use for testing and so: check this git repo
var _ = require('underscore');
var dummyJson = [
{
"id": "cb55524d-1454-4b12-92a8-0437e8e6ede7",
"name": "john",
"age": "25",
"country": "germany"
},
{
"id": "ab55524d-1454-4b12-92a8-0437e8e6ede8",
"name": "tom",
"age": "28",
"country": "canada"
}
]
var requiredID = "cb55524d-1454-4b12-92a8-0437e8e6ede7";
var reuiredObject = _.find(dummyJson, function (d) {
return d.id === requiredID;
})
Get JSON object using JSON.parse('users.json') and store it in a variable users.
Loop through array of users using for .. in and using if condition on id update the object if required.
Stringify the updated users object using JSON.stringify(users); and write this string to users.json file using fs.write() module in NodeJS so you will have updated objects in your json file.

Map one JSON schema to a different JSON schema

I have a bunch of JSON files, thousands of different schemas. Using GenSON (the Python JSON schema generator), I managed to create schema files for each of the input files. Now, what I'd like to do is standardize all these different files to one defined schema. Here's an example:
Input
{
"name": "Bob Odenkirk",
"title": "Software Engineer",
"location": {
"locality": "San Francisco",
"region": "CA",
"country": "United States"
},
"age": 62,
"status": "Active"
}
Output
{
"names": ["Bob Odenkirk"],
"occupations": ["Software Engineer"],
"locations": ["San Francisco, CA"]
}
Essentially, I am looking for a language agnostic method (i.e., I don't care what programming language is used) of defining how an input JSON file should be parsed to an output JSON file.
The url https://github.com/bazaarvoice/jolt#jolt
says that Jolt may be what you're looking for.
Jolt
JSON to JSON transformation library written in Java where the "specification" for the transform is itself a JSON document.
Useful For
Transforming JSON data from ElasticSearch, MongoDb, Cassandra, etc before sending it off to the world
Extracting data from a large JSON documents for your own consumption
Jolt Spec
[
// First build the "city, state" string for location
{
"operation": "modify-default-beta",
"spec": {
"location": {
"locConcat": "=concat(#(1,locality),', ',#(1,region))"
}
}
},
// Then map the fields as needed to positions in an output json
{
"operation": "shift",
"spec": {
"name": "name[0]",
"title": "occupations[0]",
"location": {
"locConcat": "locations[0]"
}
}
}
]
I am not sure is your expecting like below. Long time back I have created flat object and output format object. It will return output format object with data filled.
var input = {
"name": "Bob Odenkirk",
"title": "Software Engineer",
"location": {
"locality": "San Francisco",
"region": "CA",
"country": "United States"
},
"age": 62,
"status": "Active"
};
var outputFormat = {
"name": "name",
"occupations": "title",
"locations": "location.locality, location.region"
};
var flatInput = {};
function generateFlatInput(input, parent){
for (var prop in input) {
if(input.hasOwnProperty(prop) && typeof input[prop] === 'object')
flatInput = generateFlatInput(input[prop], parent + prop + '.');
else
flatInput[parent + prop] = input[prop];
}
return flatInput;
}
function generateOutput(input, outputFormat, delimiter){
input = generateFlatInput(input, '');
for (var prop in outputFormat) {
var fields = outputFormat[prop].split(delimiter);
var fieldValue = [];
for(i = 0; i < fields.length; i++){
if(!input.hasOwnProperty(fields[i].trim())) continue;
fieldValue.push(input[fields[i].trim()]);
}
outputFormat[prop] = fieldValue.join(delimiter);
}
return outputFormat;
}
console.log(generateOutput(input, outputFormat, ', '));
https://jsfiddle.net/u2yyuguk/1/
I think the best, fastest, easiest way to parse many JSON files together is using python.
I was doing something similar to your project and ran into the same problem.
I found this site which teaches how to use python to actually parse JSON files together. Turns out there is a library on python called json(use pip to download json dependencies) which enables JSON file processing. If you already have a python editor, This method would be easier and faster then using Jolt
Check This website for more info: https://code.tutsplus.com/tutorials/how-to-work-with-json-data-using-python--cms-25758.
You can also use JS, which is again faster than Jolt. this is the website: https://learn.microsoft.com/en-us/scripting/javascript/reference/json-parse-function-javascript . It is very easy as you can use JSON.parse() function

Streaming huge json with Akka Stream

I have a problem of huge http response with a json slab, where only portion is point of interest.
I cannot change the response structure.
here is an example
{
"searchString": "search",
"redirectUrl": "",
"0": {
"numRecords": 123,
"refinementViewModelCollector": {},
// Lots of data here
"results": [
{
"productCode": "123",
"productShortDescription": "Desc",
"brand": "Brand",
"productReview": {
"reviewScore": 0
},
"priceView": {
"salePriceDisplayable": false,
},
"productImageUrl": "url",
"alternateImageUrls": [
"url1"
],
"largeProductImageUrl": "url4",
"videoUrl": ""
},
{
"productCode": "124",
"productShortDescription": "Desc",
"brand": "Brand",
"productReview": {
"reviewScore": 0
},
"priceView": {
"salePriceDisplayable": false,
},
"preOrder": false,
"productImageUrl": "url",
"alternateImageUrls": [
"url1"
],
"largeProductImageUrl": "url4",
"videoUrl": ""
}
]
//lots of data here
}
}
My point of interest is entries in results Jason Array, but the are sitting in the middle of json
I created a small Play WS Client like this:
val wsClient: WSClient = ???
val ret = wsClient.url("url").stream()
ret.flatMap { response =>
response.body.via(JsonFraming.objectScanner(1024))
.map(_.utf8String)
.runWith(Sink.foreach(println))
}
this will not work because it will take whole json slab as Json object. I need to skip some data until "results": entry appear in the stream, then start parsing entries and skip all the rest.
Any ideas how to do this?
Check out Alpakka's JSON module, which can stream specific parts of a nested JSON structure:
response
.body
.via(JsonReader.select("$.0.results[*]"))
.map(_.utf8String)
.runWith(Sink.foreach(println)) // or runForeach(println)
There are parsers that support parsing as a stream. For a good example check out this Circe example https://github.com/circe/circe/tree/master/examples/sf-city-lots
I'd love a better, Scala-specific answer to this question, but check out the "Mixed Reads Example" in the documentation for Google's GSON library:
https://sites.google.com/site/gson/streaming
Gson also supports mixed streaming & object model access. This lets your application have the best of both worlds: the productivity of object model access with the efficiency of streaming
...
This code reads a JSON document containing an array of messages. It steps through array elements as a stream to avoid loading the complete document into memory. It is concise because it uses Gson’s object-model to parse the individual messages
This should have great memory-performance (the code reads from a Java InputStream, so the full structure is never in memory), but may require some effort to get your results into Scala case classes.

Bash Parse JSON Objects jsawk

I´m trying to parse some JSON, which is the output of the Philips Hue API.
I found the tool jsawk, but somehow I´m not able to access the data inside the JSON Object.
The command:
... | jsawk 'return this.f709f9c0b-on-0.name'
works but unfortunately I don't have the ID of each object (e.g. f709f9c0b-on-0). Is there a way to access the object without knowing the ID and then to find out that ID. I tried to use "*" or Iterations of the objects but nothing was working.
Thanks in advance
The output looks like this:
{
"f709f9c0b-on-0": {
"name": "Badezimmer on 0",
"lights": [
"4"
],
"owner": "3e281978544fb15b42bc0e3a3f4ce3",
"recycle": true,
"locked": false,
"appdata": {},
"picture": "",
"lastupdated": "2016-02-17T17:20:06",
"version": 1
},
"69d313be0-on-0": {
"name": "Klavier on 0",
"lights": [
"1"
],
"owner": "3e281978544fb15b42bc0e3a3f4ce3",
"recycle": true,
"locked": false,
"appdata": {},
"picture": "",
"lastupdated": "2016-02-17T17:31:05",
"version": 1
},
...
}
f709f9c0b-on-0 is not a valid identifier due to the hyphens, so you can't use the dot notation. This might work (untested)
... | jsawk 'return this["f709f9c0b-on-0"].name'
I don't have jsawk, but jq can do it like this:
... | jq '.["f709f9c0b-on-0"].name'
Just for the rest of the world.
I solved the problem, by creating a .jar, which handles the problem. I find it much easier to do this in Java, than in bash.
I used this JSON-classes: https://github.com/stleary/JSON-java
Just download the files, create the package org.json and your good to go.
The Java Code, which worked for me is:
String JSON = "your JSON";
JSONObject jsonObject = new JSONObject(JSON);
ArrayList<ArrayList<String>> keyArray = new ArrayList<>();
Iterator<String> keys = jsonObject.keys(); //get all JSON keys
while (keys.hasNext()) { //for all keys do...
String key = (String) keys.next(); //get Current Key
//Now you can access the Object in the Object with:
jsonObject.getJSONObject(key).getString("name")
}
I hope this helps someone.

typeahead nested json object

I am new to Ember and JSON. I want to parse a JSON object that is below with typeahead library
and access nested object values by searching their keys.
I have this Json format:
return [
{
"id": 1,
"category_name": "Supermarket",
"category_description": "SUPER MARKET",
"image_url": "",
"merchants": [
{
"name": "CARREFOUR",
"id": 12,
"merchant_type_id": 1,
"merchant_type_description": "Gold",
"merchant_redeption_rate": 0.002500,
"image_url": "https://jpg",
"branches": [
{
"id": 123456,
"latitude": 37.939483,
"area": "ΑΓ. ΔΗΜΗΤΡΙΟΣ",
"zip": "12345"
},
{
"id": 4567890,
"longitude": 23.650622,
"area": "ΑΓ. ΙΩΑΝΝΗΣ ΡΕΝΤΗΣ",
"zip": "12345"
}
]
},
{
"name": "CAFCO",
"id": 13,
"merchant_type_id": 3,
"merchant_type_description": "None",
"merchant_redeption_rate": 0.002500,
"image_url": "https:.jpg",
"branches": [
{
"id": 127890,
"latitude": 38.027870,
"area": "ΠΕΡΙΣΤΕΡΙ",
"zip": "12345"
}
]
}
]
},
{
"id": 2,
"category_name": "Πολυκαταστήματα",
"category_description": "ΠΟΛΥΚΑΤΑΣΤΗΜΑ",
"image_url": "",
"merchants": [
{
"name": "AGGELOPOYLOS CHR.",
"id": 15,
"merchant_type_id": 2,
"merchant_type_description": "Silver",
"merchant_redeption_rate": 0.002500,
"image_url": "https://www.nbg.gr/greek/retail/cards/reward-programmes/gonational/PublishingImages/aggelopoulos.jpg",
"branches": [
{
"id": 234780,
"latitude": 35.366118,
"longitude": 24.479461,
"address": "ΕΘΝ. ΜΑΚΑΡΙΟΥ 9 & ΕΛ. ΒΕΝΙΖΕΛΟΥ 1",
"area": "Ν. ΦΑΛΗΡΟ",
"zip": "12345"
}
]
}
]
}
];
--------------------------Updated----------------------------
For example, i want to search using typeahead the name of merchants and when the letter we write to search matches the name of merchants it will appear the corresponding category_name and backwards.
Example -> when i keyboard the s it will appear :
Category : Supermarket,
Name: CARREFOUR
Name: CAFCO
And the same output on the dropdown of search when i keyboard the letter c.
Any help?
New Jsbin example
The simplest way (in my mind) to get this to work is to create a computed property that will contain an array of latitudes. But how do we get there?
To get to latitude, you need to go through array of merchants and then array of branches. Being that this will be across multiple elements, you are going to end up with "array of arrays" type data structure, which is annoying to deal with. So, to simplify this, we can create a simple flatten function as follows:
flatten: function(origArray){
var newArr = [];
origArray.forEach(function(el) {
el.forEach(function(eachEl){
newArr.push(eachEl);
});
});
return newArr;
},
In addition to our function above, Ember already provides us with many other useful functions that can be used on arrays (see here). One of those is mapBy(property) which transforms an array into another array only keeping the values of the property we specified.
So, to create a lats (for latitudes) property, we can just do this:
lats: function(){
var merchantsArr = this.get('model').mapBy('merchants');
merchantsArr = this.flatten(merchantsArr);
var branchesArr = merchantsArr.mapBy('branches');
branchesArr = this.flatten(branchesArr);
return branchesArr.mapBy("latitude").compact();
}.property('model')
Above, I am basically using mapBy, flatten (see above) and compact which
Returns a copy of the array with all null and undefined elements removed.
Once you have the lats property with all the necessary data, the rest is easy.
Your call to component becomes:
{{x-typeahead data=lats name='category_name' selection=myColor}}
Note lats instead of model you originally were passing into the component.
And now, to access the value of data property in the component, you do
`this.get('data')`
which you can just pass in as the source like so:
source: substringMatcher(self.get('data'))
Working solution here
Update
Updating my answer based on your updated question.
OK, so this is getting a little more complicated. You now need more than just one property (latitude) from the object. You need category_name and merchant name.
In addition to mapBy, which just grabs one property out of array, Ember also has map which lets you transform the array into pretty much anything you want to:
lats: function(){
var merchantsArr = this.get('model').map(function(thing){
var category_name = thing.category_name;
return thing.merchants.map(function(merchant){
return {
"name": merchant.name,
"category": category_name
};
});
});
merchantsArr = this.flatten(merchantsArr);
return merchantsArr;
}.property('model')
The code above looks complicated, but it's basically just returning an array of top level objects' merchants accompanied by category_name. Since this is an array of arrays, we will need to flatten it.
Then, inside the component, we need to keep in mind that we are not just passing in an array of strings, but rather we are passing in an array of objects. Therefore, we need to look through object's properties (name and category) for a match
$.each(strs, function(i, str) {
if (substrRegex.test(str.name) || substrRegex.test(str.category)) {
matches.push(str);
}
});
Lastly, to actually display both category and merchant name, you need to tell Typeahead how to do that:
templates: {
suggestion: Handlebars.compile('<p>{{name}} – {{category}}</p>')
}
Working solution here