how to parse and iterate a json using ruby - json

Im a beginner with ruby and learning to iterate and parse json file. The contents inside input.json
[
{
"scheme": "http",
"domain_name": "www.example.com",
"path": "path/to/file",
"fragment": "header2"
},
{
"scheme": "http",
"domain_name": "www.example2.org",
"disabled": true
},
{
"scheme": "https",
"domain_name": "www.stack.org",
"path": "some/path",
"query": {
"key1": "val1",
"key2": "val2"
}
}
]
ho do I parse print the output as:
http://www.example.com/path/to/file#header2
https://www.stack.org/some/path?key1=val1&key2=val2
Any learning references would be very helpful.

Hopefully this code is self-explanatory:
require 'URI'
require 'json'
entries = JSON.parse(File.read('input.json'))
entries.reject { |entry| entry["disabled"] }.each do |entry|
puts URI::Generic.build({
:scheme => entry["scheme"],
:host => entry["domain_name"],
:fragment => entry["fragment"],
:query => entry["query"] && URI.encode_www_form(entry["query"]),
:path => entry["path"] && ("/" + entry["path"])
}).to_s
end
# Output:
# http://www.example.com/path/to/file#header2
# https://www.stack.org/some/path?key1=val1&key2=val2

The first step is to turn this JSON into Ruby data:
require 'json'
data = JSON.load(DATA)
Then you need to iterate over this and reject all those that are flagged as disabled:
data.reject do |entry|
entry['disabled']
end
Which you can chain together with an operation that leverages the URI library to build your output:
require 'uri'
uris = data.reject do |entry|
entry['disabled']
end.map do |entry|
case (entry['scheme'])
when 'https'
URI::HTTPS
else
URI::HTTP
end.build(
host: entry['domain_name'],
path: normalized_path(entry['path']),
query: entry['query'] && URI.encode_www_form(entry['query'])
).to_s
end
#=> ["http://www.example.com/path/to/file", "http://www.stack.org/some/path?key1=val1&key2=val2"]
This requires a function called normalized_path to deal with nil or invalid paths and fix them:
def normalized_path(path)
case (path and path[0,1])
when nil
'/'
when '/'
path
else
"/#{path}"
end
end

Related

Creating json with RUBY looping through SQL Server table

This is a followup to this question:
Ruby create JSON from SQL Server
I was able to create nested arrays in JSON. But I'm struggling with looping through records and appending a file with each record. Also how would I add a root element just at the top of the json and not on each record. "aaSequences" needs to be at the top just once... I also need a comma between each record.
here is my code so far
require 'pp'
require 'tiny_tds'
require 'awesome_print'
require 'json'
class Document
def initialize strategy
#document = strategy
#load helper functions
load "helpers_ruby.rb"
#set environment 'dev', 'qa', or 'production'
load "envconfig_ruby.rb"
end
def StartUP
#document.StartUP
end
def getseqrecord
#document.getseqrecord
end
end
class GetSqlaaSequence
def StartUP
##system "clear" ##linux
system "cls" ##Windows
# create connection to db
$connReportingDB = createReportingxxSqlConn($ms_sql_host, $ms_sql_user, $ms_sql_password, $ms_sql_dbname)
##$currentDateTime = DateTime.now
##pp 'def StartUP ran at: '+$currentDateTime.to_s
end
def getseqrecord
# get the aaaaSequences data
#result = $connReportingDB.execute("SELECT
[jsonFile]
,[id]
,[title]
,[authorIds]
,[name]
,[aminoAcids]
,[schemaId]
,[registryId]
,[namingStrategy]
FROM tablename
")
$aaSequences = Array.new
#i = 0
#result.each do |aaSequence|
jsonFile = aaSequence['jsonFile']
id = aaSequence['id']
title = aaSequence['title']
authorIds = aaSequence['authorIds']
name = aaSequence['name']
aminoAcids = aaSequence['aminoAcids']
schemaId = aaSequence['schemaId']
registryId = aaSequence['registryId']
namingStrategy = aaSequence['namingStrategy']
##end
#hash = Hash[
"jsonFile", jsonFile,
"id", id,
"title", title,
"authorIds", authorIds,
"name", name,
"aminoAcids", aminoAcids,
"schemaId", schemaId,
"registryId", registryId,
"namingStrategy", namingStrategy
]
#filename = jsonFile
jsonFileOutput0 = {:"#{title}" => [{:authorIds => ["#{authorIds}"],:aminoAcids => "#{aminoAcids}",:name => "#{name}",:schemaId => "#{schemaId}",:registryId => "#{registryId}",:namingStrategy => "#{namingStrategy}"}]}
jsonFileOutput = JSON.pretty_generate(jsonFileOutput0)
File.open(jsonFile,"a") do |f|
f.write(jsonFileOutput)
####ad the comma between records...Not sure if this is the best way to do it...
# File.open(jsonFile,"a") do |f|
# f.write(',')
# end
end
$aaSequences[#i] = #hash
#i = #i + 1
###createReportingSqlConn.close
end
end
end
Document.new(GetSqlaaSequence.new).StartUP
#get aaSequences and create json files
Document.new(GetSqlaaSequence.new).getseqrecord
here is a sample of the json it creates so far...
{
"aaSequences": [
{
"authorIds": [
"fff_fdfdfdfd"
],
"aminoAcids": "aminoAcids_data",
"name": "fdfdfddf-555_1",
"schemaId": "5555fdfd5",
"registryId": "5fdfdfdf",
"namingStrategy": "NEW_IDS"
}
]
}{
"aaSequences": [
{
"authorIds": [
"fff_fdfdfdfd"
],
"aminoAcids": "aminoAcids_data",
"name": "fdfdfddf-555_2",
"schemaId": "5555fdfd5",
"registryId": "5fdfdfdf",
"namingStrategy": "NEW_IDS"
}
]
}
and here is an example of what I need it to look like
{
"aaSequences": [
{
"authorIds": [
"authorIds_data"
],
"aminoAcids": "aminoAcids_data",
"name": "name_data",
"schemaId": "schemaId_data",
"registryId": "registryId_data",
"namingStrategy": "namingStrategy_data"
},
{
"authorIds": [
"authorIds_data"
],
"aminoAcids": "aminoAcids_data",
"name": "name_data",
"schemaId": "schemaId_data",
"registryId": "registryId_data",
"namingStrategy": "namingStrategy_data"
}
]
}
You can just do the whole thing in SQL using FOR JSON.
Unfortunately, arrays are not possible using this method. There are anumber of hacks, but the easiest one in your situation is to just append to [] using JSON_MODIFY
SELECT
authorIds = JSON_MODIFY('[]', 'append $', a.authorIds),
[aminoAcids],
[name],
[schemaId],
[registryId],
[namingStrategy]
FROM aaSequences a
FOR JSON PATH, ROOT('aaSequences');
db<>fiddle

Seeding rails project with Json file

I'm at a lost and my searches have gotten me nowhere.
In my seeds.rb file I have the following code
require 'json'
jsonfile = File.open 'db/search_result2.json'
jsondata = JSON.load jsonfile
#jsondata = JSON.parse(jsonfile)
jsondata[].each do |data|
Jobpost.create!(post: data['title'],
link: data['link'],
image: data['pagemap']['cse_image']['src'] )
end
Snippet of the json file looks like this:
{
"kind": "customsearch#result",
"title": "Careers Open Positions - Databricks",
"link": "https://databricks.com/company/careers/open-positions",
"pagemap": {
"cse_image": [
{
"src": "https://databricks.com/wp-content/uploads/2020/08/careeers-new-og-image-sept20.jpg"
}
]
}
},
Fixed jsondata[].each to jasondata.each. Now I'm getting the following error:
TypeError: no implicit conversion of String into Integer
jsondata[] says to call the [] method with no arguments on the object in the jsondata variable. Normally [] would take an index like jsondata[0] to get the first element or a start and length like jsondata[0, 5] to get the first five elements.
You want to call the each method on jsondata, so jsondata.each.
So this is very specific to what you have posted:
require 'json'
file = File.open('path_to_file.json').read
json_data = JSON.parse file
p json_data['kind'] #=> "customsearch#result"
# etc for all the other keys
now maybe the json you posted is just the first element in an array:
[
{}, // where each {} is the json you posted
{},
{},
// etc
]
in which case you will indeed have to iterate:
require 'json'
file = File.open('path_to_file.json').read
json_data = JSON.parse file
json_data.each do |data|
p data['kind'] #=> "customsearch#result"
end

Replace and access values in nested hash/json by path in Ruby

Asking for a advice what would be in your opinion best and simple solution to replace and access values in nested hash or json by path ir variable using ruby?
For example imagine I have json or hash with this kind of structure:
{
"name":"John",
"address":{
"street":"street 1",
"country":"country1"
},
"phone_numbers":[
{
"type":"mobile",
"number":"234234"
},
{
"type":"fixed",
"number":"2342323423"
}
]
}
And I would like to access or change fixed mobile number by path which could be specified in variable like this: "phone_numbers/1/number" (separator does not matter in this case)
This solution is necessary to retrieve values from json/hash and sometimes replace variables by specifying path to it. Found some solutions which can find value by key, but this solution wouldn't work as there is some hashes/json where key name is same in multiple places.
I saw this one: https://github.com/chengguangnan/vine , but it does not work when payload is like this as it is not kinda hash in this case:
[
{
"value":"test1"
},
{
"value":"test2"
}
]
Hope you have some great ideas how to solve this problem.
Thank you!
EDIT:
So I tried code below with this data:
x = JSON.parse('[
{
"value":"test1"
},
{
"value":"test2"
}
]')
y = JSON.parse('{
"name":"John",
"address":{
"street":"street 1",
"country":"country1"
},
"phone_numbers":[
{
"type":"mobile",
"number":"234234"
},
{
"type":"fixed",
"number":"2342323423"
}
]
}')
p x
p y.to_h
p x.get_at_path("0/value")
p y.get_at_path("name")
And got this:
[{"value"=>"test1"}, {"value"=>"test2"}]
{"name"=>"John", "address"=>{"street"=>"street 1", "country"=>"country1"}, "phone_numbers"=>[{"type"=>"mobile", "number"=>"234234"}, {"type"=>"fixed", "number"=>"2342323423"}]}
hash_new.rb:91:in `<main>': undefined method `get_at_path' for [{"value"=>"test1"}, {"value"=>"test2"}]:Array (NoMethodError)
For y.get_at_path("name") got nil
You can make use of Hash.dig to get the sub-values, it'll keep calling dig on the result of each step until it reaches the end, and Array has dig as well, so when you reach that array things will keep working:
# you said the separator wasn't important, so it can be changed up here
SEPERATOR = '/'.freeze
class Hash
def get_at_path(path)
dig(*steps_from(path))
end
def replace_at_path(path, new_value)
*steps, leaf = steps_from path
# steps is empty in the "name" example, in that case, we are operating on
# the root (self) hash, not a subhash
hash = steps.empty? ? self : dig(*steps)
# note that `hash` here doesn't _have_ to be a Hash, but it needs to
# respond to `[]=`
hash[leaf] = new_value
end
private
# the example hash uses symbols as the keys, so we'll convert each step in
# the path to symbols. If a step doesn't contain a non-digit character,
# we'll convert it to an integer to be treated as the index into an array
def steps_from path
path.split(SEPERATOR).map do |step|
if step.match?(/\D/)
step.to_sym
else
step.to_i
end
end
end
end
and then it can be used as such (hash contains your sample input):
p hash.get_at_path("phone_numbers/1/number") # => "2342323423"
p hash.get_at_path("phone_numbers/0/type") # => "mobile"
p hash.get_at_path("name") # => "John"
p hash.get_at_path("address/street") # => "street 1"
hash.replace_at_path("phone_numbers/1/number", "123-123-1234")
hash.replace_at_path("phone_numbers/0/type", "cell phone")
hash.replace_at_path("name", "John Doe")
hash.replace_at_path("address/street", "123 Street 1")
p hash.get_at_path("phone_numbers/1/number") # => "123-123-1234"
p hash.get_at_path("phone_numbers/0/type") # => "cell phone"
p hash.get_at_path("name") # => "John Doe"
p hash.get_at_path("address/street") # => "123 Street 1"
p hash
# => {:name=>"John Doe",
# :address=>{:street=>"123 Street 1", :country=>"country1"},
# :phone_numbers=>[{:type=>"cell phone", :number=>"234234"},
# {:type=>"fixed", :number=>"123-123-1234"}]}

transform JSON to hashmap in Ruby

I am trying to convert a json file which contain object and array to a JSON file.
Below is the JSON file
{
"localbusiness":{
"name": "toto",
"phone": "+11234567890"
},
"date":"05/02/2016",
"time":"5:00pm",
"count":"4",
"userInfo":{
"name": "John Doe",
"phone": "+10987654321",
"email":"john.doe#unknown.com",
"userId":"user1234333"
}
}
my goal is to save this is a database such as MongoId. I would like to use map to get something like:
localbusiness_name => "toto",
localbusiness_phone => "+11234567890",
date => "05/02/2016",
...
userInfo_name => "John Doe"
...
I have tried map but it's not splitting the array of local business or userInfo
def format_entry
ps = #params.map do | h |
ps.merge!(h)
##logger.info("entry #{h}")
end
##logger.info("formatting the data #{ps}")
ps
end
I do not really how to parse each entry and rebuild the name
It looks like to me you are trying to "flatten" the inner hashes into one big hash. Flatten being incorrect because you want to prepend the hash's key to the sub-hash's key. This will require looping through the hash, and then looping again through each sub hash. This code example will only work if you have 1 layer deep. if you have multiple layers, then I would suggest making two methods, or a recursive method.
#business = { # This is a hash, not a json blob, but you can take json and call JSON.parse(blob) to turn it into a hash.
"localbusiness":{
"name": "toto",
"phone": "+11234567890"
},
"date":"05/02/2016",
"time":"5:00pm",
"count":"4",
"userInfo":{
"name": "John Doe",
"phone": "+10987654321",
"email":"john.doe#unknown.com",
"userId":"user1234333"
}
}
#squashed_business = Hash.new
#business.each do |k, v|
if v.is_a? Hash
v.each do |key, value|
#squashed_business.merge! (k.to_s + "_" + key.to_s) => value
end
else
#squashed_business.merge! k => v
end
end
I noticed that you are getting "unexpected" outcomes when enumerating over a hash #params.each { |h| ... } because it gives you both a key and a value. Instead you want to do #params.each { |key, value| ... } as I did in the above code example.

Sensu checks results event data

I am working on Sensu. I have installed sensu in CentOS. I need to get the event messages which is generated by Sensu checks.I have added some of the sensu community plugins like check-procs.rb,check-load.rb,check-banner.rb, metrics-ebs-volume.rb etc. I have written some handler files to event handle these .rb files. I am getting events in sensu-server.log.
Example:
{"timestamp":"2016-08-10T07:32:08.000003+0000","level":"info","message":"publishing check request","payload":{"name":"swap-free","issued":1470814327,"command":"check-swap.sh 20 10"},"subscribers":["base_centos_monitoring"]}
I have written a ruby file "nephele_events_handler.rb" which sends events messages through rest call to another server. The ruby file is in the location "/etc/sensu/handlers/". I am reading events from STDIN.read, i have read from official sensu documentation that events will be stored inside STDIN.
#!/opt/sensu/embedded/bin/ruby
require "#{File.dirname(__FILE__)}/base"
require 'rubygems'
require 'json'
require 'uri'
require 'net/http'
require 'net/https'
require 'json'
class RunProcs < BaseHandler
def payload_check
#Read event data
sensuhash = "{ \"SensuMessage\"" + ":"
braces = "}"
s = sensuhash.to_s
event = JSON.parse(STDIN.read, :symbolize_names => true)
eventPayload = event.to_json
sensujson = s + eventPayload + braces
uri = URI.parse("https://localhost:8080/Processor/services/sourceEvents/requestMsg")
https = Net::HTTP.new(uri.host,uri.port)
https.use_ssl = false
req = Net::HTTP::Post.new(uri.path, initheader = {'Content-Type' =>'application/json'})
req.body = "[ #{sensujson} ]"
res = https.request(req)
end
info = RunProcs.new
info.payload_check
end
Am writing handler json file "processor.json" inside the location "/etc/sensu/conf.d/handlers".
{
"handlers": {
"nephele_processor": {
"type": "pipe",
"command": "nephele_events_handler.rb"
}
}
}
But the issue am facing is am only getting events from 'check-procs'
{"client":{"address":"10.81.1.105","subscriptions":["base_centos","base_chef-client","python","base_centos_monitoring","base_centos_monitoring_metrics","sensu_client","base_aws","base_aws_monitoring","sensu_master","all"],"name":"ip-localhost.internal","hostname":"ip-localhost","version":"0.25.3","timestamp":1470896756},"check":{"command":"check-procs.rb --pattern='chef-client' -W=1","subscribers":["base_centos_monitoring"],"handlers":["base_with_jira"],"interval":60,"team":"ops","aggregate":true,"occurrences":3,"refresh":300,"ticket":true,"name":"process-chef-client","issued":1470896771,"executed":1470896771,"duration":0.864,"output":"CheckProcs CRITICAL: Found 0 matching processes; cmd /chef-client/\n","status":2,"type":"standard","history":["2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2","2"],"total_state_change":0},"occurrences":19879,"action":"create","timestamp":1470896772,"id":"dc2b0698-dbac-416d-a9ae-42aa09d53cc3","last_state_change":1469690268,"last_ok":null}
The Check which is getting executed
{
"checks": {
"process-chef-client": {
"command": "check-procs.rb --pattern='chef-client' -W=1",
"subscribers": [
"base_centos_monitoring"
],
"handlers": [
"base_with_jira"
],
"interval": 60,
"team": "ops",
"aggregate": true,
"occurrences": 3,
"interval": 60,
"refresh": 300,
"ticket": true
}
}
}
base_with_jira.json
{
"handlers": {
"base_with_jira": {
"type": "set",
"handlers": [
"jira",
"nephele_processor"
],
"config": "http://apache.enron.nephele.solutions/uchiwa"
}
}
}
I am not getting events from other plugins. Can you explain what i have to do for this.