Better way to extract json map into struct - json

I'm new to elixir and I want to parse a json file. One of the parts is a question answer array of objects.
[
{
"questionId":1,
"question":"Information: Personal Information: First Name",
"answer":"Joe"
},
{
"questionId":3,
"question":"Information: Personal Information: Last Name",
"answer":"Smith"
},
...
]
I know what questionId's I want and I'm going to make a map for 1 = First Name, 2 = Last Name.
But currently I'm doing the following to put the data into the struct.
defmodule Student do
defstruct first_name: nil, last_name: nil, student_number: nil
defguard is_first_name(id) when id == 1
defguard is_last_name(id) when id == 3
defguard is_student_number(id) when id == 7
end
defmodule AFMC do
import Student
#moduledoc """
Documentation for AFMC.
"""
#doc """
Hello world.
## Examples
iex> AFMC.hello
:world
"""
def main do
get_json()
|> get_outgoing_applications
end
def get_json do
with {:ok, body} <- File.read("./lib/afmc_import.txt"),
{:ok,body} <- Poison.Parser.parse(body), do: {:ok,body}
end
def get_outgoing_applications(map) do
{:ok,body} = map
out_application = get_in(body,["outgoingApplications"])
Enum.at(out_application,0)
|> get_in(["answers"])
|> get_person
end
def get_person(answers) do
student = Enum.reduce(answers,%Student{},fn(answer,acc) ->
if Student.is_first_name(answer["questionId"]) do
acc = %{acc | first_name: answer["answer"]}
end
if Student.is_last_name(answer["questionId"]) do
acc = %{acc | last_name: answer["answer"]}
end
if Student.is_student_number(answer["questionId"]) do
acc = %{acc | student_number: answer["answer"]}
end
acc
end)
IO.inspect "test"
s
end
end
I'm wondering what is a better way to do get_person with out having to do if statements. If I know I will be mapping 1 to questionId 1 in the array of objects.
The data will then be saved into a DB.
Thanks

I'd store a mapping of id to field name. With that you don't need any if inside the reduce. Some pattern matching will also make it unnecessary to do answer["questionId"] etc.
defmodule Student do
defstruct first_name: nil, last_name: nil, student_number: nil
#fields %{
1 => :first_name,
3 => :last_name,
7 => :student_number
}
def parse(answers) do
Enum.reduce(answers, %Student{}, fn %{"questionId" => id, "answer" => answer}, acc ->
%{acc | #fields[id] => answer}
end)
end
end
IO.inspect(
Student.parse([
%{"questionId" => 1, "question" => "", "answer" => "Joe"},
%{"questionId" => 3, "question" => "", "answer" => "Smith"},
%{"questionId" => 7, "question" => "", "answer" => "123"}
])
)
Output:
%Student{first_name: "Joe", last_name: "Smith", student_number: "123"}
Edit: to skip ids not present in the map, change:
%{acc | #fields[id] => answer}
to:
if field = #fields[id], do: %{acc | field => answer}, else: acc

Related

Karate: How to create a json object/map from 2 arrays and merge them?

I am having 2 arrays
array1 = ['a','b']
array2 = [1,2]
I want to merge these 2 arrays and convert them to map like below:
[
{
"firstparam": 'a'
"secondparam": 1
},
{
"firstparam": 'b'
"secondparam": 2
}
]
I am trying this code:
* def map1 = array1
* def map1 = karate.mapWithKey(map1, 'firstparam')
* def map2 = array2
* def map2 = karate.mapWithKey(map2, 'secondparam')
this code is creating map1 & map2. now I want to merge these 2 maps in the above format. how to do it?
basically, i want to send this map to a feature file which is expected 2 parameters.
* def result = karate.call('*.feature', map)
'*.feature' is expecting 2 parameters per call i.e, firstparam & secondparam
Here you go:
* def array1 = ['a', 'b']
* def array2 = [1, 2]
* def array3 = array1.map((x, i) => ({ firstparam: x, secondparam: array2[i] }))
* match array3 == [{ firstparam: 'a', secondparam: 1 }, { firstparam: 'b', secondparam: 2 }]

Ruby Threads and MySQL connection

I'm trying the Ruby Threads, i have a simple script that need to iterate a json for getting certain data, all works fine but in one moment the shell shows:
`_query': This connection is in use by: #<Thread:0x00007f9c73973eb8#thread.rb:62 sleep> (Mysql2::Error)
How can i close that connection that i need for the data.
And the most important, the Threads are actually in the right way?
This is the script, it will run with a crontab:
require 'firebase'
require 'conekta'
require 'json'
require 'savon'
require "crack"
require 'active_support/core_ext/hash' #from_xml
require 'nokogiri'
require 'xmlsimple'
require 'mysql2'
class Cron
def generate_activation_code(size = 10)
charset = %w{ 1 2 3 4 5 6 7 8 9 A B C D E F G H I J K L M N O P Q R S T U V W X Y Z}
(0...size).map{ charset.to_a[rand(charset.size)] }.join
end
def construct()
base_uri = 'FIREBASE_URL'
file = File.open("FIREBASE_CREDENTIALS", "rb")
firebase = Firebase::Client.new(base_uri, file.read)
Conekta.locale = :es
Conekta.api_key = 'MY_KEY'
#response = firebase.get('users', nil)
#client = Savon.client(wsdl: 'MY_URL', ntlm: ["user", "pass"] , :convert_request_keys_to => :camelcase )
#client_mysql = Mysql2::Client.new(:host => "localhost", :username => "root", :password => "", :database => "masaldo_api")
end
def get_comision()
last_validity = #client_mysql.query("SELECT comision * 100 as comision FROM configuration")
last_validity.each do |validityr|
#comision = validityr["comision"]
end
end
def create_transaction(sku, token, phone, userid, card)
validity = #client_mysql.query("SELECT precio * 100 as precio_total, vigencia, descripcion, precio as precio_base FROM bluesoft_services_validity WHERE sku='#{sku}'")
validity.each do |row|
#vigencia = row["vigencia"]
#descipcion = row["descripcion"]
#precio = row["precio_total"]
#precio_base = row["precio_base"].to_i
end
if #vigencia.to_i > 0
last_current = #client_mysql.query("SELECT * FROM transactions WHERE number='#{phone}' ORDER BY trandate DESC LIMIT 1")
last_current.each do |last|
#trandate = last["trandate"]
#trandate_result = #trandate.strftime("%Y%m%d %H:%M:%S")
end
end
#last_with_validty = (#trandate + (#vigencia).to_i.day).strftime("%Y-%m-%d")
#today = (Time.now).strftime("%Y-%m-%d")
if #last_with_validty == #today
conekta_charges = Conekta::Order.create({
:currency => "MXN",
:customer_info => {
:customer_id => user['customer_id']
},
:line_items => [{
:name => #descipcion,
:unit_price => #precio.to_i,
:quantity => 1
},
{
:name => 'Comision de Recarga',
:unit_price => #comision.to_i,
:quantity => 1
}],
:charges => [{
:payment_method => {
:type => "card",
:payment_source_id => user['fav_card']
}
}]
})
if conekta_charges['payment_status'] == 'paid'
begin
response = #client.call(:venta, message: { 'sku' => 'TELCPA100MXN', 'fechaLocal' => '20180117 14:55:00', 'referencia' => '818181818181', 'monto' => '100', 'id_cadena' => '30', 'id_tienda' => '30', 'id_terminal' => '1', 'folio' => 'LUCOPCIHOW' })
parameters = response.body
parameters.each do |response, data|
if data[:return][:respuesta][:codigo_respuesta] == 0
puts data[:return][:respuesta]
else
puts data[:return][:respuesta]
end
end
rescue Exception => e
puts e.message
puts e.backtrace.inspect
end
end
end
end
def init()
threads = []
hash = #response.body
hash.each do |token , user|
threads << Thread.new do
#Check if user is current for transaction if not need to check agenda
if user['is_current']
self.create_transaction(user['sku'], token, user['phoneNumber'], user['customer_id'], user['fav_card'])
user['addressBook'].each do |userid , user_address_book|
if user['is_current']
replacements = { '+521' => '' }
phone_number = user_address_book['phoneNumber'].gsub(Regexp.union(replacements.keys), replacements)
self.create_transaction(user_address_book['sku'], token, phone_number, user['customer_id'], user_address_book['fav_card'])
end
end
end
end
end
threads.each { |t| t.join }
end
end
classCron = Cron.new()
classCron.construct()
classCron.get_comision()
classCron.init()
Regards
When doing multi-threaded code in Ruby you'll need to be careful about not sharing resources like database connections between threads unless the driver makes it abundantly clear that kind of operation is supported. The ones I'm familiar with don't, and Mysql2 is not thread safe that way.
You can use Thread[:db] to store a local database connection per-thread, like:
def db
Thread.current[:db] ||= Mysql2::Client.new(...)
end
Where then you can refer to it like this:
db.query(...)
That will automatically instantiate the connection as required.
It's worth noting that mysql2 is a low-level driver and isn't very pleasant to use. A higher level abstraction like Sequel provides a number of significant benefits: Migrations, (optional) model layer, and a very robust query builder with support for placeholder values and easy escaping.

How to get first element from Any

My Json is a list of objects. I want to get the first one, but Any is making it difficult:
scala> import scala.util.parsing.json._
import scala.util.parsing.json._
scala> val str ="""
| [
| {
| "UserName": "user1",
| "Tags": "one, two, three"
| },
| {
| "UserName": "user2",
| "Tags": "one, two, three"
| }
| ]""".stripMargin
str: String =
"
[
{
"UserName": "user1",
"Tags": "one, two, three"
},
{
"UserName": "user2",
"Tags": "one, two, three"
}
]"
scala> val parsed = JSON.parseFull(str)
parsed: Option[Any] = Some(List(Map(UserName -> user1, Tags -> one, two, three), Map(UserName -> user2, Tags -> one, two, three)))
scala> parsed.getOrElse(0)
res0: Any = List(Map(UserName -> user1, Tags -> one, two, three), Map(UserName -> user2, Tags -> one, two, three))
scala> parsed.getOrElse(0)(0)
<console>:13: error: Any does not take parameters
parsed.getOrElse(0)(0)
How do I get the first element?
You need to pattern match the result(Option[Any]) to List[Map[String, String]].
1) Patter match example,
scala> val parsed = JSON.parseFull("""[{"UserName":"user1","Tags":"one, two, three"},{"UserName":"user2","Tags":"one, two, three"}]""")
scala> parsed.map(users => users match { case usersList : List[Map[String, String]] => usersList(0) case _ => Option.empty })
res8: Option[Equals] = Some(Map(UserName -> user1, Tags -> one, two, three))
better pattern match,
scala> parsed.map(_ match { case head :: tail => head case _ => Option.empty })
res13: Option[Any] = Some(Map(UserName -> user1, Tags -> one, two, three))
2) Or else you can cast the result (Option[Any]) but not recommended(as cast might throw ClassCastException),
scala> parsed.map(_.asInstanceOf[List[Map[String, String]]](0))
res10: Option[Map[String,String]] = Some(Map(UserName -> user1, Tags -> one, two, three))

Scala : How to do GroupBy sum for String values?

I have RDD[Row] :
|---itemId----|----Country-------|---Type----------|
| 11 | US | Movie |
| 11 | US | TV |
| 101 | France | Movie |
How to do GroupBy itemId so that I can save the result as List of json where each row is separate json object(each row in RDD) :
{"itemId" : 11,
"Country": {"US" :2 },"Type": {"Movie" :1 , "TV" : 1} },
{"itemId" : 101,
"Country": {"France" :1 },"Type": {"Movie" :1} }
RDD :
I tried :
import com.mapping.data.model.MappingUtils
import com.mapping.data.model.CountryInfo
val mappingPath = "s3://.../"
val input = sc.textFile(mappingPath)
The input is list of jsons where each line is json which I am mapping to the POJO class CountryInfo using MappingUtils which takes care of JSON parsing and conversion:
val MappingsList = input.map(x=> {
val countryInfo = MappingUtils.getCountryInfoString(x);
(countryInfo.getItemId(), countryInfo)
}).collectAsMap
MappingsList: scala.collection.Map[String,com.mapping.data.model.CountryInfo]
def showCountryInfo(x: Option[CountryInfo]) = x match {
case Some(s) => s
}
val events = sqlContext.sql( "select itemId EventList")
val itemList = events.map(row => {
val itemId = row.getAs[String](1);
val çountryInfo = showTitleInfo(MappingsList.get(itemId));
val country = if (countryInfo.getCountry() == 'unknown)' "US" else countryInfo.getCountry()
val type = countryInfo.getType()
Row(itemId, country, type)
})
Can some one let me know how can I achieve this ?
Thank You!
I can't afford the extra time to complete this, but can give you a start.
The idea is that you aggregate the RDD[Row] down into a single Map that represents your JSON structure. Aggregation is a fold that requires two function parameters:
seqOp How to fold a collection of elements into the target type
combOp How to merge two of the target types.
The tricky part comes in combOp while merging, as you need to accumulate the counts of values seen in the seqOp. I have left this as an exercise, as I have a plane to catch! Hopefully someone else can fill in the gaps if you have trouble.
case class Row(id: Int, country: String, tpe: String)
def foo: Unit = {
val rows: RDD[Row] = ???
def seqOp(acc: Map[Int, (Map[String, Int], Map[String, Int])], r: Row) = {
acc.get(r.id) match {
case None => acc.updated(r.id, (Map(r.country, 1), Map(r.tpe, 1)))
case Some((countries, types)) =>
val countries_ = countries.updated(r.country, countries.getOrElse(r.country, 0) + 1)
val types_ = types.updated(r.tpe, types.getOrElse(r.tpe, 0) + 1)
acc.updated(r.id, (countries_, types_))
}
}
val z = Map.empty[Int, (Map[String, Int], Map[String, Int])]
def combOp(l: Map[Int, (Map[String, Int], Map[String, Int])], r: Map[Int, (Map[String, Int], Map[String, Int])]) = {
l.foldLeft(z) { case (acc, (id, (countries, types))) =>
r.get(id) match {
case None => acc.updated(id, (countries, types))
case Some(otherCountries, otherTypes) =>
// todo - continue by merging countries with otherCountries
// and types with otherTypes, then update acc
}
}
}
val summaryMap = rows.aggregate(z) { seqOp, combOp }

Why can I not update the foreign key of an object in a 1:1 association?

I have two models, User and Profile.
A User has_one Profile and a Profile belongs_to User.
Correspondingly, the Profile model has a user_id attribute.
The association works:
p = Profile.first
=> #<Profile id: 1, name: "Jack", ... , user_id: 1>
u = User.first
=> #<User id: 1, email: "jack#example.com", ... >
u.profile.id
=> 1
p.user.id
=> 1
p.user == u
=> true
u.profile == p
=> true
I can set the user_id field on a Profile directly:
p.user_id = 2
=> 2
p.save!
=> true
p.user_id
=> 2
But why can I not set the user_id like this:
u.profile.user_id = 2
=> 2
u.profile.save!
=> 2
u.profile.user_id
=> 1
You must refresh u.profile object. Try this:
u.profile.user_id = 2
=> 2
u.profile.save!
=> 2
u.profile.reload.user_id
=> 2
This is because original profile object is still loaded on memory in u.
Hope this help :)