How to store sequence of JsValue into a ArrayBuffer - json

I have the following json object what contain a list of json objects. Each of these objects contain informations about a station object.
"stations": [
{
"cryptName": "Tv_Gara Nord 1",
"easyName": "Gara de Nord",
"lat": 45.750287,
"lng": 21.207498
},
{
"cryptName": "Tv_R Carol 1",
"easyName": "Regele Carol I",
"lat": 45.745068,
"lng": 21.211034
},
{
"cryptName": "Tv_Mocioni 1",
"easyName": "Piata Mocioni (Sinaia)",
"lat": 45.746343,
"lng": 21.21551
}]
I've tried this and I am unable to convert this to a ArrayBuffer with format:
ArrayBuffer(List("cryptName", "easyName", "lat", "lng"), ...)
[Line 73 .. 77] The problem is with (js \\\ "cryptName")(0) ... (js \\\ "lng")(0).
How to store correctly all of these stations not a single one into my ArrayBuffer?
package backend
import scala.concurrent.duration._
import scala.collection.immutable.Seq
import scala.concurrent.forkjoin.ThreadLocalRandom
import akka.actor.{ ActorRef, Props }
import play.api.libs.json.Json
import play.extras.geojson.{ LineString, LatLng, FeatureCollection }
import play.api.Logger
import actors.GeoJsonBot
import java.net.URL
import akka.actor.Actor
import scala.io.Source
import akka.cluster.Cluster
import play.api.libs.json.JsValue
import org.apache.commons.lang3.Validate
object BotManager
{
def props(regionManagerClient: ActorRef, data: Seq[URL]): Props = Props(new BotManager(regionManagerClient, data))
private case object Tick
}
/**
* Loads and starts GeoJSON bots
*/
class BotManager(regionManagerClient: ActorRef, data: Seq[URL]) extends Actor
{
import BotManager._
var total = 0
val max = Settings(context.system).TotalNumberOfBots
import context.dispatcher
val tickTask = context.system.scheduler.schedule(1.seconds, 3.seconds, self, Tick)
val port = Cluster(context.system).selfAddress.port.get
override def postStop(): Unit = tickTask.cancel()
def receive =
{
case Tick if total >= max => tickTask.cancel()
case Tick =>
val totalBefore = total
val originalTrail = total == 0
println("NODE: " + data)
data.zipWithIndex.foreach
{
case (url, id) =>
val json = Json.parse(Source.fromURL(url).mkString)
println("URL: " + url + "ID: " + id)
Json.fromJson[FeatureCollection[LatLng]](json).fold(
{
invalid =>
Logger.error("Error loading geojson bot: " + invalid)
}, valid => valid.features.zipWithIndex.map
{
feature => feature._1.geometry match
{
case route: LineString[LatLng] if total < max => total += 1
val userId = "bot-" + total + "-" + port + "-" + id + "-" + feature._1.id.getOrElse(feature._2) + "-" + feature._1.properties.flatMap(js => (js \ "name").asOpt[String]).getOrElse("") + feature._1.properties.flatMap(js => (js \ "segment").asOpt[String]).getOrElse("")
val stations = feature._1.properties.flatMap(js => (js \ "stations").asOpt[JsValue])
import play.api.libs.json.JsObject
import play.api.libs.json.JsArray
var stationss = feature._1.properties.flatMap(js => (js \ "stations").asOpt[JsValue])
val crypt = stationss.flatMap(js => (js \\ "cryptName")(0).asOpt[JsValue])
val ename = stationss.flatMap(js => (js \\ "easyName")(0).asOpt[JsValue])
val lat = stationss.flatMap(js => (js \\ "lat")(0).asOpt[JsValue])
val lng = stationss.flatMap(js => (js \\ "lng")(0).asOpt[JsValue])
case class Station(cryptName: String, easyName: String, lat: Float, lng: Float)
implicit val readsStation = Json.reads[Station]
case class Stations(stations: List[Station])
implicit val readsStations = Json.reads[Stations]
val stations = Json.parse(text).as[Stations].stations.map(s => List(s.cryptName, s.easyName, s.lat, s.lng)).to[ArrayBuffer]
println("List: [" + crypt.get + ", " + ename.get + ", " + lat.get + ", " + lng.get + "]")
val offset =
if (originalTrail) (0.0, 0.0)
else (ThreadLocalRandom.current.nextDouble() * 15.0,
ThreadLocalRandom.current.nextDouble() * -30.0)
context.actorOf(GeoJsonBot.props(route, stations, offset, userId, regionManagerClient))
case other =>
}
})
}
println("Started " + (total - totalBefore) + " bots, total " + total)
}
}

I'd do it like this:
case class Station(cryptName: String, easyName: String, lat: Float, lng: Float)
implicit val readsStation = Json.reads[Station]
case class Stations(stations: List[Station])
implicit val readsStations = Json.reads[Stations]
(assuming JSON is { stations: [ ... ] })
Then use the following:
val stations = Json.parse(text).as[Stations].stations
.map(s => List(s.cryptName, s.easyName, s.lat, s.lng)).to[ArrayBuffer])
which'll produce the following (given your sample):
ArrayBuffer(List(Tv_Gara Nord 1, Gara de Nord, 45.750286, 21.207499), List(Tv_R Carol 1, Regele Carol I, 45.745068, 21.211035), List(Tv_Mocioni 1, Piata Mocioni (Sinaia), 45.74634, 21.21551))

Use Reads\Writes macro.
Create case classes, something like,
case class Station(cryptName: String, easyName: String, lat: String, long: String)
case class Stations(stations: Seq[Station])
Create reads,
import play.api.libs.json._
import play.api.libs.functional.syntax._
implicit val stationReads = Json.reads[Station]
implicit val stationsReads = Json.reads[Stations]
And do conversion,
val json: JsValue = Json.parse(jsonString)
Json.fromJson[Stations](json) match {
case JsSuccess(t, _) => ???
case any: JsError => ???
}
Go through read/write combinators and read/write macros

Related

Scala JSON Parsing Nested Array into Case Class

I'm using the standard Scala json parsing and I am running into issues with the nested arrays.
Here is sample JSON:
{
"_id": "id_here",
"_rev": "rev_here",
"RandomHosts": [
"randomhosts_1",
"randomhosts_2",
"randomhosts_3",
"randomhosts_4"
],
"FirstObject": {
"Host": "ActualHost",
"Port": 8888,
"DB": 0,
"WFMDB": 1,
"ETLDB": 2,
"HostListPrefix": "Dev1",
"ExtraHostsBands": [
{
"Host": "dev2",
"Port": 2222,
"DB": 0,
"WFMDB": 1,
"ETLDB": 2,
"HostListPrefix": "Dev2"
},
{
"Host": "dev3",
"Port": 3333,
"DB": 0,
"WFMDB": 1,
"ETLDB": 3,
"HostListPrefix": "Dev3"
}
],
"RandomObject":{}
}
}
// I HAVE OTHER IMPORTS AS WELL
import scala.util.parsing.json._;
case class BandClass (
Host: String,
Port:Int,
DB:Int,
WFMDB:Int,
ETLDB:Int,
HostListPrefix:String
);
var jsonString = "<myjson>";
val bandconfig = JSON.parseFull(jsonString);
// THIS WORKS PERFECT AND GIVES ME JOINED STRING
val random_hosts = bandconfig.get.asInstanceOf[Map[String, Any]]("RandomHosts").asInstanceOf[List[String]].mkString(",");
//THIS ALSO WORKS PERFECT AND GIVES ME HOST AND PORT
val firstObject = bandconfig.get.asInstanceOf[Map[String, Any]]("FirstObject").asInstanceOf[Map[String, Any]];
val firstObjectHost = firstObject("Host").asInstanceOf[String]
val firstObjectPort = firstObject("Port").asInstanceOf[Double].toInt
//THIS IS WHERE EVERYTHING FALLS APART
val extraBands = firstObject("ExtraHostBands").asInstanceOf[List[BandClass]]
//EVEN THIS DOESNT WORK
val extraBands2 = firstObject("ExtraHostBands").asInstanceOf[Map[String, Any]]
Caused by: java.lang.ClassCastException: scala.collection.immutable.HashMap$HashTrieMap cannot be cast to $BandClass
I'm not sure how to force that nested json array into my case class. Id even settle for a map or seq or anything I could iterate over to get the host/ports out of the ExtraHostBand json objects.
Can anyone point me in the correct direction to get that json array into case class? I also have access to play-json but cant seem to figure that out either.
Ended up going to play-json and it worked really well. Here is a solution in case someone needs this in the future:
import play.api.libs.json.Json;
import play.api.libs.json;
import play.api.libs.json.Writes;
import play.api.libs.json._;
import play.api.libs._;
import play.api.libs.functional.syntax._;
import play.api.libs.json.Reads._;
case class BandClass (
Host: String,
Port:Int,
DB:Int,
WFMDB:Int,
ETLDB:Int,
HostListPrefix:String
)
case class FirstObject (
Host: String,
Port:Int,
DB:Int,
WFMDB:Int,
ETLDB:Int,
HostListPrefix:String,
ExtraHostsBands: List[BandClass]
)
case class RawConfig (
_id: String,
_rev: String,
RandomHosts: List[String],
FirstObject: FirstObject
)
implicit val bandClassFormat = Json.format[BandClass];
implicit val firstObjectFormat = Json.format[FirstObject];
implicit val rawConfigFormat = Json.format[RawConfig];
val playJsonParse = Json.parse(<myjson>).as[RawConfig];
println("playJSON ID " + playJsonParse._id)
println("playJSON REV " + playJsonParse._rev)
playJsonParse.FirstObject.ExtraHostBands.foreach
{
case(r) => {
println("Host " + r.Host);
println("Host Prefix " + r.HostListPrefix);
println("ETLDB " + r.ETLDB);
}
}

How to add each element of Vec?

For example, let's assume that I have following data:
class VectorElemAdd extends Module {
val io = IO (new Bundle {
val input = Input(Vec(64, UInt(64.W)))
val out = Output(UInt(64.W))
})
/*
if the vector data is : 0, 1, 2, 3, 4, 5, ..., 63,
I have to add each element : 0 + 1 + 2 + ... + 63 by indexing Vec,
for example : input(0) + input(1) + ... + input(63),
But, it needs kind of for loop to add all elements of Vec
and the final output(io.out) will be the input(0) + input(1) + ... +
input(63)
*/
The things that I need to do are described in the comment. Is it possible to such kind of operation easily?(for example, using for loop or whatever)
This particular problem is very easy to represent in Scala.
class VectorElemAdd extends Module {
val io = IO (new Bundle {
val input = Input(Vec(64, UInt(64.W)))
val out = Output(UInt(64.W))
})
io.out := io.input.reduce(_ + _)
}

Using ScalaCheck with PeekPokeTester

Here is chisel3 test that uses ScalaCheck to perform property checking on a simple combinational circuit.
package stackoverflow
import org.scalatest.{ Matchers, FlatSpec, GivenWhenThen}
import org.scalacheck.{ Properties, Gen, Arbitrary}
import org.scalacheck.Prop.{ forAll, AnyOperators, collect}
import chisel3._
import firrtl_interpreter.InterpretiveTester
object G {
val width = 8
}
class Add extends Module {
val io = IO(new Bundle {
val a = Input(UInt(G.width.W))
val b = Input(UInt(G.width.W))
val o = Output(UInt(G.width.W))
})
io.o := io.a + io.b
}
class AddTester {
val s = chisel3.Driver.emit( () => new Add)
val tester = new InterpretiveTester( s)
def run( a : Int, b : Int) = {
val result = (a + b) & ((1 << G.width)-1)
tester.poke( s"io_a", a)
tester.poke( s"io_b", b)
tester.peek( s"io_o") ?= result
}
}
object AddTest extends Properties("Add") {
val t = new AddTester
val gen = Gen.choose(0,(1 << G.width)-1)
property("Add") = forAll( gen, gen) {
case (a:Int,b:Int) => t.run( a, b)
}
}
This uses the firrtl interpreter directly. Does anyone know how to do something similar using the PeekPokeTester so I can use the verilator and vcs backends as well?
Is this close to what you have in mind? It's a lot more scalatesty in form. I haven't been able to get the gen stuff working here (there is some kind of weird interaction with chisel), and I'm more familiar with FreeSpec so I started with it. I also threw a printf and a println in so you could believe it was working. This works with the interpreter backend as well.
import org.scalatest.FreeSpec
import org.scalacheck.Prop.AnyOperators
import chisel3._
import chisel3.iotesters.PeekPokeTester
class Add2 extends Module {
val io = IO(new Bundle {
val a = Input(UInt(G.width.W))
val b = Input(UInt(G.width.W))
val o = Output(UInt(G.width.W))
})
io.o := io.a + io.b
printf("%d = %d + %d\n", io.o, io.a, io.b)
}
class ScalaTestyTester extends FreeSpec {
"scalatest verilator test" in {
iotesters.Driver.execute(Array("--backend-name", "verilator"), () => new Add2) { c =>
new PeekPokeTester(c) {
for(_ <- 0 until 10) {
val a = BigInt(G.width, scala.util.Random)
val b = BigInt(G.width, scala.util.Random)
println(s"testing a = $a b = $b")
val result = (a + b) & ((1 << G.width) - 1)
poke(c.io.a, a)
poke(c.io.b, b)
step(1)
peek(c.io.o) ?= result
}
}
}
}
}

Compare json equality in Scala

How can I compare if two json structures are the same in scala?
For example, if I have:
{
resultCount: 1,
results: [
{
artistId: 331764459,
collectionId: 780609005
}
]
}
and
{
results: [
{
collectionId: 780609005,
artistId: 331764459
}
],
resultCount: 1
}
They should be considered equal
You should be able to simply do json1 == json2, if the json libraries are written correctly. Is that not working for you?
This is with spray-json, although I would expect the same from every json library:
import spray.json._
import DefaultJsonProtocol._
Welcome to Scala version 2.10.4 (OpenJDK 64-Bit Server VM, Java 1.7.0_51).
Type in expressions to have them evaluated.
Type :help for more information.
scala> val json1 = """{ "a": 1, "b": [ { "c":2, "d":3 } ] }""".parseJson
json1: spray.json.JsValue = {"a":1,"b":[{"c":2,"d":3}]}
scala> val json2 = """{ "b": [ { "d":3, "c":2 } ], "a": 1 }""".parseJson
json2: spray.json.JsValue = {"b":[{"d":3,"c":2}],"a":1}
scala> json1 == json2
res1: Boolean = true
Spray-json uses an immutable scala Map to represent a JSON object in the abstract syntax tree resulting from a parse, so it is just Map's equality semantics that make this work.
You can also use scalatest-json
Example:
it("should fail on slightly different json explaining why") {
val input = """{"someField": "valid json"}""".stripMargin
val expected = """{"someField": "different json"}""".stripMargin
input should matchJson(expected)
}
When the 2 jsons doesn't match, a nice diff will be display which is quite useful when working with big jsons.
Can confirm that it also works just fine with the Jackson library using == operator:
val simpleJson =
"""
|{"field1":"value1","field2":"value2"}
""".stripMargin
val simpleJsonNode = objectMapper.readTree(simpleJson)
val simpleJsonNodeFromString = objectMapper.readTree(simpleJsonNode.toString)
assert(simpleJsonNode == simpleJsonNodeFromString)
spray-json is definitely great, but I use Gson since I already had dependency on Gson library on my project. I am using these in my unit tests, works well for simple json.
import com.google.gson.{JsonParser}
import org.apache.flume.event.JSONEvent
import org.scalatest.FunSuite
class LogEnricherSpec extends FunSuite {
test("compares json to json") {
val parser = new JsonParser()
assert(parser.parse("""
{
"eventType" : "TransferItems",
"timeMillis" : "1234567890",
"messageXml":{
"TransferId" : 123456
}
} """.stripMargin)
==
parser.parse("""
{
"timeMillis" : "1234567890",
"eventType" : "TransferItems",
"messageXml":{
"TransferId" : 123456
}
}
""".stripMargin))
}
Calling the method compare_2Json(str1,str2) will return a boolean value.
Please make sure that the two string parameters are json.
Welcome to use and test.
def compare_2Json(js1:String,js2:String): Boolean = {
var js_str1 = js1
var js_str2 = js2
js_str1=js_str1.replaceAll(" ","")
js_str2=js_str2.replaceAll(" ","")
var issame = false
val arrbuff1 = ArrayBuffer[String]()
val arrbuff2 = ArrayBuffer[String]()
if(js_str1.substring(0,1)=="{" && js_str2.substring(0,1)=="{" || js_str1.substring(0,1)=="["&&js_str2.substring(0,1)=="["){
for(small_js1 <- split_JsonintoSmall(js_str1);small_js2 <- split_JsonintoSmall((js_str2))) {
issame = compare_2Json(small_js1,small_js2)
if(issame == true){
js_str1 = js_str1.substring(0,js_str1.indexOf(small_js1))+js_str1.substring(js_str1.indexOf(small_js1)+small_js1.length)
js_str2 = js_str2.substring(0,js_str2.indexOf(small_js2))+js_str2.substring(js_str2.indexOf(small_js2)+small_js2.length)
}
}
js_str1 = js_str1.substring(1,js_str1.length-1)
js_str2 = js_str2.substring(1,js_str2.length-1)
for(str_js1 <- js_str1.split(","); str_js2 <- js_str2.split(",")){
if(str_js1!="" && str_js2!="")
if(str_js1 == str_js2){
js_str1 = js_str1.substring(0,js_str1.indexOf(str_js1))+js_str1.substring(js_str1.indexOf(str_js1)+str_js1.length)
js_str2 = js_str2.substring(0,js_str2.indexOf(str_js2))+js_str2.substring(js_str2.indexOf(str_js2)+str_js2.length)
}
}
js_str1=js_str1.replace(",","")
js_str2=js_str2.replace(",","")
if(js_str1==""&&js_str2=="")return true
else return false
}
else return false
}
def split_JsonintoSmall(js_str: String):ArrayBuffer[String]={
val arrbuff = ArrayBuffer[String]()
var json_str = js_str
while(json_str.indexOf("{",1)>0 || json_str.indexOf("[",1)>0){
if (json_str.indexOf("{", 1) < json_str.indexOf("[", 1) && json_str.indexOf("{",1)>0 || json_str.indexOf("{", 1) > json_str.indexOf("[", 1) && json_str.indexOf("[",1)<0 ) {
val right = findrealm(1, json_str, '{', '}')
arrbuff += json_str.substring(json_str.indexOf("{", 1), right + 1)
json_str = json_str.substring(0,json_str.indexOf("{",1))+json_str.substring(right+1)
}
else {
if(json_str.indexOf("[",1)>0) {
val right = findrealm(1, json_str, '[', ']')
arrbuff += json_str.substring(json_str.indexOf("[", 1), right + 1)
json_str = json_str.substring(0, json_str.indexOf("[", 1)) + json_str.substring(right + 1)
}
}
}
arrbuff
}
def findrealm(begin_loc: Int, str: String, leftch: Char, rightch: Char): Int = {
var left = str.indexOf(leftch, begin_loc)
var right = str.indexOf(rightch, left)
left = str.indexOf(leftch, left + 1)
while (left < right && left > 0) {
right = str.indexOf(rightch, right + 1)
left = str.indexOf(leftch, left + 1)
}
right
}

Customizing JSON serialization in Play

I'm using renderJSON(Object) to return some objects as JSON values, and it's working fine except for one field. Is there an easy way to add in that one field without having to manually create the whole json template?
Play uses GSON to build the JSON string. If your one field is a specific object type, then you can easily do this by providing a customised serialisation for that type. See the documentation here
http://sites.google.com/site/gson/gson-user-guide#TOC-Custom-Serialization-and-Deserializ
However, if it is an Integer class for example, that you want to work in one way for one, and another way for another, then you may have a little more difficulty.
Example
GsonBuilder gson = new GsonBuilder();
gson.registerTypeAdapter(SpecificClass.class, new MySerializer());
private class MySerializer implements JsonSerializer<DateTime> {
public JsonElement serialize(SpecificClass src, Type typeOfSrc, JsonSerializationContext context) {
String res = "special format of specificClass"
return new JsonPrimitive(res);
}
}
Simply do a
JsonElement elem = new Gson().toJsonTree(yourObject);
JsonObject obj = elem.getAsJsonObject();
obj.remove("xxx");
obj.addProperty("xxx", "what you want");
// other stuff ...
renderJSON(obj.toString());
etc.
After evaluating the play framework we hit a stumbling block and decision choice on serializing JSON for an external API. Allot of articles out there suggest using the Lift framework within play which just seem like extra overhead.After trying some of the frameworks / modules with in the play framework a college and myself decided to write a light weight code block that could cater for our needs.
case class User (
user_id: Int,
user_name: Option[String],
password: Option[String],
salt: Option[String]
) extends Serializable {
def toXml =
<user>
<user_id>{user_id}</user_id>
<user_name>{user_name.getOrElse("")}</user_name>
</user>
override def toJson =
"{" + JSON.key("user_id") + JSON.value(user_id) + ", " + JSON.key("user_name") + JSON.value(user_name) + "}"
}
class Serializable {
def toJson = ""
}
object JSON {
def key(x:String) = value(x) + ": "
def value(x:Any):String = {
x match {
case s:String => "\"" + s + "\""
case y:Some[String] => value(y.getOrElse(""))
case i:Int => value(i.toString)
case s:Serializable => s.toJson
case xs:List[Any] => "[" + xs.map(x => value(x)).reduceLeft(_ + ", " + _) + "]"
}
}
}
def searchUserByName(user_name: String) = {
(for (
u <- Users if u.user_name.like(("%"+user_name+"%").bind)
) yield u.*)
.list
.map(User.tupled(_))
}
def toXml(users:List[User]) = {
<users>
{ users.map(u => u.toXml) }
</users>
}
def toJson(users:List[User]) = {
"[" + users.map(u => u.toJson).reduceLeft(_ + ", " + _) + "]"
}
And from the controller.
// -- http://localhost:9000/api/users/getUser/xml
// -- http://localhost:9000/api/users/getUser/json
def getUser(requestType:String) = {
db withSession{
val user = Users.byUserName("King.Kong")
if(requestType == "xml") {
Xml(user.toXml)
} else {
user.toJson
}
}
}
//--- http://localhost:9000/api/users/searchuser/xml
//--- http://localhost:9000/api/users/searchuser/json
def searchUser(requestType:String) = {
db withSession{
val users = Users.searchUserByName("Doctor.Spoc")
if(requestType == "xml") {
Xml(Users.toXml(users))
} else {
val jsonList = Users.toJson(users)
Json(jsonList)
}
}