some times we have JSON looks like this:
{a:{}, b:{c:{}, d:123}}
you want to remove the empty structures and make it into {b:{d:123}}
here is how you can do it simply in Scala by using Jackson:
val json = """ {"a":{}, "b": {"c": {}, "d": 123}} """
val mapper = new ObjectMapper()
mapper
.setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
.setSerializationInclusion(JsonInclude.Include.NON_NULL)
.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
val node = mapper.readTree(json)
removeEmptyFields(node)
val cleanJson = mapper.writeValueAsString(node) // {"b": {"d": 123}}
private def removeEmptyFields(obj: Object): Boolean = {
if (obj.isInstanceOf[ArrayNode]) {
val array = obj.asInstanceOf[ArrayNode]
val iter = array.elements()
var i = 0
while (iter.hasNext) {
if(!removeEmptyFields(iter.next())) array.remove(i)
i += 1
}
true
} else if (obj.isInstanceOf[ObjectNode]) {
val json = obj.asInstanceOf[ObjectNode]
val names = json.fieldNames().asScala.toList
if (names == null || names.isEmpty) return false
var removeRoot = true
names.foreach (
name => {
if (!removeEmptyFields(json.get(name))) {
json.remove(name)
} else removeRoot = false
}
)
!removeRoot
} else true
}
Pure, stack-safe implementation with circe:
import cats.Eval
import cats.implicits._
import io.circe.JsonObject
import io.circe.literal._
import io.circe.syntax._
object Main extends App {
def removeEmpty(jo: JsonObject): JsonObject = {
//`Eval` is trampolined so this is stack-safe
def loop(o: JsonObject): Eval[JsonObject] =
o.toList.foldLeftM(o) { case (acc, (k, v)) =>
v.asObject match {
case Some(oo) if oo.isEmpty => acc.remove(k).pure[Eval]
case Some(oo) => Eval.defer(loop(oo)).map(_o => acc.add(k, _o.asJson))
case _ => acc.pure[Eval]
}
}
loop(jo).value
}
//this is a json literal
// if it's from a dynamic string please parse it with `io.circe.parser.parse` first
val json = json"""{"a":{}, "b": {"c": {}, "d": 123}}"""
val res = json.asObject.map(removeEmpty(_).asJson)
println(res)
}
I try to save a DataSet into an index ElasticSearch everyday (schedule with Oozie) but I have sometimes this error java.lang.NoClassDefFoundError: Could not initialize class org.apache.spark.util.JsonProtocol so the job failed immediately. I don't know why this error appears.
Code :
private def readSource1()(implicit spark: SparkSession): DataFrame = {
import spark.implicits._
val sourceName = "dictionary.source1"
val plantsPath: String = config.getString("sources." + sourceName + ".path")
spark.read
.option("delimiter", ";")
.option("header", "true")
.csv(plantsPath)
.select('id as "sourceId", 'assembly_site_id)
}
private def readSource2()(implicit spark: SparkSession): DataFrame = {
import spark.implicits._
val source2: SourceIO = SourceManager(config)("source2")
(startDate, endDate) match {
case (Some(sd), Some(ed)) ⇒ source2.loadDf()
.where('assemblyEndDate.between(Date.valueOf(sd), Date.valueOf(ed)) ||
'tctDate.between(Date.valueOf(sd), Date.valueOf(ed)))
case _ ⇒ source2.loadDf()
}
}
def saveSourceToEs(implicit sparkSession: SparkSession): Unit = {
val source1: DataFrame = readSource1()
val source2: DataFrame = readSource2()
val source: Dataset[Source] = buildSource(this.getSource(source1, source2))
source.saveToEs(s"source_${createDateString()}/_doc")
}
object SourceIndexer extends SparkApp with Configurable with Logging {
val config: Config = ConfigFactory.load()
def apply(
sourceID: Option[String] = None,
startDate: Option[LocalDate] = None,
endDate: Option[LocalDate] = None
): SourceIndexer = {
new SourceIndexer(config, sourceID, startDate, endDate)
}
def main(args: Array[String]): Unit = {
try {
val bootConfig = BootConfig.parseSourceIndexer(args)
this.apply(bootConfig.sourceID, bootConfig.startDate, bootConfig.endDate)
.saveSourceToEs(spark)
} finally {
spark.sparkContext.stop()
}
}
}
Thanks for your help.
I have a scala application and have a case class like -
case class SR(
systemId: Option[String] = None,
x: Map[Timestamp, CaseClass1] = Map.empty,
y: Map[Timestamp, CaseClass2] = Map.empty,
y: Map[Timestamp, CaseClass3] = Map.empty
)
Now I have to provide an implicit read and write JSON format for properties x,y,z for SR case class like -
implicit val mapCMPFormat = new Format[Map[Timestamp, CaseClass1]] {
def writes(obj: Map[Timestamp, CaseClass1]): JsValue =
JsArray(obj.values.toSeq.map(Json.toJson(_)))
def reads(jv: JsValue): JsResult[Map[Timestamp, CaseClass1]] = jv.validate[scala.collection.Seq[CaseClass1]] match {
case JsSuccess(objs, path) => JsSuccess(objs.map(obj => obj.dataDate.get -> obj).toMap, path)
case err: JsError => err
}
}
And so on similarly for Y and Z, and in future, I will be adding many more properties like x,y,z in SR case class and then need to provide the formators.
So Can I get some Generic Formater that will take care for all types?
To my knowledge, a simple way to achieve this does not exists, however, to create a "default" reader for each object should not be hard to do, something like:
case class VehicleColorForAdd(
name: String,
rgb: String
)
object VehicleColorForAdd {
implicit val jsonFormat: Format[VehicleColorForAdd] = Json.formats[VehicleColorForAdd]
}
This way you have access to the implicit by simply using the object, so you could have other objects that contains this object with no problem:
case class BiggerModel(
vehicleColorForAdd: VehicleColorForAdd
)
object BiggerModel{
implicit val jsonFormat: Format[BiggerModel] = Json.format[BiggerModel]
}
Sadly, you need to do this for each class type, but you can "extend" play converters with your own, for example, this are some of my default readers:
package common.json
import core.order.Order
import org.joda.time.{ DateTime, LocalDateTime }
import org.joda.time.format.DateTimeFormat
import core.promotion.{ DailySchedule, Period }
import play.api.libs.functional.syntax._
import play.api.libs.json.Reads._
import play.api.libs.json._
import play.api.libs.json.{ JsError, JsPath, JsSuccess, Reads }
import scala.language.implicitConversions
/**
* General JSon readers and transformations.
*/
object JsonReaders {
val dateTimeFormat = "yyyy-MM-dd HH:mm:ss"
class JsPathHelper(val path: JsPath) {
def readTrimmedString(implicit r: Reads[String]): Reads[String] = Reads.at[String](path)(r).map(_.trim)
def readUpperString(implicit r: Reads[String]): Reads[String] = Reads.at[String](path)(r).map(_.toUpperCase)
def readNullableTrimmedString(implicit r: Reads[String]): Reads[Option[String]] = Reads.nullable[String](path)(r).map(_.map(_.trim))
}
implicit val localDateTimeReader: Reads[LocalDateTime] = Reads[LocalDateTime]((js: JsValue) =>
js.validate[String].map[LocalDateTime](dtString =>
LocalDateTime.parse(dtString, DateTimeFormat.forPattern(dateTimeFormat))))
val localDateTimeWriter: Writes[LocalDateTime] = new Writes[LocalDateTime] {
def writes(d: LocalDateTime): JsValue = JsString(d.toString(dateTimeFormat))
}
implicit val localDateTimeFormat: Format[LocalDateTime] = Format(localDateTimeReader, localDateTimeWriter)
implicit val dateTimeReader: Reads[DateTime] = Reads[DateTime]((js: JsValue) =>
js.validate[String].map[DateTime](dtString =>
DateTime.parse(dtString, DateTimeFormat.forPattern(dateTimeFormat))))
implicit def toJsPathHelper(path: JsPath): JsPathHelper = new JsPathHelper(path)
val defaultStringMax: Reads[String] = maxLength[String](255)
val defaultStringMinMax: Reads[String] = minLength[String](1) andKeep defaultStringMax
val rgbRegex: Reads[String] = pattern("""^#([\da-fA-F]{2})([\da-fA-F]{2})([\da-fA-F]{2})$""".r, "error.invalidRGBPattern")
val plateRegex: Reads[String] = pattern("""^[\d\a-zA-Z]*$""".r, "error.invalidPlatePattern")
val minOnlyWordsRegex: Reads[String] = minLength[String](2) keepAnd onlyWordsRegex
val positiveInt: Reads[Int] = min[Int](1)
val zeroPositiveInt: Reads[Int] = min[Int](0)
val zeroPositiveBigDecimal: Reads[BigDecimal] = min[BigDecimal](0)
val positiveBigDecimal: Reads[BigDecimal] = min[BigDecimal](1)
def validLocalDatePeriod()(implicit reads: Reads[Period]) =
Reads[Period](js => reads.reads(js).flatMap { o =>
if (o.startPeriod isAfter o.endPeriod)
JsError("error.startPeriodAfterEndPeriod")
else
JsSuccess(o)
})
def validLocalTimePeriod()(implicit reads: Reads[DailySchedule]) =
Reads[DailySchedule](js => reads.reads(js).flatMap { o =>
if (o.dailyStart isAfter o.dailyEnd)
JsError("error.dailyStartAfterDailyEnd")
else
JsSuccess(o)
})
}
Then, you only need to import this object to have access to all this implicit converters:
package common.forms
import common.json.JsonReaders._
import play.api.libs.json._
/**
* Form to add a model with only one string field.
*/
object SimpleCatalogAdd {
case class Data(
name: String
)
implicit val dataReads: Reads[Data] = (__ \ "name").readTrimmedString(defaultStringMinMax).map(Data.apply)
}
I am trying to write a json reads combinator for type Map[Int, Long]
I have this so far:
implicit val mapWrites = Json.writes[Map[Int, Long]]
implicit val mapReads: Reads[Map[Int, Long]] = (
// ???
) // ?
I'm not sure how this will work, I tried doing (Map[Int, Long].apply but that apply method was not available.
Looking for some help to write this Reads combinator.
This should do it, with the caveat that it doesn't handle NumberFormatException in the reads case:
//
// scala> Json.toJson(Map(1 -> 2L, 2 -> 3L))
// res0: play.api.libs.json.JsValue = {"1":2,"2":3}
//
implicit val formatter: Format[Map[Int, Long]] = {
new Format[Map[Int, Long]] {
def writes(m: Map[Int, Long]) = {
Json.toJson(m.map {
case (key, value) => key.toString -> value
})
}
def reads(json: JsValue) = {
json.validate[Map[String, Long]].map(_.map {
case (key, value) => key.toInt -> value
})
}
}
}
As separate Reads and Writes instances:
implicit val readsInstance: Reads[Map[Int, Long]] = {
new Reads[Map[Int, Long]] {
def reads(json: JsValue) = {
json.validate[Map[String, Long]].map(_.map {
case (key, value) => key.toInt -> value
})
}
}
}
implicit val writesInstance: Writes[Map[Int, Long]] = {
def writes(m: Map[Int, Long]) = {
Json.toJson(m.map {
case (key, value) => key.toString -> value
})
}
}
}
I am attempting to migrate a Rails/Mongodb application to Play 2.3 using play-reactivemongo and reactivemongo-extensions. In modeling my data I am running across a problem serializing and deserializing a Map[Int,Boolean].
When I try to define my formats via macro like so
implicit val myCaseClass = Json.format[MyCaseClass]
where MyCaseClass has a few string fields, a BSONObjectID field, and a Map[Int,Boolean] field the compiler complains with:
No Json serializer found for type Map[Int,Boolean]. Try to implement an implicit Writes or Format for this type.
No Json deserializer found for type Map[Int,Boolean]. Try to implement an implicit Reads or Format for this type.
Looking at the source code for Play in Reads.scala I see a Reads defined for Map[String,_] but none for Map[Int,_].
Is there a reason why Play has default Read/Writes for string maps but not for other simple types?
I don't fully understand the Map[String,_] defined by play because I am fairly new to scala. How would I go about translating that into a Map[Int,_]? If that is not possible for some technical reason how would I define a Reads/Writes for Map[Int,Boolean]?
you can write your own reads and writes in play.
in your case, this would look like this:
implicit val mapReads: Reads[Map[Int, Boolean]] = new Reads[Map[Int, Boolean]] {
def reads(jv: JsValue): JsResult[Map[Int, Boolean]] =
JsSuccess(jv.as[Map[String, Boolean]].map{case (k, v) =>
Integer.parseInt(k) -> v .asInstanceOf[Boolean]
})
}
implicit val mapWrites: Writes[Map[Int, Boolean]] = new Writes[Map[Int, Boolean]] {
def writes(map: Map[Int, Boolean]): JsValue =
Json.obj(map.map{case (s, o) =>
val ret: (String, JsValueWrapper) = s.toString -> JsBoolean(o)
ret
}.toSeq:_*)
}
implicit val mapFormat: Format[Map[Int, Boolean]] = Format(mapReads, mapWrites)
I have tested it with play 2.3. I'm not sure if it's the best approach to have a Map[Int, Boolean] on server side and a json object with string -> boolean mapping on the client side, though.
JSON only allows string keys (a limitation it inherits from JavaScript).
Play Json provides built-in mapReads and mapWrites for reading and writing Maps.
mapReads takes a (String => JsResult[K]) to let you convert the key to your custom type.
mapWrites returns a Writes[Map[String, Boolean]], and you can use contramap to modify that writer into one that works with a Map[Int, Boolean]
import play.api.libs.json.{JsResult, Reads, Writes}
import scala.util.Try
import play.api.libs.json.Reads.mapReads
import play.api.libs.json.MapWrites.mapWrites
object MapExample {
implicit val reads: Reads[Map[Int, Boolean]] =
mapReads[Int, Boolean](s => JsResult.fromTry(Try(s.toInt)))
implicit val writes: Writes[Map[Int, Boolean]] =
mapWrites[Boolean].contramap(_.map { case (k, v) => k.toString -> v})
}
Thanks to Seth Tisue. This is my "generics" (half) way.
"half" because it does not handle a generic key. one can copy paste and replace the "Long" with "Int"
"Summary" is a type I've wanted to serialize (and it needed its own
serializer)
/** this is how to create reader and writer or format for Maps*/
// implicit val mapReads: Reads[Map[Long, Summary]] = new MapLongReads[Summary]
// implicit val mapWrites: Writes[Map[Long, Summary]] = new MapLongWrites[Summary]
implicit val mapLongSummaryFormat: Format[Map[Long, Summary]] = new MapLongFormats[Summary]
This is the required implementation:
class MapLongReads[T]()(implicit reads: Reads[T]) extends Reads[Map[Long, T]] {
def reads(jv: JsValue): JsResult[Map[Long, T]] =
JsSuccess(jv.as[Map[String, T]].map{case (k, v) =>
k.toString.toLong -> v .asInstanceOf[T]
})
}
class MapLongWrites[T]()(implicit writes: Writes[T]) extends Writes[Map[Long, T]] {
def writes(map: Map[Long, T]): JsValue =
Json.obj(map.map{case (s, o) =>
val ret: (String, JsValueWrapper) = s.toString -> Json.toJson(o)
ret
}.toSeq:_*)
}
class MapLongFormats[T]()(implicit format: Format[T]) extends Format[Map[Long, T]]{
override def reads(json: JsValue): JsResult[Map[Long, T]] = new MapLongReads[T].reads(json)
override def writes(o: Map[Long, T]): JsValue = new MapLongWrites[T].writes(o)
}
We can generalize the solution of 3x14159265 and Seth Tisue thanks to 2 small type classes:
import play.api.libs.json.Json.JsValueWrapper
import play.api.libs.json._
import simulacrum._
object MapFormat {
#typeclass trait ToString[A] {
def toStringValue(v: A): String
}
#typeclass trait FromString[A] {
def fromString(v: String): A
}
implicit final def mapReads[K: FromString, V: Reads]: Reads[Map[K, V]] =
new Reads[Map[K, V]] {
def reads(js: JsValue): JsResult[Map[K, V]] =
JsSuccess(js.as[Map[String, V]].map { case (k, v) => FromString[K].fromString(k) -> v })
}
implicit final def mapWrites[K: ToString, V: Writes]: Writes[Map[K, V]] =
new Writes[Map[K, V]] {
def writes(map: Map[K, V]): JsValue =
Json.obj(map.map {
case (s, o) =>
val ret: (String, JsValueWrapper) = ToString[K].toStringValue(s) -> o
ret
}.toSeq: _*)
}
implicit final def mapFormat[K: ToString: FromString, V: Format]: Format[Map[K, V]] = Format(mapReads, mapWrites)
}
Note that I use Simulacrum (https://github.com/mpilquist/simulacrum) to define my type classes.
Here is an example of how to use it:
final case class UserId(value: String) extends AnyVal
object UserId {
import MapFormat._
implicit final val userToString: ToString[UserId] =
new ToString[UserId] {
def toStringValue(v: UserId): String = v.value
}
implicit final val userFromString: FromString[UserId] =
new FromString[UserId] {
def fromString(v: String): UserId = UserId(v)
}
}
object MyApp extends App {
import MapFormat._
val myMap: Map[UserId, Something] = Map(...)
Json.toJson(myMap)
}
if IntelliJ says that your import MapFormat._ is never used, you can and this: implicitly[Format[Map[UserId, Something]]] just below the import. It'll fix the pb. ;)
A specific KeyWrites and KeyReads is available in play-json 2.9.x
private implicit val longKeyWrites = KeyWrites[Int](_.toString)
private implicit val longKeyReads =
KeyReads[Int](str => Try(str.toInt).fold(e => JsError(e.getMessage), JsSuccess(_)))
Json.obj("1" -> "test").validate[Map[Int,String]] // JsSuccess(Map(1 -> test))
Like the accepted answer - a bit shorter:
implicit val mapReads: Reads[Map[Int, Boolean]] = (jv: JsValue) =>
JsSuccess(jv.as[Map[String, Boolean]].map { case (k, v) =>
k.toInt -> v
})
implicit val mapWrites: Writes[Map[Int, Boolean]] = (map: Map[Int, Boolean]) =>
Json.toJson(map.map { case (s, o) =>
s.toString -> o
})
implicit val jsonMapFormat: Format[Map[Int, Boolean]] = Format(mapReads, mapWrites)
Here a little test:
val json = Json.toJson(Map(1 -> true, 2 -> false))
println(json) // {"1":true,"2":false}
println(json.validate[Map[Int, Boolean]]) // JsSuccess(Map(1 -> true, 2 -> false),)
https://gist.github.com/fancellu/0bea53f1a1dda712e179892785572ce3
Here is a way to persist a Map[NotString,...]