kotlin: Compile time check - json

I'm trying to serialize a Map<String,*> type.
Currently I'm decalring a SerializersModule, with any type that I'm storing in the map added here.
val module = SerializersModule {
polymorphic(Any::class) {
subclass(Int::class, PolymorphicPrimitiveSerializer(Int.serializer()))
subclass(String::class, PolymorphicPrimitiveSerializer(String.serializer()))
subclass(ComplexType::class)
}
}
I would like to enforce the compiler that only types that were defined in SerializersModule can be added to the map?
I'm open to hear other strategies to serialize star projected types
Full code:
import kotlinx.serialization.*
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.json.Json
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.descriptors.buildClassSerialDescriptor
import kotlinx.serialization.encoding.decodeStructure
import kotlinx.serialization.encoding.encodeStructure
import kotlinx.serialization.modules.*
import java.io.Serializable
#OptIn( ExperimentalSerializationApi::class )
class PolymorphicPrimitiveSerializer<T> (val typeSerializer: KSerializer<T>) : KSerializer<T>
{
override val descriptor: SerialDescriptor = buildClassSerialDescriptor( typeSerializer.descriptor.serialName )
{
element( "value", typeSerializer.descriptor )
}
override fun deserialize( decoder: Decoder ): T =
decoder.decodeStructure( descriptor )
{
decodeElementIndex( descriptor )
//TODO: what is this?
decodeSerializableElement( descriptor, 0,typeSerializer)
}
override fun serialize(encoder: Encoder, value: T) {
encoder.encodeStructure( descriptor )
{
encodeSerializableElement( descriptor, 0, typeSerializer, value )
}
}
}
data class MetaKey<T>(val name: String, val typ: Class<T>) : java.io.Serializable {
companion object {
inline fun <reified T: Serializable> named(name: String) =
MetaKey(name, T::class.javaObjectType)
}
}
fun emptyMetaMap() = mapOf<MetaKey<*>, Any>()
#kotlinx.serialization.Serializable
data class ComplexType(val i: Int)
fun main() {
val typesDictionary = mapOf(
"kotlin.Int" to Int::class.javaObjectType,
"ComplexType" to ComplexType::class.javaObjectType,
"kotlin.String" to String::class.javaObjectType )
val module = SerializersModule {
polymorphic(Any::class) {
subclass(Int::class, PolymorphicPrimitiveSerializer(Int.serializer()))
subclass(String::class, PolymorphicPrimitiveSerializer(String.serializer()))
subclass(ComplexType::class)
}
}
val format = Json { serializersModule = module }
val mm = emptyMetaMap()
.plus(MetaKey("complex type", ComplexType::class.javaObjectType) to ComplexType(1) )
.plus(MetaKey("Int one", Int::class.javaObjectType) to 1)
.plus(MetaKey("string value", String::class.javaObjectType) to "2B||!2B")
val jsoned = mm.keys.map {
listOf(
format.encodeToString(it.name),
format.encodeToString(PolymorphicSerializer(Any::class),mm[it] as Any),
)
}
var mmDecoded = emptyMetaMap()
jsoned.forEach {
val strKey = format.decodeFromString<String>(it[0])
val decodedVal = format.decodeFromString(PolymorphicSerializer(Any::class), it[1])
val strType = it[1].substringAfter("\"type\":\"").substringBefore("\"")
val metaKeyDeserialized = MetaKey(strKey, typesDictionary[strType] ?: throw IllegalArgumentException())
mmDecoded = mmDecoded.plus(metaKeyDeserialized to decodedVal)
}
assert(mmDecoded == mm)
}

Related

Kotlinx Deserialization Object or List of Object to List of Object

I am currently implementing an API client with Ktor. The API I am requesting does not return a consistent JSON format.
for Example:
sometimes the JSON looks like this:
{
"description": {
"lang": "en",
"value": "an English description..."
},
...
}
and sometimes like this:
{
"description": [
{
"lang": "en",
"value": "an English description..."
},
{
"lang": "fr",
"value": "a French description..."
}
],
...
}
Now my Question:
How can I implement a Custom Kotlinx Deserializer to Decode an Object of T or a List<T> to a List<T>
My classes look like this:
#Serializable
class ResourceResponse(
#SerialName("description")
val descriptions: List<Description>
) {
#Serializable
data class Description(
#SerialName("value")
val value: String,
#SerialName("lang")
val language: String,
)
}
I want that a Json with only one Description-Object will be deserialized to a List with one Object and not specifically for the description, but in general for classes.
I've found nothing really helpful in the Web.
One solution is to first deserialize it to JsonElement, introspect and then decide how to deserialize it further into ResourceResponse:
fun decode(s: String): ResourceResponse {
val json = Json.parseToJsonElement(s).jsonObject
return when (val desc = json["description"]) {
is JsonArray -> Json.decodeFromJsonElement(json)
is JsonObject -> {
val json2 = json.toMutableMap()
json2["description"] = JsonArray(listOf(desc))
Json.decodeFromJsonElement(JsonObject(json2))
}
else -> throw IllegalArgumentException("Invalid value for \"description\": $desc")
}
}
This solution is definitely not ideal. It may be potentially less performant as we need to deserialize the whole tree into the tree of JsonElement objects only to transform it to the final types (although, maybe the library does this internally anyway). It works only for json and it is tricky to use this solution if ResourceResponse is somewhere deep into the data structure.
You can use a JsonContentPolymorphicSerializer to choose a deserializer based on the form of the JSON.
This one should work:
#Suppress("UNCHECKED_CAST")
class DescriptionsSerializer : JsonContentPolymorphicSerializer<List<ResourceResponse.Description>>(
List::class as KClass<List<ResourceResponse.Description>>
) {
// Here we check the form of the JSON we are decoding, and choose
// the serializer accordingly
override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out List<ResourceResponse.Description>> {
return if (element is JsonArray)
ListSerializer(ResourceResponse.Description.serializer())
else
SingleDescriptionAsList()
}
class SingleDescriptionAsList : KSerializer<List<ResourceResponse.Description>> {
override val descriptor: SerialDescriptor
get() = ResourceResponse.Description.serializer().descriptor
override fun deserialize(decoder: Decoder): List<ResourceResponse.Description> {
return listOf(ResourceResponse.Description.serializer().deserialize(decoder))
}
override fun serialize(encoder: Encoder, value: List<ResourceResponse.Description>) {
throw Exception("Not in use")
}
}
}
You must also amend your original class to tell it to use this serializer:
#Serializable
class ResourceResponse(
#SerialName("description")
#Serializable(with = DescriptionsSerializer::class) val descriptions: List<Description>
) {
#Serializable
data class Description(
#SerialName("value")
val value: String,
#SerialName("lang")
val language: String,
)
}
Then you will be able to decode JSON objects with the single key "descriptions" using the ResourceResponse serializer.
For avoidance of doubt, if there are other keys in the JSON (it's not entirely clear from the question) then those should also be written into ResourceResponse definition.
After my research, I have now come up with a solution. For this you need a wrapper class. (here GenericResponse). I hope I can help others who have the same problem.
This is the Wrapper-Class
#Serializable(with = ListOrObjectSerializer::class)
class GenericResponse<T>(
val data: List<T> = emptyList()
) {
private var _isNothing : Boolean = false
val isNothing: Boolean
get() {
return this._isNothing
}
companion object {
fun <T> nothing(): GenericResponse<T> {
val o = GenericResponse(emptyList<T>())
o._isNothing = true
return o
}
}
}
And the Serializer looks like:
import kotlinx.serialization.KSerializer
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.json.*
class ListOrObjectSerializer<T : Any>(private val tSerializer: KSerializer<T>): KSerializer<GenericResponse<T>> {
override val descriptor: SerialDescriptor
get() = tSerializer.descriptor
override fun deserialize(decoder: Decoder): GenericResponse<T> {
val input = decoder as JsonDecoder
val jsonObj = input.decodeJsonElement()
return when(jsonObj) {
is JsonObject -> GenericResponse(listOf(Json.decodeFromJsonElement(tSerializer, jsonObj)))
is JsonArray -> GenericResponse(Json.decodeFromJsonElement(ListSerializer(tSerializer), jsonObj))
else -> return GenericResponse.nothing()
}
}
override fun serialize(encoder: Encoder, value: GenericResponse<T>) {
throw IllegalAccessError("serialize not supported")
}
}
My Data-Class look now like:
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
#Serializable
class ResourceResponse(
#SerialName("description")
val descriptions: GenericResponse<Description>? = null,
) {
#Serializable
data class Description(
#SerialName("value")
val value: String? = null,
#SerialName("lang")
val language: String? = null,
)
}
data class ResourceResponse(
#SerializedName("description") val descriptions: List<Description>,
)
data class Description(
#SerializedName("value") val value: String,
#SerializedName("lang") val language: String,
)
it should be like that

JsonProtocol NoClassDefFoundError

I try to save a DataSet into an index ElasticSearch everyday (schedule with Oozie) but I have sometimes this error java.lang.NoClassDefFoundError: Could not initialize class org.apache.spark.util.JsonProtocol so the job failed immediately. I don't know why this error appears.
Code :
private def readSource1()(implicit spark: SparkSession): DataFrame = {
import spark.implicits._
val sourceName = "dictionary.source1"
val plantsPath: String = config.getString("sources." + sourceName + ".path")
spark.read
.option("delimiter", ";")
.option("header", "true")
.csv(plantsPath)
.select('id as "sourceId", 'assembly_site_id)
}
private def readSource2()(implicit spark: SparkSession): DataFrame = {
import spark.implicits._
val source2: SourceIO = SourceManager(config)("source2")
(startDate, endDate) match {
case (Some(sd), Some(ed)) ⇒ source2.loadDf()
.where('assemblyEndDate.between(Date.valueOf(sd), Date.valueOf(ed)) ||
'tctDate.between(Date.valueOf(sd), Date.valueOf(ed)))
case _ ⇒ source2.loadDf()
}
}
def saveSourceToEs(implicit sparkSession: SparkSession): Unit = {
val source1: DataFrame = readSource1()
val source2: DataFrame = readSource2()
val source: Dataset[Source] = buildSource(this.getSource(source1, source2))
source.saveToEs(s"source_${createDateString()}/_doc")
}
object SourceIndexer extends SparkApp with Configurable with Logging {
val config: Config = ConfigFactory.load()
def apply(
sourceID: Option[String] = None,
startDate: Option[LocalDate] = None,
endDate: Option[LocalDate] = None
): SourceIndexer = {
new SourceIndexer(config, sourceID, startDate, endDate)
}
def main(args: Array[String]): Unit = {
try {
val bootConfig = BootConfig.parseSourceIndexer(args)
this.apply(bootConfig.sourceID, bootConfig.startDate, bootConfig.endDate)
.saveSourceToEs(spark)
} finally {
spark.sparkContext.stop()
}
}
}
Thanks for your help.

How to deserialize complex JSON with Scala SprayJSON?

I have the following JSON:
{
"key1":[
{
"key2":{
"key3":"value3",
"key4":"value4"
},
"key5":{
"key6":"value6"
},
"key7":[
{
"key8":"value8",
"key9":"value9"
}
],
"key10":"value10",
"key11":"value11"
}
],
"key12":"value12"
}
How can I retrieve nested elements (e.g. value6) using SprayJson.
I managed to only retrieve the top level key "key1".
case class Key1(key1: JsArray)
object Key1Protocol extends DefaultJsonProtocol {
implicit val key1: RootJsonFormat[Key1] = jsonFormat1(Key1)
}
<jsonString>.parseJson.convertTo[Key1]
This can help:
case class Key1(key1: JsArray)
object Key1Protocol extends DefaultJsonProtocol {
implicit val key1: RootJsonFormat[Key1] = jsonFormat1(Key1)
}
import spray.json._
import DefaultJsonProtocol._
object MainJson {
import Key1Protocol._
def main(args: Array[String]): Unit = {
val jsonAst = TestComplexJson.str.parseJson.convertTo[Key1]
val result = jsonAst.key1
.elements(0)
.asJsObject
.getFields("key5")(0)
.asJsObject()
.getFields("key6")(0)
println(result)
}
}
gives "value6"

How to edit existing JSON object with sprayJSON

I am using akka with spray json support for which I need to edit value in the recieved json.
import akka.http.scaladsl.server.Directives
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport
import spray.json._
final case class Item(name: String, id: Long)
final case class Order(items: List[Item],orderTag:String)
trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol {
implicit val itemFormat = jsonFormat2(Item)
implicit val orderFormat = jsonFormat2(Order)
}
In my use case I recieve the json with orderTag value as null, all I need to do is edit the orderTag value with and then use it as entity value.Is it possible to write/edit jsonObject and How to do that ?
class MyJsonService extends Directives with JsonSupport {
// format: OFF
val route =
get {
pathSingleSlash {
complete(Item("thing", 42)) // will render as JSON
}
} ~
post {
entity(as[Order]) { order => // will unmarshal JSON to Order
val itemsCount = order.items.size
val itemNames = order.items.map(_.name).mkString(", ")
complete(s"Ordered $itemsCount items: $itemNames")
}
}
}
You can just edit the json AST like ..
val json = """{"orderTag":null}"""
val jsVal = json.parseJson
val updatedJs = if (jsObj.fields.get("orderTag") == Some(JsNull)) {
JsObject(jsObj.fields + ("orderTag" -> JsString("new tag")))
} else {
jsObj
}
updatedJs.compactPrint
res26: String = """
{"orderTag":"new tag"}
"""

Implicit parameters not found

I'm having some trouble figuring out why compiler complains about not finding an implicit parameter for reads because I'm almost sure that it is in the scope. The error is the following:
Error:(13, 18) No Json deserializer found for type Config. Try to implement an implicit Reads or Format for this type.
test.validate[Config].map {
^
Error:(13, 18) not enough arguments for method validate: (implicit rds: play.api.libs.json.Reads[Config])play.api.libs.json.JsResult[wings.m2m.conf.model.Config].
Unspecified value parameter rds.
test.validate[Config].map {
^
and it happens in the following code:
import play.api.libs.json._
import play.api.libs.json.Reads._
import Config.JsonImplicits._
import scala.util.Try
object Test {
def main(args: Array[String]) {
val test = Json.obj("action" -> Config.Action.nameAcquisitionRequest.toString, "value" -> "hola")
test.validate[Config].map {
t => println(t)
t
}
}
}
/**
* Config companion object
*/
object Config {
type ValueType = String
val ActionKey = "action"
val ValueKey = "value"
object Action extends Enumeration {
type Action = Value
val nameAcquisitionRequest = Value("nameAcquisitionRequest")
val nameAcquisitionReject = Value("nameAcquisitionReject")
val nameAcquisitionAck = Value("nameAcquisitionAck")
val broadcast = Value("broadcast")
}
/**
* Json implicit conversions
*/
object JsonImplicits {
implicit object ConfigReads extends Reads[Config] {
def hTypeCast(action: Config.Action.Value, value: Config.ValueType): Config = {
action match {
case Config.Action.nameAcquisitionRequest => NameAcquisitionRequest(value)
case Config.Action.nameAcquisitionReject => NameAcquisitionReject(value)
case Config.Action.nameAcquisitionAck => NameAcquisitionAck(value)
}
}
override def reads(json: JsValue): JsResult[Config] = json match {
case json: JsObject =>
val action = (json \ ActionKey).as[String]
Try(Config.Action.withName(action)) map {
a =>
val value = (json \ ValueKey).as[String]
JsSuccess(hTypeCast(a, value))
} getOrElse (JsError("Can't convert to Config"))
case _ => JsError("Can't convert to Config")
}
}
implicit object ConfigWrites extends OWrites[Config] {
def jsObjectCreator(action: Config.Action.Value, value: Config.ValueType): JsObject = {
Json.obj(ActionKey -> action.toString, ValueKey -> Json.toJson(value))
}
override def writes(o: Config): JsObject = o match {
case c: NameAcquisitionRequest => jsObjectCreator(Config.Action.nameAcquisitionRequest, c.value)
case c: NameAcquisitionReject => jsObjectCreator(Config.Action.nameAcquisitionReject, c.value)
case c: NameAcquisitionAck => jsObjectCreator(Config.Action.nameAcquisitionAck, c.value)
}
}
}
}
sealed trait Config {
val value: Config.ValueType
}
/**
* Intermediate config message
* #param value
*/
case class NameAcquisitionRequest(override val value: String)
extends Config
case class NameAcquisitionReject(override val value: String)
extends Config
case class NameAcquisitionAck(override val value: String)
extends Config
case class Broadcast(override val value: String)
extends Config
the error occurs when executing the main method on the Test object. To make this example work, make sure to add the following dependency in the SBT: "com.typesafe.play" %% "play-json" % "2.4.1" . And I'm not sure, but maybe this resolver is needed: resolvers += "Typesafe Repo" at "http://repo.typesafe.com/typesafe/releases/"
I am not sure what you are trying to achieve and whether this solves your problem but here you go:
test.validate[Config](Config.JsonImplicits.ConfigReads).map {
t => println(t)
t
}