I am facing this error Caused: java.io.NotSerializableException: groovy.json.JsonBuilder while building a jenkins pipeline. However the same code is working in jenkins scriptler. Should I do any changes on this code to write a Json file through jenkins pipeline
def vars = ["a": 1,"b": 2,"c": 3,"d": 4,]
def json = new groovy.json.JsonBuilder()
def final_out = json "sample": vars
json "file": final_out
def file = new File("/tmp/sample.json")
file.write(groovy.json.JsonOutput.prettyPrint(json.toString()))
Sample output looks like:
{
"file": {
"sample": {
"a": 1,
"b": 2,
"c": 3,
"d": 4
}
}
}
Related
How can I save result of groovy script to a new file? C:/temp/all1.csv. I want to parse json file to csv, script is working fine but I don't know how can I save result in a new file. Please help.
import groovy.json.*
import java.io.File
def json ='''
{
"expand": "schema,names",
"startAt": 0,
"maxResults": 50,
"total": 21,
"issues": [
{
"expand": "operations,versionedRepresentations",
"id": "217580",
"self": "issue/217580",
"key": "ART-4070",
"fields": {"summary": "#[ART] Pre.3 Verification \\"S\\""}
},
{
"expand": "operations,versionedRepresentations",
"id": "217579",
"self": "issue/217579",
"key": "ART-4069",
"fields": {"summary": "Verification \\"C\\""}
},
{
"expand": "operations,versionedRepresentations",
"id": "217577",
"self": "issue/217577",
"key": "ART-4068",
"fields": {"summary": "#[ART] Enum type"}
}
]
}
'''
File csvFile = new File( 'C:/temp/all1.csv')
def jsonSlurper = new JsonSlurper()
def config = [ // header -> extractor
"key": { it.key },
"summary": { it.fields.summary }
]
def encode(e) { // help with nulls; quote the separator
(e ?: "").replaceAll(";", "\\;")
}
def csvLine(items) { // write items as "CSV"
println(items.collect{ encode it }.join(";"))
}
def obj = new JsonSlurper().parseText(json)
csvLine(config.keySet())
obj.issues.each{ issue ->
csvLine(config.values().collect{ f -> f issue })
}
result:
key;summary
ART-4070;#[ART] Pre.3 Verification "S"
ART-4069;Verification "C"
ART-4068;#[ART] Enum type
To go with the current code, you could use csvFile.append(...) instead of println inside your
csvLine function and depending on your amount of real data, this might
be a good compromise between performance and resource.
Or you can write the whole CSV at once. E.g.
// prepare whole table
def data = [config.keySet()]
data.addAll(
obj.issues.collect{ issue ->
config.values().collect{ f -> f issue }
}
)
// write table as csv
def csvFile = "/tmp/out.csv" as File
csvFile.text = data.collect{
it.collect{ encode it }.join(";")9
}.join("\n")
Whenever I try to execute below script I keep getting error, saying unexpected character (g).
Basically I want to be able to parse the json response and be able to get the upstream job name from it.
Script:
#Grapes([
#Grab(group='org.codehaus.groovy.modules.http-builder', module='http-builder', version='0.7.1'),
#Grab(group='commons-collections', module='commons-collections', version='3.2.1'),
#Grab(group='org.jsoup', module='jsoup', version='1.10.2'),
#Grab(group='org.json', module='json', version='20190722'),
#Grab(group='com.googlecode.json-simple', module='json-simple', version='1.1.1')
])
import static groovyx.net.http.ContentType.*
import groovyx.net.http.HttpResponseException
import groovyx.net.http.RESTClient
import groovy.json.JsonSlurper
import groovy.json.JsonOutput
import java.net.*
import java.util.*
import org.json.simple.*
import org.json.simple.parser.JSONParser;
def getRestClient(){
String jenkinsUrl="http://somedomainname:8080"
def restClient = new RESTClient(jenkinsUrl)
return restClient
}
def getJobsInfo(String jobname,RESTClient restClient){
def requrl= '/job/'+jobname+'/lastBuild/api/json/?pretty=true'
def response = restClient.get( path : requrl)
return response
}
def writeToPropertyFile(){
def jsonResponseObject = getJobsInfo("sampleJobName",getRestClient())
println "\n\n\n\n\n Json String :---- "+ jsonResponseObject.toString()
JSONParser jsonParser = new JSONParser();
JSONObject jsonObject = (JSONObject) jsonParser.parse(jsonResponseObject.toString());
JSONArray upstreamJobInfoArray = jsonObject.getJSONArray("causes");
for (int i = 0; i < upstreamJobInfoArray.length(); i++) {
JSONObject jobCauses = upstreamJobInfoArray.getJSONObject(i);
String upstreamProjectName = jobCauses.getString("upstreamProject");
println upstreamProjectName
}
}
writeToPropertyFile()
Error :
Json String :---- groovyx.net.http.HttpResponseDecorator#6b063470
Caught: Unexpected character (g) at position 0.
Unexpected character (g) at position 0.
at org.json.simple.parser.Yylex.yylex(Yylex.java:610)
at org.json.simple.parser.JSONParser.nextToken(JSONParser.java:269)
at org.json.simple.parser.JSONParser.parse(JSONParser.java:118)
at org.json.simple.parser.JSONParser.parse(JSONParser.java:81)
at org.json.simple.parser.JSONParser.parse(JSONParser.java:75)
at org.json.simple.parser.JSONParser$parse.call(Unknown Source)
at getUpstreamJob.writeToPropertyFile(getUpstreamJob.groovy:39)
at getUpstreamJob.run(getUpstreamJob.groovy:50)
EDIT 1 : START
JSON response that I am trying to parse :
{
"_class": "hudson.model.FreeStyleBuild",
"actions": [
{
"_class": "hudson.model.CauseAction",
"causes": [
{
"_class": "hudson.model.Cause$UpstreamCause",
"shortDescription": "Started by upstream project \"sampleJobName\" build number 712",
"upstreamBuild": 712,
"upstreamProject": "sampleJobName",
"upstreamUrl": "job/sampleJobName/"
},
{
"_class": "hudson.model.Cause$UserIdCause",
"shortDescription": "Started by user Malick, Asif",
"userId": "asifma00",
"userName": "Malick, Asif"
},
{
"_class": "com.sonyericsson.rebuild.RebuildCause",
"shortDescription": "Rebuilds build #300",
"upstreamBuild": 300,
"upstreamProject": "sampleJobName",
"upstreamUrl": "view/ABCProjectView/job/sampleJobName/"
}
]
},
{
"_class": "hudson.model.ParametersAction",
"parameters": [
{
"_class": "hudson.model.StringParameterValue",
"name": "SNAPSHOTNAME",
"value": "ABCDE_12121.2000-2121212121212"
},
{
"_class": "hudson.model.StringParameterValue",
"name": "BUILD_LABEL",
"value": "ABCDE_12121.2000"
}
]
},
{},
{},
{},
{},
{
"_class": "hudson.plugins.parameterizedtrigger.BuildInfoExporterAction"
},
{},
{},
{},
{}
],
"artifacts": [],
"building": false,
"description": null,
"displayName": "#301",
"duration": 1199238,
"estimatedDuration": 1194905,
"executor": null,
"fullDisplayName": "sampleJobName #301",
"id": "301",
"keepLog": false,
"number": 301,
"queueId": 189076,
"result": "SUCCESS",
"timestamp": 1583500786857,
"url": "http://somedomainname:8080/job/sampleJobName/301/",
"builtOn": "Server12345",
"changeSet": {
"_class": "hudson.scm.EmptyChangeLogSet",
"items": [],
"kind": null
},
"culprits": []
}
EDIT 1 : END
I have tried looking at several Stack Overflow issues, but still haven't been able to resolve it.
Please guide.
It's because you're not getting the actual JSON string, you're getting the toString() output on a groovyx.net.http.HttpResponseDecorator class.
You can see it printing it out here (it's easy to miss though... I missed it the first look 🙂):
Json String :---- groovyx.net.http.HttpResponseDecorator#6b063470
I think you need to call jsonResponseObject.data to get the parsed data, and it might even return you a Map parsed form the JSON (so you can get rid of all your JSONObject code). Test it by changing to:
def data = jsonResponseObject.data
println "\n\n\n\n\n Json ${data.getClass().name} :---- $data" jsonResponseObject.toString()
If it is a java.util.Map, you can simplify your second part from:
JSONParser jsonParser = new JSONParser();
JSONObject jsonObject = (JSONObject) jsonParser.parse(jsonResponseObject.toString());
JSONArray upstreamJobInfoArray = jsonObject.getJSONArray("causes");
for (int i = 0; i < upstreamJobInfoArray.length(); i++) {
JSONObject jobCauses = upstreamJobInfoArray.getJSONObject(i);
String upstreamProjectName = jobCauses.getString("upstreamProject");
println upstreamProjectName
}
To (using data from above):
data.actions.causes.upstreamProject.flatten().each { println it }
predict = [0.1,0.2]
payload = {
"instances": [
{
"features": predict
}
]
}
response = linear_regressor.predict(json.dumps(payload))
predictions = json.loads(response)
print(json.dumps(predictions, indent=2))
The above code is able to invoke the endpoint which is the linear-learner endpoint and give the below result.
{
"predictions": [
{
"score": 0.13421717286109924
}
]
}
But when I try to invoke the endpoint using below lambda function,
import json
import io
import boto3
client = boto3.client('runtime.sagemaker')
def lambda_handler(event, context):
print("Received event: " + json.dumps(event, indent=2))
#data = json.loads(json.dumps(event))
#payload = data['data']
#print(payload)
response = client.invoke_endpoint(EndpointName='linear-learner-2019-12-12-16-xx-xx-xxx',
ContentType='application/json',
Body=json.dumps(event))
return response
I get the following error,
An error occurred during JSON serialization of response:
<botocore.response.StreamingBody object at 0x7fc941e7e828> is not JSON serializable
The input for the lambda function is the same,
{
"instances": [
{
"features": [
0.1,
0.2
]
}
]
}
Not sure what is causing the issue here. Any help would be appreciated, Thanks in advance.
I have the following JSON with in an array and when I try to set a value for this JSON, script passes but value isn't set:
{
"langauageCode": "en-US",
"Test": [{
"_modificationTypeCode": "added",
"allocationTypeCode": "3",
"code": "Test1"
}]
}
My code:
def jsonRequest = slurper.parseText(rawRequest)
def builder = new JsonBuilder(jsonRequest)
builder.content.Test.code[0] ='Test2' //Code value is not getting set to 'Test2'
log.info builder.toPrettyString()
Am I not setting the value correctly?
I assume that slurper is an instance of JsonSlurper. If so, there's no need to use JsonBuilder at all, since sluper returns an instance of a Map. So:
import groovy.json.JsonSlurper
import groovy.json.JsonOutput
def req = '''{
"langauageCode": "en-US",
"Test": [{
"_modificationTypeCode": "added",
"allocationTypeCode": "3",
"code": "Test1"
}]
}'''
def slurped = new JsonSlurper().parseText(req)
slurped.Test[0].code = 'Test2'
println JsonOutput.prettyPrint(JsonOutput.toJson(slurped))
I am new in SparkSQL, I am trying to parse and show the data of a JSON file.
So My question is I cannot understand the Line number 2 in my code given below, why the builder function is not like the Documentation(.setAppName instead of .appName etc..) I have mentioned given below? What does mean of this added portion- "some-value").getOrCreate() in Line number 2 of my code?
I will be thankful from the bottom of my heart if someone help me to understand this.
employee.json
{"name":"John", "age":28}
{"name":"Andrew", "age":36}
{"name":"Clarke", "age":22}
{"name":"Kevin", "age":42}
{"name":"Richard","age":51}
Code:
1. import org.apache.spark.sql.SparkSession
2. val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
3. import spark.implicits._
4. val df = spark.read.json("examples/src/main/resources/employee.json")
5. df.show()
Output:
+---+-------+
|age| name|
+---+-------+
| 28| John|
| 36| Andrew|
| 22| Clarke|
| 42| Kevin|
| 51|Richard|
+---+-------+
=============================>>>>>>>>>>
Please Note:
According to the Documentation of SparkConf passed to your SparkContext. SparkConf allows you to configure some of the common properties (e.g. master URL and application name), as well as arbitrary key-value pairs through the set() method. as follows::
val conf = new
SparkConf().setMaster("local[2]").setAppName("CountingSheep")
val sc = new SparkContext(conf)
See nowadays from Spark 2.0.0 , if you are using the sql part of the Spark then SparkSession is the de-facto standard that you need to follow to create.
In the Second line i.e.
2. val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
You can write this same thing using the spark conf as you used to do it earlier that you have pasted in the note :
val conf = new SparkConf().setMaster("local[2]").setAppName("CountingSheep")
val sc = new SparkContext(conf)
Now what you can do is
val conf = new SparkConf().setMaster("local[2]").setAppName("CountingSheep")
val spark = SparkSession.builder().config(conf).getOrCreate()
And if you want sc to be specific you can extract it from SparkSession variable itself.
val sc = spark.sparkContext
Now coming to your getOrCreate():
See we cannot have two SparkContext for the same application, same goes for the SparkSession so what getOrCreate() does is it searches if there is an already existing SparkContext or SparkSession object and if there is then just get its reference and provide it to the variable. If its not present then create one and pass it on to the variable.
P.S : I hope this explanation helps you. :)
You code is of spark 2.0 or newer verions but the documentation you are referring is of older versions of spark.
In 2.0 version SparkSession was introduced which is a combination of SparkContext for computing rdds and SqlContext for computing SQL queries on dataframes. So the api is changed.
In older versions of spark, SparkConf is used to set the configuration, SparkContext is used to set contexts for rdds and SQLContext is used to set contexts for dataframes and datasets.
val conf = new SparkConf().setMaster("local[2]").setAppName("CountingSheep").set("spark.some.config.option", "some-value")
val sc = new SparkContext(conf)
val sqlContext = new SQLContext(sc)
But in newer version greater than 2.0, it is done as
val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
val sc = spark.sparkContext
val sqlContext = spark.sqlContext
Now spark.some.config.option and some-value are key value pairs of configuation properties. for example spark configuration properties and yarn properties etc.
spark.some.config.option = textinputformat.record.delimiter
some-value = ~%$
I hope the answer is helpful
I try to convert a JSON file to a conf file structure using "io.circe" library.
My implementation is like:
Step: 1
.sbt file looks like:
organization := "com.sample.json.convert"
name := "convert-json-conf"
version := "1.0.0"
scalaVersion := "2.11.8"
scalacOptions ++= Seq("-feature")
javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint")
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % "2.3.0" % "provided",
"io.circe" %% "circe-core" % "0.10.0-M2",
"io.circe" %% "circe-parser" % "0.10.0-M2",
"io.circe" %% "circe-generic" % "0.10.0-M2",
"io.circe" %% "circe-config" % "0.4.1"
)
Step: 2
Sample input JSON file:
{
"id": "0001",
"address" : {
"country": {
"state":{
"city":{
"locality": "some_place"
}
}
}
},
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}
Step: 3
JSON File Reading code:
sealed trait FileRead [T, R] extends (T => R) with Serializable
object FileRead {
implicit object FileReadImpl extends FileRead[String, String] {
override def apply(inputSource: String): String = {
val source = scala.io.Source.fromFile(inputSource)
try source.mkString finally source.close()
}
}
}
Step: 4
JSON to Config file conversion code:
import io.circe.Json
import io.circe.config.{parser, printer}
sealed trait JsonToConfig[T, R] extends (T => R) with Serializable
object JsonToConfig {
implicit object JsonToConfigImpl extends JsonToConfig[String, String] {
override def apply(input: String): String = {
val options = printer.DefaultOptions.setFormatted(false)
val jsonFormatInput = parser.parse(input).right.get
val inputJson = Json.fromJsonObject(jsonFormatInput.asObject.get)
printer.print(inputJson, options)
}
}
}
Step 5:
Execute the program:
object ConversionRun {
def main(args: Array[String]): Unit = {
val file = "sample.json"
import FileRead._
import JsonToConfig._
val fileRead = implicitly[FileRead[String, String]]
val convert = implicitly[JsonToConfig[String, String]]
val out = convert(fileRead(file))
println(out)
}
}
Output:
address{country{state{city{locality="some_place"}}}},batters{batter=[{id="1001",type=Regular},{id="1002",type=Chocolate},{id="1003",type=Blueberry},{id="1004",type="Devil's Food"}]},id="0001",name=Cake,ppu=0.55,topping=[{id="5001",type=None},{id="5002",type=Glazed},{id="5005",type=Sugar},{id="5007",type="Powdered Sugar"},{id="5006",type="Chocolate with Sprinkles"},{id="5003",type=Chocolate},{id="5004",type=Maple}],type=donut