Related
[PS : I have gone through all the possible issues under same error and tried to fix my Json Schema]
I have the following Avro Schema (Converted to JSON using http://json-schema-validator.herokuapp.com/avro.jsp) :
{
"definitions" : {
"record:parentTest.test.MyPlatformData" : {
"description" : "representsthetestevent",
"type" : "object",
"required" : [ "event", "subjects" ],
"additionalProperties" : false,
"properties" : {
"event" : {
"$ref" : "#/definitions/record:parentTest.test.event_record"
},
"subjects" : {
"type" : "array",
"items" : {
"$ref" : "#/definitions/record:parentTest.test.TESTEVENT_record"
}
}
}
},
"record:parentTest.test.event_record" : {
"type" : "object",
"required" : [ "created_at", "id", "event_at", "subject", "action", "up_event_id", "up_source" ],
"additionalProperties" : false,
"properties" : {
"created_at" : {
"type" : "string"
},
"id" : {
"type" : "integer",
"minimum" : -2147483648,
"maximum" : 2147483647
},
"event_at" : {
"type" : "string"
},
"subject" : {
"type" : "string"
},
"action" : {
"type" : "string"
},
"up_event_id" : {
"default" : "null",
"oneOf" : [ {
"type" : "string"
}, {
"type" : "null"
} ]
},
"up_source" : {
"default" : "null",
"oneOf" : [ {
"type" : "string"
}, {
"type" : "null"
} ]
}
}
},
"record:parentTest.test.TESTEVENT_record" : {
"type" : "object",
"required" : [ "mydata", "id", "user_id", "data_id", "place_id", "my_object" ],
"additionalProperties" : false,
"properties" : {
"mydata" : {
"$ref" : "#/definitions/record:parentTest.test.mydata_record"
},
"id" : {
"type" : "integer",
"minimum" : -9223372036854775808,
"maximum" : 9223372036854775807
},
"user_id" : {
"type" : "integer",
"minimum" : -9223372036854775808,
"maximum" : 9223372036854775807
},
"data_id" : {
"type" : "integer",
"minimum" : -9223372036854775808,
"maximum" : 9223372036854775807
},
"place_id" : {
"type" : "integer",
"minimum" : -2147483648,
"maximum" : 2147483647
},
"my_object" : {
"$ref" : "#/definitions/record:parentTest.test.my_object_record"
}
}
},
"record:parentTest.test.mydata_record" : {
"type" : "object",
"required" : [ "id", "del" ],
"additionalProperties" : false,
"properties" : {
"id" : {
"type" : "integer",
"minimum" : -9223372036854775808,
"maximum" : 9223372036854775807
},
"del" : {
"$ref" : "#/definitions/record:parentTest.test.del_record"
}
}
},
"record:parentTest.test.del_record" : {
"type" : "object",
"required" : [ "status", "track_details", "cr", "service_level" ],
"additionalProperties" : false,
"properties" : {
"status" : {
"$ref" : "#/definitions/record:parentTest.test.status_record"
},
"tracking_details" : {
"type" : "array",
"items" : {
"$ref" : "#/definitions/record:parentTest.test.tracking_number_detail_record"
}
},
"cr" : {
"$ref" : "#/definitions/record:parentTest.test.cr_record"
},
"service_level" : {
"$ref" : "#/definitions/record:parentTest.test.service_level_record"
}
}
},
"record:parentTest.test.status_record" : {
"type" : "object",
"required" : [ "name", "status_at" ],
"additionalProperties" : false,
"properties" : {
"name" : {
"type" : "string"
},
"status_at" : {
"type" : "string"
}
}
},
"record:parentTest.test.tracking_number_detail_record" : {
"type" : "object",
"required" : [ "id", "cr", "status" ],
"additionalProperties" : false,
"properties" : {
"id" : {
"type" : "string"
},
"cr" : {
"$ref" : "#/definitions/record:parentTest.test.tracking_cr_record"
},
"status" : {
"$ref" : "#/definitions/record:parentTest.test.trac_status_record"
}
}
},
"record:parentTest.test.tracking_cr_record" : {
"type" : "object",
"required" : [ "id" ],
"additionalProperties" : false,
"properties" : {
"id" : {
"type" : "integer",
"minimum" : -2147483648,
"maximum" : 2147483647
}
}
},
"record:parentTest.test.tracking_status_record" : {
"type" : "object",
"required" : [ "name", "status_at" ],
"additionalProperties" : false,
"properties" : {
"name" : {
"oneOf" : [ {
"type" : "string"
}, {
"type" : "null"
} ]
},
"status_at" : {
"oneOf" : [ {
"type" : "string"
}, {
"type" : "null"
} ]
}
}
},
"record:parentTest.test.cr_record" : {
"type" : "object",
"required" : [ "id" ],
"additionalProperties" : false,
"properties" : {
"id" : {
"type" : "integer",
"minimum" : -2147483648,
"maximum" : 2147483647
}
}
},
"record:parentTest.test.service_level_record" : {
"type" : "object",
"required" : [ "sc_id", "ss_id", "ss_class" ],
"additionalProperties" : false,
"properties" : {
"sc_id" : {
"type" : "integer",
"minimum" : -2147483648,
"maximum" : 2147483647
},
"ss_id" : {
"type" : "integer",
"minimum" : -2147483648,
"maximum" : 2147483647
},
"ss_class" : {
"type" : "string"
}
}
},
"record:parentTest.test.mydata_record" : {
"type" : "object",
"required" : [ "s_id" ],
"additionalProperties" : false,
"properties" : {
"s_id" : {
"oneOf" : [ {
"type" : "integer",
"minimum" : -2147483648,
"maximum" : 2147483647
}, {
"type" : "null"
} ]
}
}
}
},
"$ref" : "#/definitions/record:parentTest.test.MyPlatformData"
}
However on inserting the following data, I get following error
Input Data
{
"event": {
"action": "TESTEVENT",
"created_at": "2021-01-21T14:16:23+00:00",
"event_at": "2021-01-21T14:16:23.747",
"id": 28001755,
"subject": "TEST_ITEM",
"up_source": {
"string": "TEST_W"
},
"up_event_id": {
"string": "59c7eec8-9bf6-4907-9df9-628d92abd278"
}
},
"subjects": [
{
"id": 12345678,
"user_id": 5197509841,
"data_id": 3497218791,
"place_id": 49,
"mydata": {
"s_id": {
"int": 29489
}
},
"my_object": {
"id": 5417499011,
"del": {
"status": {
"name": "TESTEVENT",
"status_at": "2021-01-21T09:09:00"
},
"tracking_details": [
{
"id": "CS303755237",
"cr": {
"id": 961
},
"status": {
"name": {
"string": "TESTEVENT"
},
"status_at": {
"string": "2021-01-21T09:09:00"
}
}
}
],
"cr": {
"id": 961
},
"service_level": {
"sc_id": 2,
"ss_id": 5,
"ss_class": "MY_TEST_SS"
}
}
}
}
]
}
Error
Caused by: org.apache.avro.AvroTypeException: Expected start-union. Got VALUE_NUMBER_INT
I was able to finally find the issue.
Being a beginner, I wasn't able to comprehend that time.
For anyone facing similar issue :
Rule : Whenever a field is marked "datatype" and "null" both, it is
important to mention the datatype, if the value is not null.
For example :
For Avro schema, field definition :
"up_source" : {
"default" : "null",
"oneOf" : [ {
"type" : "string"
}, {
"type" : "null"
} ]
}
Possible declarations :
"up_source": {
"string": "TEST_123"
},
OR
"up_source": null,
One of my fields was missing this.
I am trying to validate this specific schema:
{
"messages": [
{
"name": "test msg",
"id": "0x100",
"signals": {
"0": {"name": "Engine RPM", "bit_length": 16},
"16": {"name": "Gear", "bit_length": 3},
"19": {"name": "Battery Voltage", "bit_length": 5}
}
}
]
}
I am using the python from jsonschema import Draft4Validator to validate this schema... however I am not sure how to continue.
This is my current schema validation so far:
{
"$schema" : "https://json-schema.org/schema#",
"type" : "object",
"properties" :
{
"messages" :
{
"type" : "array",
"items" :
{
"properties" :
{
"name" :
{
"type" : "string"
},
"id" :
{
"type" : "string"
},
"signals" :
{
"type" : "object"
},
"properties" :
{
}
},
"required": ["name", "id", "signals"]
}
}
}
}
The problem I am facing is I am not sure how to deal with the objects that are in the "signals" key as they start with a string and are NOT consistent ("0", "16", "19")... How could I go about validating this by ensuring the type is always a string, disregarding whether or not the string is consistent.
Thanks to all of those who reply in advance.
I was able to accomplish this by doing the following:
{
"$schema" : "https://json-schema.org/schema#",
"type" : "object",
"properties" :
{
"messages" :
{
"type" : "array",
"items" :
{
"properties" :
{
"name" :
{
"type" : "string"
},
"id" :
{
"type" : "string"
},
"signals" :
{
"type" : "object"
},
"properties" :
{
"start_bit" :
{
"type" : "object",
"properties" :
{
"name" :
{
"type" : "string"
},
"bit_length" :
{
"type" : "string"
},
"factor" :
{
"type" : "string"
},
"offset" :
{
"type" : "string"
}
},
"required" : ["name", "bit_length", "factor", "offset"]
}
}
},
"required": ["name", "id", "signals"]
}
}
}
}
To "avoid" having to keep the string consistent, in the validator file, I can put any string (obviously it makes more sense to name the string what is represents, in my case "start_bit") and then by NOT having it be required.
I have to automate this line "AWS": "arn:aws:iam::684821578293:user/jenkins" on my cloudformation template but while using join it will not working can somebody help me in this.
Working template is below you can use following snap to parameters list
StackName : test
CreateCodeDeployRole : false
CreateECSRole: false
CreateJenkinsRole: true
CustomerPrefix : kfc ( anyname)
Environment : dt
GroupName : sogetiadmin
RoleName: Jenkins_Tool_Access
UserName: jenkins
https://s3.amazonaws.com/linuxblogger-k8s-state/iamcreation_working.json
Problem :
But once i update this entry on working template from "AWS": "arn:aws:iam::684821578293:user/admin" to "AWS": "arn:aws:iam::684821578293:user/jenkins" it will not working.
I try with join function with Jenkins user but it won't working you can view this json from below
https://s3.amazonaws.com/linuxblogger-k8s-state/iamcreation_not_working.json
{
"AWSTemplateFormatVersion" : "2010-09-09",
"Description" : "IAM groups and account-wide role configurations",
"Parameters" : {
"CustomerPrefix" : {
"Type" : "String",
"Default" : "testcust",
"Description" : "Enter Customer Prefix"
},
"Environment" : {
"Type" : "String",
"Default" : "dt",
"Description" : "Enter Environment (Input Format - d=development, t=test, a=acceptance, p=production, dt=devtest, ap=acceptanceproduction)",
"AllowedValues" : [
"d",
"t",
"a",
"p",
"dt",
"ap"
]
},
"CreateCodeDeployRole" : {
"Type" : "String",
"Default" : "true",
"Description" : "Whether a role should be created for use with AWS CodeDeploy",
"AllowedValues" : ["true", "false"],
"ConstraintDescription" : "Must be true or false."
},
"CreateECSRole" : {
"Type" : "String",
"Default" : "true",
"Description" : "Whether a role should be created for use with AWS EC2 Container Service",
"AllowedValues" : ["true", "false"],
"ConstraintDescription" : "Must be true or false."
},
"CreateJenkinsRole" : {
"Type" : "String",
"Default" : "true",
"Description" : "Whether a role should be created for use with Aws Jenkins Service",
"AllowedValues" : ["true", "false"],
"ConstraintDescription" : "Must be true or false."
},
"UserName" : {
"Type" : "String",
"Default" : "jenkins",
"Description" : "Please Provide Name of the IAM user"
},
"RoleName" : {
"Type" : "String",
"Default" : "Jenkins_Tool_Access",
"Description" : "Please Provide Name of the IAM Role"
},
"GroupName" : {
"Type" : "String",
"Default" : "sogetiadmin",
"Description" : "Please Provide Name of the IAM Role"
}
},
"Conditions" :{
"IsDev" : {
"Fn::Equals" : [ { "Ref" : "Environment" }, "dev" ]
},
"IsQet" : {
"Fn::Equals" : [ { "Ref" : "Environment" }, "qet" ]
},
"IsStg" : {
"Fn::Equals" : [ { "Ref" : "Environment" }, "stg" ]
},
"IsPrd" : {
"Fn::Equals" : [ { "Ref" : "Environment" }, "prd" ]
},
"CreateCodeDeployRole" : {
"Fn::Equals" : [ { "Ref" : "CreateCodeDeployRole" }, "true" ]
},
"CreateECSRole" : {
"Fn::Equals" : [ { "Ref" : "CreateECSRole" }, "true" ]
},
"CreateJenkinsRole" : {
"Fn::Equals" : [ { "Ref" : "CreateJenkinsRole" }, "true" ]
}
},
"Resources" : {
"AWSCodeDeployRole" : {
"Type" : "AWS::IAM::Role",
"Condition" : "CreateCodeDeployRole",
"Properties" : {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": {
"Fn::Join": [
".",
[
"codedeploy",
{ "Ref" : "AWS::Region" },
"amazonaws.com"
]
]
}
},
"Action": "sts:AssumeRole"
}
]
},
"Policies" : [
{
"PolicyName" : "AWSCodeDeployPolicy",
"PolicyDocument" : {
"Statement": [
{
"Action": [
"autoscaling:PutLifecycleHook",
"autoscaling:DeleteLifecycleHook",
"autoscaling:RecordLifecycleActionHeartbeat",
"autoscaling:CompleteLifecycleAction",
"autoscaling:DescribeAutoscalingGroups",
"autoscaling:PutInstanceInStandby",
"autoscaling:PutInstanceInService",
"ec2:Describe*"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"s3:Get*",
"s3:List*"
],
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"-",
[
"arn:aws:s3:::deployments",
{ "Ref" : "CustomerPrefix" },
{ "Ref" : "Environment" },
"/artifacts/projects/*"
]
]
}
}
]
}
}
]
}
},
"JenkinsUser" : {
"Type" : "AWS::IAM::User",
"Condition" : "CreateJenkinsRole",
"Properties" : {
"UserName" : { "Ref" : "UserName" },
"ManagedPolicyArns":
[
"arn:aws:iam::aws:policy/AdministratorAccess"
]
}
},
"AWSJenkinsServiceRole" : {
"Type": "AWS::IAM::Role",
"Condition" : "CreateJenkinsRole",
"DependsOn" : "JenkinsUser",
"Properties" : {
"RoleName": { "Ref" : "RoleName" },
"AssumeRolePolicyDocument": {
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "cloudformation.amazonaws.com"
},
"Action": "sts:AssumeRole"
},
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": { "Fn::Join" : [ "/", [ "arn:aws:iam::684821578293:user", { "Ref" : "UserName" } ]]},
"Service": "cloudformation.amazonaws.com"
},
"Action": "sts:AssumeRole"
}]
},
"ManagedPolicyArns":
[
"arn:aws:iam::aws:policy/AdministratorAccess"
]
}
},
"JenkinsUserAccessKey" : {
"Type" : "AWS::IAM::AccessKey",
"Properties" : {
"UserName" : { "Ref" : "JenkinsUser" }
}
},
"ServiceAccountsGroup" : {
"Type": "AWS::IAM::Group",
"Properties" : {
"GroupName" : { "Ref" : "GroupName" }
}
},
"UserToGroupAddition" : {
"Type": "AWS::IAM::UserToGroupAddition",
"Properties" : {
"GroupName" : { "Ref" : "ServiceAccountsGroup" },
"Users" : [ { "Ref" : "UserName" } ]
}
}
},
"Outputs" : {
"JenkinsUserAccessKey" : {
"Description" : "The access key for the Jenkins user",
"Value" : { "Ref" : "JenkinsUserAccessKey" }
},
"JenkinsUserSecret" : {
"Description" : "The secret key for the Jenkins user",
"Value" : { "Fn::GetAtt" : [ "JenkinsUserAccessKey", "SecretAccessKey" ] }
}
}
}
I'm new to Elasticsearch and I have the following question about parameter comparison at filter/query:
In SQL I have something like the code below, where I compare a SQL value with a parameter value and the parameter value with a fixed value
WHERE (customer.id = :customer_id OR :customer_id = '111')
The left part of the OR above is ok, but how I compare the right part?
{ "bool": {
"should": [
{ "term":
{ "customer.id": "#customer_id" }
},
{
????
}
]
}}
Thanks in advance.
UPDATE
The mapping:
{
"_id" : {
"store" : true,
"index" : "not_analyzed"
},
"_timestamp" : {
"enabled" : true,
"store" : true
},
"properties" : {
...
"processDefinition.id" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"processDefinition.key" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"processDefinition.name" : {
"type" : "string"
},
"variables" : {
"type" : "nested",
"properties" : {
...
"executionId" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"id" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"name" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"originalType" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"rawValue" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"storedType" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"stringValue" : {
"type" : "string"
}
}
}
}
}
I want every "variables.rawValue": "#parameter" OR "#parameter": "ABC". Keep in mind that if #parameter is equals to "a", then the filter returns only info related to "a". If #parameter is equals to "ABC", then the filter returns info related to everything, because company ABC is owner of everything sub company (like "a). I can have a user from "a" using the filter or a user from "ABC".
I tried this:
"bool": {
"should": {
"terms": {
"variables.rawValue": [
"#currentUser_company",
"ABC"
]
}
}
}
But the filter join the result from "a" with "ABC".
One last thing. The ES version is 1.7.
Have you tried this:
{
"query": {
"bool": {
"should": [
{
"terms": {
"FIELD": [
"#customer_id",
"111"
]
}
}
]
}
}
}
#customer_id is the value which user passes
The following schema and document validate just fine even though the last line of the document uses "none" as a value which is not in the enum.
{
"$schema" : "http://json-schema.org/draft-04/schema#",
"title" : "Test schema",
"definitions" : {
"xxx_type" : {
"enum" : [ "X1", "X2", "X3" ]
},
"xxx_info" : {
"type" : "object",
"properties" : {
"date" : { "type" : "string" },
"category" : {
"type" : "array",
"items" : { "$ref" : "#/definitions/xxx_type" }
}
},
"required" : [ "date", "category" ]
}
},
"XXX" : {
"type" : "array",
"items" : {
"$ref" : "#/definitions/xxx_info"
}
}
}
{
"XXX" : [ { "date" : "2015/01/01", "category" : [ "X1" ] },
{ "date" : "2015/02/01", "category" : [ "X2" ] },
{ "date" : "2015/03/01", "category" : [ "X3" ] }
{ "date" : "2015/04/01", "category" : [ "none" ] }
]
}
The following says invalid. (The two lines before the "XXX" definition are the only real difference.) Here the "none" is invalid.
{
"$schema" : "http://json-schema.org/draft-04/schema#",
"title" : "Test schema",
"definitions" : {
"xxx_type" : {
"enum" : [ "X1", "X2", "X3" ]
},
"xxx_info" : {
"type" : "object",
"properties" : {
"date" : { "type" : "string" },
"category" : {
"type" : "array",
"items" : { "$ref" : "#/definitions/xxx_type" }
}
},
"required" : [ "date", "category" ]
}
},
"type" : "object",
"properties" : {
"XXX" : {
"type" : "array",
"items" : {
"$ref" : "#/definitions/xxx_info"
}
}
}
}
{
"XXX" : [ { "date" : "2015/01/01", "category" : [ "X1" ] },
{ "date" : "2015/02/01", "category" : [ "X2" ] },
{ "date" : "2015/03/01", "category" : [ "X3" ] }
{ "date" : "2015/04/01", "category" : [ "none" ] }
]
}
The first one says it is valid because it does not validate the xxx property.
But on the second one you have changed the jsonSchema to meet the JSON document and this time it is validating the xxx property using the jsonSchema and the property is invalid because it contains an invalid node the one with the category "none"