POSTMAN - Schema validation is passed even for bad response data - json

Question

tests["Valid schema"] = tv4.validate(jsonData, schema); is passed even if "error" and "responseType" is missing in the schema. How to make sure that response and schema both are matching for JSON schema.
when I hit post request on postman, the following is the Response Body in postman
{
"statusCode": 400,
"error": "Bad Request",
"message": "Email/Phone number not found",
"responseType": "EMAIL_NOT_FOUND",
"arabicMessage": "البريد الإلكتروني / رقم الهاتف غير موجود"
}
Tests in Postman
var jsonData=JSON.parse(responseBody)
var schema ={
"statusCode": {"type":"integer"},
"message": {"type":"string"},
"arabicMessage":{"type":"string"},
"data": {
"accessToken": {"type":"string"},
"userDetails": {
"_id": {"type":"string"},
"deviceType": {"type":"string"},
"countryCode": {"type":"string"},
"OTPCode": {"type":"integer"},
"invitationCode": {"type":"string"},
"availableCredits": {"type":"integer"},
"totalBookings": {"type":"integer"},
"promoCodes": {"type":"array"},
"updatedAt": {"type":"string"},
"createdAt": {"type":"string"},
"language": {"type":"string"},
"IsDeleted": {"type":"boolean"},
"IsVerified": {"type":"boolean"},
"IsBlock": {"type":"boolean"},
"customerAddresses": {"type":"array"},
"address":{"type":"string"},
"phoneVerified": {"type":"boolean"},
"currentLocation": {
"type": "Point",
"coordinates": [
{"type":"integer"},
{"type":"integer"}
]
},
"appVersion": {"type":"integer"},
"profilePicURL": {
"thumbnail": {"type":"string"},
"original": {"type":"string"}
},
"password": {"type":"string"},
"socialId": {"type":"string"},
"phoneNo": {"type":"integer"},
"email": {"type":"string"},
"LastName": {"type":"string"},
"firstName": {"type":"string"},
"__v": {"type":"integer"},
"referralCode": {"type":"string"},
"accessToken": {"type":"string"},
"deviceToken": {"type":"string"}
},
"updateAvailable": {"type":"boolean"},
"stateCallBookingIds": {"type":"array"},
"forceUpdate": {"type":"boolean"}
}
};
tests["Valid schema"] = tv4.validate(jsonData, schema);
//here the test is passing even with invalid jsonData which is the data
console.log("Validation failed: ", tv4.error);

Answer 1

There are lots of open issues on the Postman github account about the tv4 module.
There is a similar question on SO here, could your jsonData be different than your schema?
This is an example from a link on the tv4 github page.
"title": "Person",
"type": "object",
"properties": {
"firstName": {
"type": "string"
},
"lastName": {
"type": "string"
},
"age": {
"description": "Age in years",
"type": "integer",
"minimum": 0
}
},
"required": ["firstName", "lastName"]
}
You could try adding those fields as required?

Answer 2

Just leaving this here in case it helps anyone else. tv4.validate has two additional boolean parameters: checkRecursive and banUnkownProperties.
Especially the last one can help finding errors in JSON responses when they contain attributes that weren't defined in the schema.
Reference

Answer 3

I prefer to use jsonSchema this way:
var strSchema =
pm.collectionVariables.get("variable_that_contains_the_schema");
pm.test("Validate SCHEMA is OK", () => {
pm.response.to.have.jsonSchema(JSON.parse(strSchema));
});
And if you want to auto-generate the schema, you can use the generate-schema plugin https://www.npmjs.com/package/generate-schema
I use to store the content of this plugin in a collection variable, on the pre-request of my collection, like this:
Then in the test tab of your request, you can call the javascript eval() to load the generate-schema source-code in the postman's test environment:
// eval will evaluate the JavaScript generateSchema code and
// initialize the js on postman environment
eval(pm.collectionVariables.get("generate-schema"));
var strSchema =
pm.collectionVariables.get("variable_that_contains_the_schema");
if (strSchema === undefined){
console.log('>>>> variable for schema not defined!');
return;
}
if(strSchema == '' | strSchema === null) {
// call function generateSchema (external)
var schema = generateSchema.json("variable_that_contains_the_schema", jsonData);
delete schema.$schema; // remove the element '$schema' --> causes error in validator
var strSchema = JSON.stringify(schema);
//save the generated schema
pm.collectionVariables.set("variable_that_contains_the_schema", strSchema);
console.log(" >> schema was generated from the response, validation will proceed on the next request.");
}
else
{
console.log(" >> schema recovered from variable.");
// Schema test ------------------------------------------------
pm.test("Validate SCHEMA is OK", () => {
pm.response.to.have.jsonSchema(JSON.parse(strSchema));
});
}