Artillery output flag does not provide plugin:ensure metric - json

Question

I am currently working on Performance Testing against our API, while facing the following problem.
Running command: artillery run --output reports/report.json tests/dev/get-user-settings.yml, gives back an output/report in JSON format in the terminal, including 'Checks' tag (at the very bottom), that I've included in the plugin portion of the yml configuration file, like so:
However, the issue, is that this same JSON output does not provide the 'Checks' metrics (in my case they're max and maxErrorRate) in the report.json:
{
"aggregate": {
"counters": {
"vusers.created_by_name.0": 300,
"vusers.created": 300,
"http.requests": 300,
"http.codes.200": 300,
"http.responses": 300,
"vusers.failed": 0,
"vusers.completed": 300
},
"rates": {
"http.request_rate": 10
},
"firstCounterAt": 1673975540669,
"firstHistogramAt": 1673975542459,
"lastCounterAt": 1673975570709,
"lastHistogramAt": 1673975570709,
"firstMetricAt": 1673975540669,
"lastMetricAt": 1673975570709,
"period": 1673975570000,
"summaries": {
"http.response_time": {
"min": 409,
"max": 1749,
"count": 300,
"p50": 478.3,
"median": 478.3,
"p75": 507.8,
"p90": 671.9,
"p95": 1465.9,
"p99": 1587.9,
"p999": 1587.9
},
"vusers.session_length": {
"min": 534.6,
"max": 1880.2,
"count": 300,
"p50": 620.3,
"median": 620.3,
"p75": 685.5,
"p90": 837.3,
"p95": 1620,
"p99": 1755,
"p999": 1826.6
}
},
"histograms": {
"http.response_time": {
"min": 409,
"max": 1749,
"count": 300,
"p50": 478.3,
"median": 478.3,
"p75": 507.8,
"p90": 671.9,
"p95": 1465.9,
"p99": 1587.9,
"p999": 1587.9
},
"vusers.session_length": {
"min": 534.6,
"max": 1880.2,
"count": 300,
"p50": 620.3,
"median": 620.3,
"p75": 685.5,
"p90": 837.3,
"p95": 1620,
"p99": 1755,
"p999": 1826.6
}
}
},
"intermediate": [
{
"counters": {
"vusers.created_by_name.0": 97,
"vusers.created": 97,
"http.requests": 97,
"http.codes.200": 90,
"http.responses": 90,
"vusers.failed": 0,
"vusers.completed": 90
},
"rates": {
"http.request_rate": 10
},
"http.request_rate": null,
"firstCounterAt": 1673975540669,
"firstHistogramAt": 1673975542459,
"lastCounterAt": 1673975549975,
"lastHistogramAt": 1673975549912,
"firstMetricAt": 1673975540669,
"lastMetricAt": 1673975549975,
"period": "1673975540000",
"summaries": {
"http.response_time": {
"min": 424,
"max": 1749,
"count": 90,
"p50": 497.8,
"median": 497.8,
"p75": 658.6,
"p90": 1495.5,
"p95": 1556.5,
"p99": 1587.9,
"p999": 1587.9
},
"vusers.session_length": {
"min": 550.9,
"max": 1880.2,
"count": 90,
"p50": 632.8,
"median": 632.8,
"p75": 804.5,
"p90": 1686.1,
"p95": 1720.2,
"p99": 1826.6,
"p999": 1826.6
}
},
"histograms": {
"http.response_time": {
"min": 424,
"max": 1749,
"count": 90,
"p50": 497.8,
"median": 497.8,
"p75": 658.6,
"p90": 1495.5,
"p95": 1556.5,
"p99": 1587.9,
"p999": 1587.9
},
"vusers.session_length": {
"min": 550.9,
"max": 1880.2,
"count": 90,
"p50": 632.8,
"median": 632.8,
"p75": 804.5,
"p90": 1686.1,
"p95": 1720.2,
"p99": 1826.6,
"p999": 1826.6
}
}
},
{
"counters": {
"http.codes.200": 100,
"http.responses": 100,
"vusers.failed": 0,
"vusers.completed": 100,
"vusers.created_by_name.0": 100,
"vusers.created": 100,
"http.requests": 100
},
"rates": {
"http.request_rate": 10
},
"http.request_rate": null,
"firstCounterAt": 1673975550024,
"firstHistogramAt": 1673975550298,
"lastCounterAt": 1673975559974,
"lastHistogramAt": 1673975559701,
"firstMetricAt": 1673975550024,
"lastMetricAt": 1673975559974,
"period": "1673975550000",
"summaries": {
"http.response_time": {
"min": 422,
"max": 753,
"count": 100,
"p50": 468.8,
"median": 468.8,
"p75": 497.8,
"p90": 645.6,
"p95": 685.5,
"p99": 713.5,
"p999": 713.5
},
"vusers.session_length": {
"min": 544,
"max": 1626.6,
"count": 100,
"p50": 608,
"median": 608,
"p75": 645.6,
"p90": 837.3,
"p95": 854.2,
"p99": 1587.9,
"p999": 1587.9
}
},
"histograms": {
"http.response_time": {
"min": 422,
"max": 753,
"count": 100,
"p50": 468.8,
"median": 468.8,
"p75": 497.8,
"p90": 645.6,
"p95": 685.5,
"p99": 713.5,
"p999": 713.5
},
"vusers.session_length": {
"min": 544,
"max": 1626.6,
"count": 100,
"p50": 608,
"median": 608,
"p75": 645.6,
"p90": 837.3,
"p95": 854.2,
"p99": 1587.9,
"p999": 1587.9
}
}
},
{
"counters": {
"http.codes.200": 100,
"http.responses": 100,
"vusers.failed": 0,
"vusers.completed": 100,
"vusers.created_by_name.0": 100,
"vusers.created": 100,
"http.requests": 100
},
"rates": {
"http.request_rate": 10
},
"http.request_rate": null,
"firstCounterAt": 1673975560024,
"firstHistogramAt": 1673975560274,
"lastCounterAt": 1673975569975,
"lastHistogramAt": 1673975569710,
"firstMetricAt": 1673975560024,
"lastMetricAt": 1673975569975,
"period": "1673975560000",
"summaries": {
"http.response_time": {
"min": 409,
"max": 700,
"count": 100,
"p50": 468.8,
"median": 468.8,
"p75": 497.8,
"p90": 518.1,
"p95": 561.2,
"p99": 685.5,
"p999": 685.5
},
"vusers.session_length": {
"min": 534.6,
"max": 1624.8,
"count": 100,
"p50": 608,
"median": 608,
"p75": 632.8,
"p90": 727.9,
"p95": 788.5,
"p99": 837.3,
"p999": 837.3
}
},
"histograms": {
"http.response_time": {
"min": 409,
"max": 700,
"count": 100,
"p50": 468.8,
"median": 468.8,
"p75": 497.8,
"p90": 518.1,
"p95": 561.2,
"p99": 685.5,
"p999": 685.5
},
"vusers.session_length": {
"min": 534.6,
"max": 1624.8,
"count": 100,
"p50": 608,
"median": 608,
"p75": 632.8,
"p90": 727.9,
"p95": 788.5,
"p99": 837.3,
"p999": 837.3
}
}
},
{
"counters": {
"http.codes.200": 10,
"http.responses": 10,
"vusers.failed": 0,
"vusers.completed": 10,
"vusers.created_by_name.0": 3,
"vusers.created": 3,
"http.requests": 3
},
"rates": {
"http.request_rate": 3
},
"firstCounterAt": 1673975570025,
"firstHistogramAt": 1673975570298,
"lastCounterAt": 1673975570709,
"lastHistogramAt": 1673975570709,
"firstMetricAt": 1673975570025,
"lastMetricAt": 1673975570709,
"period": "1673975570000",
"http.request_rate": null,
"summaries": {
"http.response_time": {
"min": 445,
"max": 587,
"count": 10,
"p50": 468.8,
"median": 468.8,
"p75": 478.3,
"p90": 550.1,
"p95": 550.1,
"p99": 550.1,
"p999": 550.1
},
"vusers.session_length": {
"min": 571.5,
"max": 730.5,
"count": 10,
"p50": 596,
"median": 596,
"p75": 620.3,
"p90": 699.4,
"p95": 699.4,
"p99": 699.4,
"p999": 699.4
}
},
"histograms": {
"http.response_time": {
"min": 445,
"max": 587,
"count": 10,
"p50": 468.8,
"median": 468.8,
"p75": 478.3,
"p90": 550.1,
"p95": 550.1,
"p99": 550.1,
"p999": 550.1
},
"vusers.session_length": {
"min": 571.5,
"max": 730.5,
"count": 10,
"p50": 596,
"median": 596,
"p75": 620.3,
"p90": 699.4,
"p95": 699.4,
"p99": 699.4,
"p999": 699.4
}
}
}
]
}
Here, I will also attach modified get-user-settings.yml file (due to corporate sensitive info) below:
config:
#Ensure api-id is up to date.
target: "api/endpoint"
plugins:
ensure: {}
phases:
- duration: 30 #seconds
arrivalRate: 10 #requests per second
http:
timeout: 30
name: lambda-name Load Test
defaults:
headers:
# replace TOKEN with authorization token
authorization: "Bearer {{jwt}}"
ensure:
max: 2000 #ms are allowed for the maximum recorded response time
maxErrorRate: 1 #% of the returned 500 error codes are allowed
scenarios:
- flow:
# Update urls as required.
- get:
url: "/settings"
I have also tried using DEBUG env variable to expand the possibilities as to how to indicate plugins:ensure metrics into the report.json file, but so far no luck.
DEBUG=plugin:expect artillery run tests/dev/get-user-settings.yml