Logstash nested array JSON parsing - json

Question

I have a JSON log of my application which contains elements in a nested array form, here is the sample of it:-
{
"msgs": [{
"ts": "2017-09-04T07:07:45.6229372Z",
"tid": 25,
"eid": 1,
"lvl": "Information",
"cat": "Microsoft.AspNetCore.Hosting.Internal.WebHost",
"msg": {
"cnt": "Request starting HTTP/1.1 POST http://localhost:20001/Processor text/xml; charset=utf-8 685",
"Protocol": "HTTP/1.1",
"Method": "POST",
"ContentType": "text/xml; charset=utf-8",
"ContentLength": 685,
"Scheme": "http",
"Host": "localhost:20001",
"PathBase": "",
"Path": "/Processor",
"QueryString": ""
}
},
{
"ts": "2017-09-04T07:07:45.6229372Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.Service.ServiceHostMiddleware",
"msg": {
"cnt": "REQ"
},
"data": {
"Headers": {
"Connection": "Keep-Alive",
"Content-Length": "685",
"Content-Type": "text/xml; charset=utf-8",
"Accept-Encoding": "gzip, deflate",
"Expect": "100-continue",
"Host": "localhost:20001",
"SOAPAction": "\"http://servereps.mtxeps.com/TransactionService/SendTransaction\""
}
}
},
{
"ts": "2017-09-04T07:07:45.6239372Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.ProcessorTransactionService",
"msg": {
"cnt": "REQ"
},
"data": {
"Trace": "Aa10031<1C>Ab1<1C>Ac000101<1C>Ad20170616145857<1C>Ae10011<1C>AhVT00009<1C>ArY<1C>Be1812<1C>BfS<1C>BnDB<1C>Bo400296<1C>Bp4803<1C>BqDebit Card<1C>Br16<1C>Da300<1C>Dc0<1C>Dk840<1C>Ga01<1C>Gb1<1C>GfG<1C>GhB/GsF>O<1C>GkOE1<1C>Ia7325980B6B284759<1C>Ib8765432100324A00313<1C>Ic0043<1C>Ig3|1|MX2015-06<1C>IjgnkA7MdNlE2EB1c2B3jlz1G3Kf2U5S3x2H9W6ldIY7QpmlUzYcNHGg==|FFFF987654323D2007CD<1C>Nc5<1C>Oa18"
}
},
{
"ts": "2017-09-04T07:07:45.6249373Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
"msg": {
"cnt": "REQ"
},
"data": {
"request": {
"Version": "1.0",
"Server": {
"LogicalDatacenterId": 0,
"PhysicalDatacenterId": 0,
"UniqueId": "G00008D4F35EDADCD368",
"UniversalTimestamp": "2017-09-04T07:07:45.6249373Z"
},
"State": {
"Status": "InFlight"
},
"Tenant": {
"CompanyNumber": 10031,
"StoreNumber": 1,
"HostType": {
"Code": 7,
"Name": "ProdConcordHC"
},
"MerchantNumber": "",
"TerminalId": ""
},
"PointOfInteraction": {
"Client": {
"Type": "OpenEPS",
"CardType": "DB",
"CardName": "Debit Card",
"CommandSequence": "B/GsF>O"
},
"Lane": {
"Type": "Attended",
"Number": 1
},
"Terminal": {
"EMVCapabilities": "FullEMV",
"SerialNumber": "0043"
},
"PointOfSale": {
"ReferenceNumber": "VT00009",
"CashierNumber": "1"
},
"LocalTimestamp": "2017-06-16T14:58:57"
},
"Request": {
"MessageType": {
"Code": 0,
"Name": "Standard"
},
"TenderType": {
"Code": 1,
"Name": "Debit"
},
"TransactionType": {
"Code": 1,
"Name": "Purchase"
},
"EntryMode": "Swiped",
"AuditId": 10011,
"CardType": {
"TenderType": {
"Code": 0,
"Name": "Unknown"
},
"Code": 0,
"Name": "Unknown"
},
"AccountNumberFirstSix": "400296",
"AccountNumberLastFour": "4803",
"AccountNumberLength": 16,
"Card": {
"EncryptedTrack": {
"EncryptionType": "Hardware",
"EncryptionKeySerialNumber": "FFFF987654323D2007CD",
"EncryptedValue": "gnkA7MdNlE2EB1c2B3jlz1G3Kf2U5S3x2H9W6ldIY7QpmlUzYcNHGg=="
},
"EncryptedPIN": {
"EncryptionType": "PassThrough",
"EncryptionKeySerialNumber": "8765432100324A00313",
"EncryptedValue": "7325980B6B284759"
}
},
"Currency": {
"Number": 840
},
"Amount": 3.0,
"CashbackAmount": 0.0,
"AllowPartialAuthorization": true,
"Host": {
"HostType": {
"Code": 7,
"Name": "ProdConcordHC"
},
"Values": {
"HostProfileId": 1,
"CompanyNumber": 10031,
"StoreNumber": 1,
"StoreProfileId": 1,
"NumericStateCode": 0,
"AlphaStateCode": "NH",
"CheckAuthService": "3"
}
}
},
"Trace": {
"Path": [{
"Type": "OpenEPS",
"Value": "OE1"
},
{
"NodeId": 0,
"Type": "Gateway",
"Value": "GAT"
},
{
"NodeId": 0,
"ElapsedSeconds": 0.7458587,
"Type": "Tenant",
"Value": "TEN"
}]
}
},
"timeoutInMilliseconds": 45000
}
},
{
"ts": "2017-09-04T07:07:45.6259373Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TenantResolutionClient",
"msg": {
"cnt": "SEND"
},
"data": {
"RequestUri": "http://153.71.66.148:20002/TenantConfigurationService/v1/Resolution",
"Method": "POST",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:45 GMT",
"User-Agent": "ConnectedPayments/1.0",
"x-ms-request-root-id": "6627de44-43a027c7567ddccd",
"x-ms-request-id": "|6627de44-43a027c7567ddccd.1.",
"Request-Id": "|6627de44-43a027c7567ddccd.1.",
"Correlation-Context": "UniqueId=G00008D4F35EDADCD368",
"Content-Type": "application/json; charset=utf-8",
"Content-Length": "181"
}
}
},
{
"ts": "2017-09-04T07:07:46.37098Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TenantResolutionClient",
"msg": {
"cnt": "RECV"
},
"data": {
"StatusCode": 200,
"ReasonPhrase": "OK",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:46 GMT",
"Transfer-Encoding": "chunked",
"X-Unique-Id": "G00008D4F35EDADCD368",
"X-Node-Id": "0",
"Content-Type": "application/json; charset=utf-8"
}
}
},
{
"ts": "2017-09-04T07:07:46.37098Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
"msg": {
"cnt": "Resolution performed."
},
"data": {
"Response": {
"Content": {
"Company": {
"Number": 10031,
"Name": "Sprouts"
},
"TransactionProfile": {
"TransactionProfileId": 1,
"CompanyNumber": 10031,
"Number": 1000,
"Description": "Transaction Profile 1000"
},
"TransactionValidation": {
"TransactionValidationId": 1,
"TransactionProfileId": 1,
"MessageTypeCode": 0,
"TenderTypeCode": 1,
"TransactionTypeCode": 1,
"CardTypeCode": 0
},
"Store": {
"Number": 1,
"Name": "Store 1",
"QueueLaneNumber": 0
},
"StoreProfile": {
"StoreProfileId": 1,
"CompanyNumber": 10031,
"StoreNumber": 1,
"TransactionProfileId": 1,
"HostTypeCode": 7,
"MerchantNumber": "",
"TerminalId": ""
},
"HostProfile": {
"HostProfileId": 1,
"CompanyNumber": 10031,
"StoreNumber": 1,
"StoreProfileId": 1,
"NumericStateCode": 0,
"AlphaStateCode": "NH",
"CheckAuthService": "3"
}
},
"NodeId": 0,
"StatusCode": 200
},
"ElapsedSeconds": 0.7458587
}
},
{
"ts": "2017-09-04T07:07:46.37198Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TransactionAuthorizationClient",
"msg": {
"cnt": "SEND"
},
"data": {
"RequestUri": "http://ser22vvm211:8082/OrchestrationService/v1/Authorization",
"Method": "POST",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:46 GMT",
"User-Agent": "ConnectedPayments/1.0",
"x-ms-request-root-id": "6627de44-43a027c7567ddccd",
"x-ms-request-id": "|6627de44-43a027c7567ddccd.2.",
"Request-Id": "|6627de44-43a027c7567ddccd.2.",
"Correlation-Context": "UniqueId=G00008D4F35EDADCD368",
"Content-Type": "application/json; charset=utf-8",
"Content-Length": "1821"
}
}
},
{
"ts": "2017-09-04T07:07:49.8941815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.SDK.TransactionAuthorizationClient",
"msg": {
"cnt": "RECV"
},
"data": {
"StatusCode": 200,
"ReasonPhrase": "OK",
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:49 GMT",
"Server": "Apache-Coyote/1.1",
"Content-Length": "1672",
"Content-Type": "application/json"
}
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
"msg": {
"cnt": "RSP"
},
"data": {
"ResponseTransaction": {
"Version": null,
"Server": {
"LogicalDatacenterId": 0,
"PhysicalDatacenterId": 0,
"TransactionId": 10909,
"UniqueId": "G00008D4F35EDADCD368",
"UniversalTimestamp": "2017-09-04T07:07:45.6249373Z"
},
"State": null,
"Tenant": {
"CompanyNumber": 10031,
"StoreNumber": 1,
"HostType": {
"Code": 7,
"Name": "ProdConcordHC"
},
"MerchantNumber": "",
"TerminalId": ""
},
"PointOfInteraction": {
"Client": {
"Type": "B/GsF>O",
"CardType": "DB",
"CardName": "Debit Card",
"CommandSequence": "B/GsF>O",
"Version": null
},
"Lane": {
"Type": "Attended",
"Number": 1
},
"Terminal": {
"EMVCapabilities": "Contact, ContactlessDisabled",
"Type": null,
"SerialNumber": null,
"EMVKernelVersion": null,
"EMVIdentifierCAPK": null,
"PINCapabilities": null
},
"PointOfSale": {
"CashierNumber": "1",
"ReferenceNumber": null
},
"LocalTimestamp": "2017-06-16T14:58:57",
"LocalTimeZoneOffset": null
},
"Request": {
"MessageType": null,
"EntryMode": null,
"CashbackAmount": 0.00,
"AllowPartialAuthorization": true,
"TenderType": null,
"TransactionType": null,
"ReversalType": null,
"ReferenceId": null,
"AuditId": null,
"AccountNumberFirstSix": null,
"AccountNumberLastFour": null,
"AccountNumberLength": null,
"CardType": null,
"PreviousTransaction": null,
"Card": null,
"Check": null,
"Identification": null,
"Currency": null,
"Amount": null,
"TipAmount": null,
"Host": null
},
"Response": {
"ErrorCode": 96,
"ErrorMessage": "E2G",
"IsApproved": false,
"ResponseCode": "96",
"HostResponseCode": "E2G",
"HostResponseMessage": "EDIT ER:OPT DATA",
"ApprovedAmount": 0.00,
"ApprovedCashbackAmount": 0.0,
"IsApprovedLocally": false,
"AuthorizationCode": null,
"ResponseMessage": null,
"BalanceAmount": null,
"Check": null,
"Currency": null,
"Card": null,
"Host": null
},
"Trace": {
"ElapsedSeconds": 4.2694351,
"Path": [{
"NodeId": 0,
"ElapsedSeconds": 4.2694351,
"Type": "Gateway",
"Value": "GAT"
}]
},
"Queue": null,
"SensitiveData": null,
"PreviousTransaction": null,
"Trace": null
}
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.GatewayService.ProcessorTransactionService",
"msg": {
"cnt": "RSP"
},
"data": {
"Trace": "Aa10031<1C>Ab1<1C>Ac000101<1C>Ad20170616145857<1C>Ae10011<1C>Af96<1C>ArY<1C>BnDB<1C>BqDebit Card<1C>Db0<1C>Dc0<1C>Ga1<1C>Gb1<1C>GfG<1C>GhB/GsF>O<1C>GkGAT<1C>Jb96<1C>Ka7<1C>MbE2G<1C>Mg0<1C>Nc5<1C>Ya10909<1C>Yb0<1C>YcG00008D4F35EDADCD368"
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"lvl": "Information",
"cat": "NCR.CP.Service.ServiceHostMiddleware",
"msg": {
"cnt": "RSP"
},
"data": {
"Headers": {
"Date": "Mon, 04 Sep 2017 07:07:49 GMT",
"Transfer-Encoding": "chunked",
"Content-Type": "text/xml; charset=utf-8",
"X-Unique-Id": "G00008D4F35EDADCD368",
"X-Node-Id": "0"
}
}
},
{
"ts": "2017-09-04T07:07:49.8951815Z",
"tid": 25,
"eid": 2,
"lvl": "Information",
"cat": "Microsoft.AspNetCore.Hosting.Internal.WebHost",
"msg": {
"cnt": "Request finished in 4272.8738ms 200 text/xml; charset=utf-8",
"ElapsedMilliseconds": 4272.8738,
"StatusCode": 200,
"ContentType": "text/xml; charset=utf-8"
}
}],
"RequestId": "G00008D4F35EDADCD368",
"RequestPath": "/Processor",
"Action": "http://servereps.mtxeps.com/TransactionService/SendTransaction",
"Contract": "NCR.CP.GatewayService.IProcessorTransactionContract",
"OperationName": "SendTransaction",
"MethodName": "SendTransaction"
}
Here "msgs" element contains nested array object into it, so to create field of every elements inside the array object i had followed the mutate and split approach both:-
Mutate filter:-
mutate {
add_field => {
"ts" => "%{[doc][msgs][0][ts]}"
"tid1" => "%{[doc][msgs][0][tid]}"
"eid1" => "%{[doc][msgs][0][eid]}"
"lvl1" => "%{[doc][msgs][0][lvl]}"
"cat1" => "%{[doc][msgs][0][cat]}"
"msg1" => "%{[doc][msgs][0][msg]}"
"data" => "%{[doc][msgs][1][data]}"
"actual-message" =>"%{[doc][msgs][3][data][Trace]}"
"error" =>"%{[doc][msgs][5][ex][exs][0][ec]}"
"error-type" =>"%{[doc][msgs][5][ex][exs][0][typ]}"
}
}
This approach is static as my logs is occurring dynamically i.e position of error is not fixed as it may come in 6th array element or may be at 7th or 8th,
so i ignored this approach and followed the split filter approach:-
Split Filter in logstash config:-
input{
file{
path=>"C:\Logs\GatewayService\GatewayService-Processor.Transactions-20170830.slog"
}
}
split {
field=>"msgs"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "g_index"
}
}
now it is showing error as:
[2017-09-19T17:40:07,557][WARN ][logstash.filters.split ] Only String and Array types are splittable. field:msg is of type = NilClass
so how should i able to retrieve every element in this JSON to be a field so as to visualize in kibana by applying metrics in dashboard.