Custom Table Container with <br> is not working in datatable in R shiny - html

##This is a container that I am using in the data table, but br tag in the column name is not working.
I used escape = FALSE in the data table but still facing the same issue.
and \n is also not working. I want something like this column name = first name (in the next line) last name.
test <- function(group,n){
htmltools::withTags(th(colspan = n, group, class = "dt-center"))
}
myContainer <- htmltools::withTags(table(
class = '',style="width:100%",
thead(
tr(
th(rowspan = 2, ' '),
th(colspan = 1, 'group 1', class = "dt-center"),
th(colspan = 2, 'group 2', class = "dt-center"),
th(colspan = 2, 'group 3', class = "dt-center")
),
tr(
th("new \\\\n ID"),
lapply(c("SUBJID","SITE<br>ID","AG<br>E","SUBJID","RACE"), th)
)
)
))
Server <- function(input, output, session) {
adae<-read_sas("C:/Arinjay_Intern/Work/ADaM/adae.sas7bdat")
output$intTable<-renderDT({adae_df %>%
datatable(class= 'compact', extensions = 'Buttons', rownames = F, container = myContainer,escape = FALSE,
callback = JS(c("$('table.dataTable thead th').css('border-top', 'none');",
"$('table.dataTable.no-footer').css('border-top', 'none');"
)),
options = list(dom = 'tB', pageLength = 5,
ordering = FALSE, class= "compact",
columnDefs = list(list(className = "dt-center", targets = "_all")),
buttons = 'pdf'
),
caption = htmltools::tags$caption(
style = 'caption-side: bottom; text-align: left;',
htmltools::em(HTML('N = number of subjects in the specified population. <br>n=number of subjects in each category. % = 100*n/N.')))
) %>%
formatStyle(c("USUBJID","SUBJID","SITEID","AGE", "SEX","RACE"), backgroundColor = 'white')
})
}
UI <-navbarPage(
"DT Interactive Tables",
tabPanel(
"ADaM DataSets",
fluidPage(
checkboxGroupInput('group','Please select a group',c('FD_Cohort','MRD_Cohort')),
textInput('n',"any value",value=2),
DTOutput("intTable")
)
)
)
shinyApp(UI,Server)
Expected output:

Neither \n or <br>, work in xtable. So, you could define the rows explicitly as shown below:
row1 <- c(" USUB","SUBJ","SITE","AG","SEX", "RACE")
row2 <- c("JID","ID","ID","E"," ", "")
myContainer <- htmltools::withTags(table(
class = 'dt-center', style="width:100%",
thead(
tr(
th(colspan = 2, 'group 1', class = "dt-center"),
th(colspan = 2, 'group 2', class = "dt-center"),
th(colspan = 2, 'group 3', class = "dt-center")
),
tr( lapply( row1, th)
),
tr( lapply( row2, th)
)
)
))
or you can write something in css or js to handle it. The above code gives the following output on a dummy dataset:

Related

JSON to Excel PowerQuery import - how to get a row per nested field

I'm looking to use the Excel Power Query to import some json that looks like the following (but much bigger, more fields etc.):
example-records.json
{
"records": {
"record_id_1": {
"file_no": "5792C",
"loads": {
"load_id_1": {
"docket_no": "3116115"
},
"load_id_2": {
"docket_no": "3116118"
},
"load_id_3": {
"docket_no": "3208776"
}
}
},
"record_id_2": {
"file_no": "5645C",
"loads": {
"load_id_4": {
"docket_no": "2000527155"
},
"load_id_5": {
"docket_no": "2000527156"
},
"load_id_6": {
"docket_no": "2000527146"
}
}
}
}
}
I want to get a table like the following at the load_id / docket level. A row per load_id
What I've tried
Clicking buttons in power query UI I get the following.
The problem is I can't include a file_no column and this doesn't work when there are lots of load ids.
let
Source = Json.Document(File.Contents("H:\Software\Site Apps\example-records.json")),
records = Source[records],
#"Converted to Table" = Record.ToTable(records),
#"Expanded Value" = Table.ExpandRecordColumn(#"Converted to Table", "Value", {"file_no", "loads"}, {"Value.file_no", "Value.loads"}),
#"Removed Columns" = Table.RemoveColumns(#"Expanded Value",{"Value.file_no"}),
#"Expanded Value.loads" = Table.ExpandRecordColumn(#"Removed Columns", "Value.loads", {"load_id_1", "load_id_2", "load_id_3", "load_id_4", "load_id_5", "load_id_6"}, {"Value.loads.load_id_1", "Value.loads.load_id_2", "Value.loads.load_id_3", "Value.loads.load_id_4", "Value.loads.load_id_5", "Value.loads.load_id_6"}),
#"Unpivoted Columns" = Table.UnpivotOtherColumns(#"Expanded Value.loads", {"Name"}, "Attribute", "Value"),
#"Expanded Value1" = Table.ExpandRecordColumn(#"Unpivoted Columns", "Value", {"docket_no"}, {"Value.docket_no"})
in
#"Expanded Value1"
You can use
let Source = JSON(Json.Document(File.Contents("c:\temp\example.json"))),
#"Removed Other Columns" = Table.SelectColumns(Source,{"Name.1", "Name.3", "Value"}),
#"Added Custom" = Table.AddColumn(#"Removed Other Columns", "Custom", each if [Name.3]=null then [Value] else null),
#"Filled Down" = Table.FillDown(#"Added Custom",{"Custom"}),
#"Filtered Rows" = Table.SelectRows(#"Filled Down", each ([Name.3] <> null))
in #"Filtered Rows"
based on this function I named JSON which comes from Imke https://www.thebiccountant.com/2018/06/17/automatically-expand-all-fields-from-a-json-document-in-power-bi-and-power-query/ which is reproduced below
let
func = (JSON) =>
let
Source = JSON,
ParseJSON = try Json.Document(Source) otherwise Source,
TransformForTable =
if Value.Is(ParseJSON, type record) then
Record.ToTable(ParseJSON)
else
#table(
{"Name", "Value"},
List.Zip({List.Repeat({0}, List.Count(ParseJSON)), ParseJSON})
),
AddSort = Table.Buffer(Table.AddColumn(TransformForTable, "Sort", each 0)),
LG = List.Skip(
List.Generate(
() => [Next = AddSort, Counter = 1, AddIndex = #table({"Sort"}, {{""}})],
each [AddIndex]{0}[Sort] <> "End",
each [
AddIndex = Table.AddIndexColumn([Next], "Index", 0, 1),
MergeSort = Table.CombineColumns(
Table.TransformColumnTypes(
AddIndex,
{{"Sort", type text}, {"Index", type text}},
"en-GB"
),
{"Sort", "Index"},
Combiner.CombineTextByDelimiter(".", QuoteStyle.None),
"Sort"
),
PJson = Table.TransformColumns(
MergeSort,
{{"Value", each try Json.Document(_) otherwise _}}
),
AddType = Table.AddColumn(
PJson,
"Type",
each
if Value.Is([Value], type record) then
"Record"
else if Value.Is([Value], type list) then
"List"
else if Value.Is([Value], type table) then
"Table"
else
"other"
),
AddStatus = Table.AddColumn(
AddType,
"Status",
each if [Type] = "other" then "Finished" else "Unfinished"
),
Finished = Table.SelectRows(AddStatus, each ([Status] = "Finished")),
Unfinished = Table.SelectRows(AddStatus, each ([Status] = "Unfinished")),
AddNext = Table.AddColumn(
Unfinished,
"Next",
each if [Type] = "Record" then {[Value]} else [Value]
),
RemoveCols = Table.RemoveColumns(AddNext, {"Value", "Type", "Status"}),
ExpandNext = Table.ExpandListColumn(RemoveCols, "Next"),
AddIndex2 = Table.AddIndexColumn(ExpandNext, "Index", 0, 1),
MergeSort2 = Table.CombineColumns(
Table.TransformColumnTypes(
AddIndex2,
{{"Sort", type text}, {"Index", type text}},
"en-GB"
),
{"Sort", "Index"},
Combiner.CombineTextByDelimiter(".", QuoteStyle.None),
"Sort"
),
TransformRecord = Table.TransformColumns(
MergeSort2,
{
{
"Next",
each try
Record.ToTable(_)
otherwise
try
if Value.Is(Text.From(_), type text) then
#table({"Value"}, {{_}})
else
_
otherwise
_
}
}
),
FilterOutNulls = Table.SelectRows(TransformRecord, each [Next] <> null),
Next =
if Table.IsEmpty(FilterOutNulls) then
#table({"Sort"}, {{"End"}})
else if Value.Is(FilterOutNulls[Next]{0}, type table) = true then
Table.ExpandTableColumn(
FilterOutNulls,
"Next",
{"Name", "Value"},
{"Name." & Text.From([Counter]), "Value"}
)
else
Table.RenameColumns(FilterOutNulls, {{"Next", "Value"}}),
Counter = [Counter] + 1
],
each Table.AddColumn([Finished], "Level", (x) => _[Counter] - 2)
)
),
Check = LG{2},
Combine = Table.Combine(LG),
Clean = Table.RemoveColumns(Combine, {"Status", "Type"}),
Trim = Table.TransformColumns(Clean, {{"Sort", each Text.Trim(_, "."), type text}}),
// Dynamic Padding for the sort-column so that it sorts by number in text strings
SelectSort = Table.SelectColumns(Trim, {"Sort"}),
SplitSort = Table.AddColumn(
SelectSort,
"Custom",
each List.Transform(try Text.Split([Sort], ".") otherwise {}, Number.From)
),
ToTable = Table.AddColumn(
SplitSort,
"Splitted",
each Table.AddIndexColumn(Table.FromColumns({[Custom]}), "Pos", 1, 1)
),
ExpandTable = Table.ExpandTableColumn(ToTable, "Splitted", {"Column1", "Pos"}),
GroupPos = Table.Group(
ExpandTable,
{"Pos"},
{{"All", each _, type table}, {"Max", each List.Max([Column1]), type text}}
),
Digits = Table.AddColumn(GroupPos, "Digits", each Text.Length(Text.From([Max]))),
FilteredDigits = List.Buffer(Table.SelectRows(Digits, each ([Digits] <> null))[Digits]),
SortNew = Table.AddColumn(
Trim,
"SortBy",
each Text.Combine(
List.Transform(
List.Zip({Text.Split([Sort], "."), List.Positions(Text.Split([Sort], "."))}),
each Text.PadStart(_{0}, FilteredDigits{_{1}}, "0")
),
"."
)
),
FilterNotNull = Table.SelectRows(SortNew, each ([Value] <> null)),
Reorder = Table.ReorderColumns(
FilterNotNull,
{"Value", "Level", "Sort", "SortBy"}
& List.Difference(
Table.ColumnNames(FilterNotNull),
{"Value", "Level", "Sort", "SortBy"}
)
),
Dots = Table.AddColumn(
#"Reorder",
"Dots",
each List.Select(Table.ColumnNames(#"Reorder"), (l) => Text.StartsWith(l, "Name"))
),
// This sort is just to view in the query editor. When loaded to the data model it will not be kept. Use "Sort by column" in the data model instead.
Sort = Table.Sort(Dots, {{"SortBy", Order.Ascending}})
in
Sort,
documentation = [
Documentation.Name = " Table.JsonExpandAll ",
Documentation.Description
= " Dynamically expands the <Json> Record and returns values in one column and additional columns to navigate. ",
Documentation.LongDescription
= " Dynamically expands the <Json> Record and returns values in one column and additional columns to navigate. Input can be JSON in binary format or the already parsed JSON. ",
Documentation.Category = " Table ",
Documentation.Version = " 1.2: Added column [Dots] (22/02/2019)",
Documentation.Author = " Imke Feldmann: www.TheBIccountant.com . ",
Documentation.Examples = {[Description = " ", Code = " ", Result = " "]}
]
in
Value.ReplaceType(func, Value.ReplaceMetadata(Value.Type(func), documentation))
Managed to use an added custom column, the action that enables the expansion to one load id per row.
#"Added Custom" = Table.AddColumn(#"Expanded Value", "Custom", each Record.ToTable([Value.loads]))
let
Source = Json.Document(File.Contents("H:\Software\Site Apps\example-records.json")),
records = Source[records],
#"Converted to Table" = Record.ToTable(records),
#"Expanded Value" = Table.ExpandRecordColumn(#"Converted to Table", "Value", {"file_no", "loads"}, {"Value.file_no", "Value.loads"}),
#"Added Custom" = Table.AddColumn(#"Expanded Value", "Custom", each Record.ToTable([Value.loads])),
#"Removed Columns" = Table.RemoveColumns(#"Added Custom",{"Value.loads"}),
#"Expanded Custom" = Table.ExpandTableColumn(#"Removed Columns", "Custom", {"Name", "Value"}, {"Custom.Name", "Custom.Value"}),
#"Expanded Custom.Value" = Table.ExpandRecordColumn(#"Expanded Custom", "Custom.Value", {"docket_no"}, {"Custom.Value.docket_no"}),
#"Renamed Columns" = Table.RenameColumns(#"Expanded Custom.Value",{{"Name", "record_id"}, {"Value.file_no", "file_no"}, {"Custom.Name", "load_id"}, {"Custom.Value.docket_no", "docket_no"}})
in
#"Renamed Columns"

How we can column header in renderTable not in renderDataTable in R shiny? I want to group of few columns in my renderTable?

I am trying in this way:
myContainer <- htmltools::withTags(table(
class = '',style="width:100%",
thead(
tr(
th(colspan = 2, 'Group 1', class = "dt-center"),
th(colspan = 2, 'Group 2', class = "dt-center")
),
tr(
lapply(names(data.frame(df)), th)
)
)
))
#UI:
library(shiny)
shinyUI <- fluidPage(
fluidRow(
htmlOutput("TextTable")
)
My rendertable in server:
output$TextTable <- renderTable({
varsub
}, width = "100%", include.colnames = TRUE,
sanitize.text.function = identity, spacing = 's',
container = myContainer,
columnDefs = list(list(className = "dt-center", targets = "_all"))
)
But this is not working, any help would be appreciated. Or please suggest any other way to do it.

Shiny Plot Height behaving different on different desktop

I am trying to make a custom UI in which i have to plot 4 plots inside each tab,
but i am getting scroll on plot rendering and when i add height as 240 scrollbar disappear and it works for the same size desktop but it behaves different on different size of desktop and i get scrollbar again.
motive is to fit the plots in screen without scrollbar, also i would like to get a feedback that am i creating UI in a right way
Thanks
UI
navbarPage("NarBar",
tabPanel("Tab1",
column(12,
column(4,
column(12,
checkboxInput("ID1", "Checkbox1", FALSE)
),
column(12,
checkboxInput("ID2", "Checkbox2", FALSE)
)
),
column(8,
column(3,
selectInput("ID1", "Select1:",
c("A" = "a",
"B" = "b",
"C" = "c"))
),
column(3,
selectInput("ID2", "Select2:",
c("A" = "a",
"B" = "b",
"C" = "c"))
),
column(3,
selectInput("ID3", "Select3:",
c("A" = "a",
"B" = "b",
"C" = "c"))
),
column(3,
selectInput("ID4", "Select4:",
c("A" = "a",
"B" = "b",
"C" = "c")
)
)
)
),
column(12,
column(4,
column(12,
selectInput("ID1", "Select1:",
c("A" = "a",
"B" = "b",
"C" = "c")
)
),
column(12,
selectInput("ID2", "Select2:",
c("A" = "a",
"B" = "b",
"C" = "c")
)
),
column(12,
selectInput("ID3", "Select3:",
c("A" = "a",
"B" = "b",
"C" = "c")
)
),
column(12,
selectInput("ID4", "Select4:",
c("A" = "a",
"B" = "b",
"C" = "c")
)
),
column(12,
selectInput("ID5", "Select5:",
c("A" = "a",
"B" = "b",
"C" = "c")
)
),
column(12,
selectInput("ID6", "Select6:",
c("A" = "a",
"B" = "b",
"C" = "c")
)
)
),
column(width = 8,
tabsetPanel(
tabPanel(title = 'Tab1',
column(width = 6,
plotOutput('plot1',height = 240)
),
column(width = 6,
plotOutput('plot2',height = 240)
),
column(width = 6,
plotOutput('plot3',height = 240)
),
column(width = 6,
plotOutput('plot4',height = 240)
)
),
tabPanel(title = 'Tab2',
column(width = 6,
plotOutput('plot5',height = 240)
),
column(width = 6,
plotOutput('plot6',height = 240)
),
column(width = 6,
plotOutput('plot7',height = 240)
),
column(width = 6,
plotOutput('plot8',height = 240)
)
),
tabPanel(title = 'Tab3',
column(width = 6,
plotOutput('plot9',height = 240)
),
column(width = 6,
plotOutput('plot10',height = 240)
),
column(width = 6,
plotOutput('plot11',height = 240)
),
column(width = 6,
plotOutput('plot12',height = 240)
)
)
)
)
)
),
tabPanel("Tab 2"
),
tabPanel("Tab 3"
),
tabPanel("Tab 4"
)
)
server just using same plot for every plotOutput
function(input, output, session) {
output$plot1 <- renderPlot({ #just use the same plot on different plots like plot2,plot3,etc
cars2 <- cars + rnorm(nrow(cars))
plot(cars2)
})
}
First, your code has a lot of repetitive parts. We can use lapply to creare these parts more efficiently as follow.
Based on this answer (https://stackoverflow.com/a/26785047/7669809), I added tags$head(tags$style(".shiny-plot-output{height:40vh !important;}")) to your navbarPage. It seems to work. You can change 40vh to other number you would like to use.
library(shiny)
library(htmlwidgets)
ui <- navbarPage("NarBar",
tags$head(tags$style(".shiny-plot-output{height:40vh !important;}")),
tabPanel("Tab1",
column(12,
column(4,
lapply(1:2, function(x){
column(12,
checkboxInput(paste0("ID", x),
paste0("Checkbox", x),
FALSE)
)
})
),
column(8,
lapply(1:4, function(x){
column(3,
selectInput(paste0("ID", x),
paste0("Select", x, ":"),
c("A" = "a",
"B" = "b",
"C" = "c")))
})
)
),
column(12,
column(4,
lapply(1:6, function(x){
column(12,
selectInput(paste0("ID", x),
paste0("Select", x, ":"),
c("A" = "a",
"B" = "b",
"C" = "c")
)
)
})
),
column(width = 8,
tabsetPanel(
tabPanel(title = 'Tab1',
lapply(1:4, function(x){
column(width = 6,
plotOutput(paste0('plot', x))
)
})
),
tabPanel(title = 'Tab2',
lapply(5:8, function(x){
column(width = 6,
plotOutput(paste0('plot', x))
)
})
),
tabPanel(title = 'Tab3',
lapply(9:12, function(x){
column(width = 6,
plotOutput(paste0('plot', x))
)
})
)
)
)
)
),
tabPanel("Tab 2"
),
tabPanel("Tab 3"
),
tabPanel("Tab 4"
)
)
server <- function(input, output, session) {
lapply(1:12, function(x) {
output[[paste0('plot', x)]] <- renderPlot({
cars2 <- cars + rnorm(nrow(cars))
plot(cars2)
})
})
}
shinyApp(ui, server)

How to add padding between regular button and actionButton in shinyApp?

I have 2 individual (differently styled) buttons in my shiny App that unfortunately have no vertical padding between them. One of them is a simple HTML button while the other is a shiny actionButton.
I have tried using tags$style but don't think I am using it correctly. My code for the relevant parts are
HTML('<button data-toggle="collapse" data-target="#addtools">Additional tools</button>'),
tags$div(id = 'addtools', class="collapse",
checkboxInput("cluster", "Cluster customers", value = TRUE),
checkboxInput("circle", "Add circles", value = FALSE),
conditionalPanel("input.circle",
selectInput("sizes", "Radius:",
c("5 miles" = "5",
"10 miles" = "10",
"20 miles" = "20"))
)
),
actionButton("runfilters", label = "Apply filters")
The entire UI part of this file is:
tabPanel("Map",
div(class="outer",
tags$head(
includeCSS("styles.css"),
includeScript("gomap.js")
),
leafletOutput(outputId = "map", width = "100%", height = "100%"),
absolutePanel(id = "controls", class = "panel panel-default", fixed = TRUE,
draggable = TRUE, top = 60, left = "auto", right = 20, bottom = "auto",
width = 330, height = "auto",
h2("Select your filters:"),
selectInput("level",
"Detail level:",
c("Customer level", "County level"),
selected = "County level"),
selectInput("mapstates","States:", c("All states"="",
structure(state.abb, names=state.name),
"Washington, DC"="DC"),
multiple = TRUE
),
conditionalPanel("input.mapstates",
selectInput("mapcounties", "Counties", c("All counties"=""),
multiple = TRUE
)
),
radioButtons("retailcorrection","Customer ownership:",
c("Private",
"Public"),
selected = "Private",
inline = TRUE),
conditionalPanel("input.level == 'Customer level'",
selectInput("var",
label = "Customer type:",
choices = c("ALL",
"ESSENTIAL",
"NON-ESSENTIAL"),
selected = "ALL"
),
radioButtons("custoporoh","Widget Category:",
c("Widget A",
"Widget B"),
selected = "Widget A",
inline = TRUE),
conditionalPanel("input.custoporoh == 'Widget A'",
numericInput("oprange",
label = "Minimum average monthly Widget A ordering (in dosage units):",
min = 0, max = 1000000, value = 0
),
numericInput("opprange",
label = "Minimum average monthly Widget A ordering as a percentage of total ordering:",
min = -1, max = 1, value = -1
)
),
conditionalPanel("input.custoporoh == 'Widget B'",
numericInput("ohrange","Minimum average monthly Widget B ordering (in dosage units):",
min = 0, max = 1000000, value = 0
),
numericInput("ohprange",
label = "Minimum average monthly Widget B ordering as a percentage of total ordering:",
min = -1, max = 1, value = -1
)
),
HTML('<button data-toggle="collapse" data-target="#addtools">Additional tools</button>'),
tags$div(id = 'addtools', class="collapse",
checkboxInput("cluster", "Cluster customers", value = TRUE),
checkboxInput("circle", "Add circles", value = FALSE),
conditionalPanel("input.circle",
selectInput("sizes", "Radius:",
c("5 miles" = "5",
"10 miles" = "10",
"20 miles" = "20"))
)
)
),
conditionalPanel("input.level == 'County level'",
selectInput("countyvar",
label = "Customer type:",
choices = c("ALL",
"ESSENTIAL"),
selected = "ALL"
),
selectInput("year", "Year:",
choices = c("2006","2007","2008","2009","2010","2011", "2012",
"2013","2014","2015","2016","2017","2018", "All years" = "2019"),
selected = "2019"
),
radioButtons("oporoh","Widget Category:",
c("Widget A",
"Widget B"),
selected = "Widget A",
inline = TRUE),
conditionalPanel("input.oporoh == 'Widget A'",
numericInput("opppcrange",
label = HTML("Minimum Widget A units per adult per month:"),
min = 0, max = 160, value = 0
),
numericInput("opcrange",
label = "Minimum average monthly Widget A ordering (in dosage units):",
min = 0, max = 10000000, value = 0
),
numericInput("oppcrange",
label = "Minimum average monthly Widget A ordering as a percentage of total ordering:",
min = -1, max = 1, value = -1
)
),
conditionalPanel("input.oporoh == 'Widget B'",
numericInput("ohppcrange",
label = "Minimum Widget B units per adult per month:",
min = 0, max = 160, value = 0
),
numericInput("ohcrange","Minimum average monthly Widget B ordering (in dosage units):",
min = 0, max = 10000000, value = 0
),
numericInput("ohpcrange",
label = "Minimum average monthly Widget B ordering as a percentage of total ordering:",
min = -1, max = 1, value = -1
)
)
),
actionButton("runfilters", label = "Apply filters")
)
)
)
I have tried to implement:
HTML('<button data-toggle="collapse" data-target="#addtools">Additional tools</button>'),
tags$div(id = 'addtools', class="collapse", tags$style = "padding=10px"
but that doesn't seem to be working. Any help would be appreciated.

how do you convert a json entries in a file into a data frame?

I am trying to read files that has json content and convert that to tabular data based on some fields.
The file includes content like this:
{"senderDateTimeStamp":"2016/04/08 10:03:18","senderHost":null,"senderCode":"web_app","senderUsecase":"appinternalstats_prod","destinationTopic":"web_app_appinternalstats_realtimedata_topic","correlatedRecord":false,"needCorrelationCacheCleanup":false,"needCorrelation":false,"correlationAttributes":null,"correlationRecordCount":0,"correlateTimeWindowInMills":0,"lastCorrelationRecord":false,"realtimeESStorage":true,"receiverDateTimeStamp":1460124283554,"payloadData":{"timestamp":"2016-04-08T10:03:18.244","status":"get","source":"MSG1","ITEM":"TEST1","basis":"","pricingdate":"","content":"","msgname":"","idlreqno":"","host":"web01","Webservermember":"Web"},"payloadDataText":"","key":"web_app:appinternalstats_prod","destinationTopicName":"web_app_appinternalstats_realtimedata_topic","esindex":"web_app","estype":"appinternalstats_prod","useCase":"appinternalstats_prod","Code":"web_app"}
I need to be able to convert timestamp, source, host, status fields withing payloadData section for each line into a data frame in R.
I've tried this:
library(rjson)
d<-fromJSON(file="file.txt")
dput(d)
structure(list(senderDateTimeStamp = "2016/04/08 10:03:18", senderHost = NULL,
senderAppcode = "web", senderUsecase = "appinternalstats_prod",
destinationTopic = "web_appinternalstats_realtimedata_topic",
correlatedRecord = FALSE, needCorrelationCacheCleanup = FALSE,
needCorrelation = FALSE, correlationAttributes = NULL, correlationRecordCount = 0,
correlateTimeWindowInMills = 0, lastCorrelationRecord = FALSE,
realtimeESStorage = TRUE, receiverDateTimeStamp = 1460124283554,
payloadData = structure(list(timestamp = "2016-04-08T10:03:18.244",
status = "get", source = "MSG1",
region = "", evetid = "", osareqid = "", basis = "",
pricingdate = "", content = "", msgname = "", recipient = "",
objid = "", idlreqno = "", host = "web01", webservermember = "webSingleton"),
.Names = c("timestamp",
"status", "source", "region", "evetid",
"osareqid", "basis", "pricingdate", "content", "msgname",
"recipient", "objid", "idlreqno", "host", "webservermember"
)), payloadDataText = "", key = "web:appinternalstats_prod",
destinationTopicName = "web_appinternalstats_realtimedata_topic",
hdfsPath = "web/appinternalstats_prod", esindex = "web",
estype = "appinternalstats_prod", useCase = "appinternalstats_prod",
appCode = "web"), .Names = c("senderDateTimeStamp", "senderHost",
"senderAppcode", "senderUsecase", "destinationTopic", "correlatedRecord",
"needCorrelationCacheCleanup", "needCorrelation", "correlationAttributes",
"correlationRecordCount", "correlateTimeWindowInMills", "lastCorrelationRecord",
"realtimeESStorage", "receiverDateTimeStamp", "payloadData",
"payloadDataText", "key", "destinationTopicName", "hdfsPath",
"esindex", "estype", "useCase", "appCode"))
Any ideas how I could convert payloadData section of the json entry into a data frame?
This might be something you want:
library(rjson)
d<-fromJSON(file="file.txt")
myDf <- do.call("rbind", lapply(d, function(x) {
data.frame(TimeStamp = x$payloadData$timestamp,
Source = x$payloadData$source,
Host = $payloadData$host,
Status = x$payloadData$status)}))
Consider the package tidyjson:
library(tidyjson)
library(magrittr)
json <- '{"senderDateTimeStamp":"2016/04/08 10:03:18","senderHost":null,"senderCode":"web_app","senderUsecase":"appinternalstats_prod","destinationTopic":"web_app_appinternalstats_realtimedata_topic","correlatedRecord":false,"needCorrelationCacheCleanup":false,"needCorrelation":false,"correlationAttributes":null,"correlationRecordCount":0,"correlateTimeWindowInMills":0,"lastCorrelationRecord":false,"realtimeESStorage":true,"receiverDateTimeStamp":1460124283554,"payloadData":{"timestamp":"2016-04-08T10:03:18.244","status":"get","source":"MSG1","ITEM":"TEST1","basis":"","pricingdate":"","content":"","msgname":"","idlreqno":"","host":"web01","Webservermember":"Web"},"payloadDataText":"","key":"web_app:appinternalstats_prod","destinationTopicName":"web_app_appinternalstats_realtimedata_topic","esindex":"web_app","estype":"appinternalstats_prod","useCase":"appinternalstats_prod","Code":"web_app"}'
json %>%
gather_keys()
# head() of above
# document.id key
# 1 1 senderDateTimeStamp
# 2 1 senderHost
# 3 1 senderCode
# 4 1 senderUsecase
# 5 1 destinationTopic
# 6 1 correlatedRecord
json %>%
enter_object("payloadData") %>%
gather_keys() %>%
append_values_string()
# head() of above
# document.id key string
# 1 1 timestamp 2016-04-08T10:03:18.244
# 2 1 status get
# 3 1 source MSG1
# 4 1 ITEM TEST1
# 5 1 basis
# 6 1 pricingdate