I am using an lmer model (from lmerTest) to understand whether size is significantly correlated with gene expression, and if so, which specific genes are correlated with size (also accounting for 'female' and 'cage' as random effects):
lmer(Expression ~ size*genes + (1|female) + (1|cage), data = df)
In the summary output, one of my genes is being used up as an intercept (since it is highest in the alphabet, 'ctsk'). After reading around, it was recommended that I choose the highest (or lowest) expressed gene as my intercept to compare everything else against. In this case, the gene 'star' was the highest expressed. After re-levelling my data, and re-running the model with 'star' as the intercept, ALL the other slopes are now significant in summary() output, although anova() output is identical.
My questions are:
Is it possible to not have one of my genes used as an intercept? If it is not possible, then how do I know which gene I should choose as an intercept?
Can I test whether the slopes are different from zero? Perhaps this is where I would specify no intercept in my model (i.e. '0+size*genes')?
Is it possible to have the intercept as the mean of all slopes?
I will then use lsmeans to determine whether the slopes are significantly different from each other.
Here is some reproducible code:
df <- structure(list(size = c(13.458, 13.916, 13.356, 13.84, 14.15,
16.4, 15.528, 13.916, 13.458, 13.285, 15.415, 14.181, 13.367,
13.356, 13.947, 14.615, 15.804, 15.528, 16.811, 14.677, 13.2,
17.57, 13.947, 14.15, 16.833, 13.2, 17.254, 16.4, 14.181, 13.367,
14.294, 13.84, 16.833, 17.083, 15.847, 13.399, 14.15, 15.47,
13.356, 14.615, 15.415, 15.596, 15.847, 16.833, 13.285, 15.47,
15.596, 14.181, 13.356, 14.294, 15.415, 15.363, 15.4, 12.851,
17.254, 13.285, 17.57, 14.7, 17.57, 13.947, 16.811, 15.4, 13.399,
14.22, 13.285, 14.344, 17.083, 15.363, 14.677, 15.945), female = structure(c(7L,
12L, 7L, 11L, 12L, 9L, 6L, 12L, 7L, 7L, 6L, 12L, 8L, 7L, 7L,
11L, 9L, 6L, 10L, 11L, 8L, 10L, 7L, 12L, 10L, 8L, 10L, 9L, 12L,
8L, 12L, 11L, 10L, 10L, 9L, 8L, 12L, 6L, 7L, 11L, 6L, 9L, 9L,
10L, 7L, 6L, 9L, 12L, 7L, 12L, 6L, 6L, 6L, 8L, 10L, 7L, 10L,
11L, 10L, 7L, 10L, 6L, 8L, 11L, 7L, 6L, 10L, 6L, 11L, 9L), .Label = c("2",
"3", "6", "10", "11", "16", "18", "24", "25", "28", "30", "31",
"116", "119", "128", "135", "150", "180", "182", "184", "191",
"194", "308", "311", "313", "315", "320", "321", "322", "324",
"325", "329", "339", "342"), class = "factor"), Expression = c(1.10620339407889,
1.06152707257767, 2.03000185674761, 1.92971750056866, 1.30833983462599,
1.02760836165184, 0.960969703469363, 1.54706275342441, 0.314774666283256,
2.63330873720495, 0.895123048920455, 0.917716470037954, 1.3178821021651,
1.57879156856332, 0.633429011784367, 1.12641940390116, 1.0117475796626,
0.687813581350802, 0.923485880847423, 2.98926377892241, 0.547685277701021,
0.967691178046748, 2.04562285257417, 1.09072264997544, 1.57682235413366,
0.967061529758701, 0.941995966023426, 0.299517719292817, 1.8654758451133,
0.651369936708288, 1, 1.04407979584122, 0.799275069735012, 1.007255409328,
0.428129727802404, 0.93927930755046, 0.987394257033815, 0.965050972503591,
2.06719308587322, 1.63846508102874, 0.997380526962644, 0.60270197593643,
2.78682867333149, 0.552922632281237, 3.06702198884562, 0.890708510580522,
1.15168812515828, 0.929205084743164, 2.27254101826041, 1, 0.958147442333527,
1.05924173014089, 0.984356852670054, 0.623630720815415, 0.796864961771971,
2.4679841984147, 1.07248904053777, 1.79630829771291, 0.929642913565982,
0.296954006040077, 2.25741254504115, 1.17188536743493, 0.849778293699644,
2.32679163466857, 0.598119006609413, 0.975660099975423, 1.01494421228949,
1.14007557533352, 2.03638316428189, 0.777347547080068), cage = structure(c(64L,
49L, 56L, 66L, 68L, 48L, 53L, 49L, 64L, 56L, 55L, 68L, 80L, 56L,
64L, 75L, 69L, 53L, 59L, 66L, 63L, 59L, 64L, 68L, 59L, 63L, 50L,
48L, 68L, 80L, 49L, 66L, 59L, 50L, 48L, 63L, 68L, 62L, 56L, 75L,
55L, 81L, 48L, 59L, 56L, 62L, 81L, 68L, 56L, 49L, 55L, 62L, 55L,
63L, 50L, 56L, 59L, 75L, 59L, 64L, 59L, 55L, 63L, 66L, 56L, 53L,
50L, 62L, 66L, 81L), .Label = c("023", "024", "041", "042", "043",
"044", "044 bis", "045", "046", "047", "049", "051", "053", "058",
"060", "061", "068", "070", "071", "111", "112", "113", "123",
"126", "128", "14", "15", "23 bis", "24", "39", "41", "42", "44",
"46 bis", "47", "49", "51", "53", "58", "60", "61", "67", "68",
"70", "75", "76", "9", "D520", "D521", "D522", "D526", "D526bis",
"D533", "D535", "D539", "D544", "D545", "D545bis", "D546", "D561",
"D561bis", "D564", "D570", "D581", "D584", "D586", "L611", "L616",
"L633", "L634", "L635", "L635bis", "L637", "L659", "L673", "L676",
"L686", "L717", "L718", "L720", "L725", "L727", "L727bis"), class = "factor"),
genes = c("igf1", "gr", "ctsk", "ets2", "ctsk", "mtor", "igf1",
"sgk1", "sgk1", "ghr1", "ghr1", "gr", "ctsk", "ets2", "timp2",
"timp2", "ets2", "rictor", "sparc", "mmp9", "gr", "sparc",
"mmp2", "ghr1", "mmp9", "sparc", "mmp2", "timp2", "star",
"sgk1", "mmp2", "gr", "mmp2", "rictor", "timp2", "mmp2",
"mmp2", "mmp2", "mmp2", "rictor", "mtor", "ghr1", "star",
"igf1", "mmp9", "igf1", "igf2", "rictor", "rictor", "mmp9",
"ets2", "ctsk", "mtor", "ghr1", "mtor", "ets2", "ets2", "igf2",
"igf1", "sgk1", "sgk1", "ghr1", "sgk1", "igf2", "star", "mtor",
"igf2", "ghr1", "mmp2", "rictor")), .Names = c("size", "female",
"Expression", "cage", "genes"), row.names = c(1684L, 2674L, 10350L,
11338L, 10379L, 4586L, 1679L, 3637L, 3610L, 5537L, 5530L, 2676L,
10355L, 11313L, 8422L, 8450L, 11322L, 6494L, 9406L, 13262L, 2653L,
9407L, 12274L, 5564L, 13256L, 9394L, 12294L, 8438L, 750L, 3614L,
12303L, 2671L, 12293L, 6513L, 8437L, 12284L, 12305L, 12267L,
12276L, 6524L, 4567L, 5545L, 733L, 1700L, 13241L, 1674L, 7471L,
6528L, 6498L, 13266L, 11308L, 10347L, 4566L, 5541L, 4590L, 11315L,
11333L, 7482L, 1703L, 3607L, 3628L, 5529L, 3617L, 7483L, 722L,
4565L, 7476L, 5532L, 12299L, 6510L), class = "data.frame")
genes <- as.factor(df$genes)
library(lmerTest)
fit1 <- lmer(Expression ~ size * genes +(1|female) + (1|cage), data = df)
anova(fit1)
summary(fit1) # uses the gene 'ctsk' for intercept, so re-level to see what happens if I re-order based on highest value (sgk1):
df$genes <- relevel(genes, "star")
# re-fit the model with 'star' as the intercept:
fit1 <- lmer(Expression ~ size * genes +(1|female) + (1|cage), data = df)
anova(fit1) # no difference here
summary(fit1) # lots of difference
My sample data is pretty long since the model wouldn't run otherwise-hopefully this is ok!
While it is possible to interpret the coefficients in your fitted model, that isn't the most fruitful or productive approach. Instead, just fit the model using whatever contrast methods are used by default, and follow-up with suitable post-hoc analyses.
For that, I suggest using the emmeans (estimated marginal means) package, which is a continuation of lsmeans where all future developments will take place. The package has several vignettes, and the one most relebant to your situation is vignette("interactions"), which you may view here -- particularly the section on interactions with covariates.
Briefly, comparing intercepts can be very misleading, since those are predictions at size = 0 which is an extrapolation; and moreover, as you suggest in a question, the real point here is probably to compare slopes more than intercepts. For that purpose, there is an emtrends() function (or, if you like, its alias lstrends()).
I also strongly recommend displaying a graph of the model predictions so you can visualize what's going on. This may be done via
library(emmeans)
emmip(fit1, gene ~ size, at = list(size = range(df$size)))
Related
I have this SQL that started to run very slow on production server ( >20s ). When I run it locally on my laptop on the same DB it runs under 1s.
I know that SQL is not optimal but I tried to run EXPLAIN but I don't quite understand the result.
SELECT ST.id,
ST.lot_id,
ST.warehouse_location_id,
WL.name,
P.id AS product_id,
P.sku,
P.code,
P.NAME AS product_name,
P.has_serial,
Sum(ST.quantity) AS qty,
Group_concat(ST.id) AS stock_transaction_ids
FROM stock_transactions ST LEFT JOIN products P
ON (P.id = ST.product_id)
LEFT JOIN warehouse_locations WL
ON (WL.id = ST.warehouse_location_id)
LEFT JOIN
(
SELECT stock_transaction_id,
Group_concat(serial_id) AS serials
FROM stock_transactions_serials
GROUP BY stock_transaction_id ) STS
ON STS.stock_transaction_id = ST.id
WHERE (
ST.document_type = "Goods transfer" AND
ST.document_id IN (806, 807, 808, 809, 810, 811, 812, 824, 827, 831, 835, 838, 839, 841, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 944, 945, 946, 954, 955, 956, 957, 961, 965, 966, 967, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446, 2447, 2448, 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 3707, 3708, 10194, 10244, 10246, 10247, 10248, 10249, 10426, 10428, 27602, 27603, 28587, 28588, 28589, 28590, 28591, 28592, 28593, 28594, 28595, 28596, 28597, 28598, 28599, 28600, 28601, 28602, 28603, 28604, 28605, 28606, 28607, 28608, 28609, 28610, 28611, 28612, 28613, 28614, 28615, 28616, 28617, 28618, 28619, 28620, 28621, 28622, 28623, 28624, 28625, 28626, 28627, 28628, 28629, 28630, 28631, 28632, 28633, 28634, 28635, 28636, 28637, 28638, 28639, 28640, 28641, 28642, 28643) )
GROUP BY ST.product_id,
ST.lot_id,
ST.warehouse_location_id
HAVING qty > 0
Explain gets me this:
Can someone help understand what's going on? I will see if I can come back with SQL fiddle as well.
It looks like the problem is your derived table:
(
SELECT stock_transaction_id,
Group_concat(serial_id) AS serials
FROM stock_transactions_serials
GROUP BY stock_transaction_id ) STS
ON STS.stock_transaction_id = ST.id
I can't see you using serials anywhere else in the query, so I'm guessing this is not doing very much for you, but the cost is high - the left join isn't using an index when joining. I'd suggest rewriting that as a join on stock_transaction_serials, rather than a derived table.
you have
ON STS.stock_transaction_id = ST.id
WHERE (
ST.document_type = "Goods transfer" AND
ST.document_id IN (806, 807, 808, 809, 810, 811, 812, -- etc
If these values are in goods_transfer_items in a field called id then this would be exactly functionally the same:
ON STS.stock_transaction_id = ST.id
JOIN goods_transfer_items ON goods_transfer_items.id = ST.document_id
AND goods_transfer_items.work_order_number=12345'
AND ST.document_type = "Goods transfer"
Not only is this much shorter but it will also be much faster because SQL servers can optimize joins.
This is functionally IDENTICAL to the NOT IN
prior answer below
You should take the values listed here
ST.document_id IN (806, 807, 808, 809, 810, 811, 812, 824, 827, 831, 835, 838, 839, 841, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 944, 945, 946, 954, 955, 956, 957, 961, 965, 966, 967, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446, 2447, 2448, 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 3707, 3708, 10194, 10244, 10246, 10247, 10248, 10249, 10426, 10428, 27602, 27603, 28587, 28588, 28589, 28590, 28591, 28592, 28593, 28594, 28595, 28596, 28597, 28598, 28599, 28600, 28601, 28602, 28603, 28604, 28605, 28606, 28607, 28608, 28609, 28610, 28611, 28612, 28613, 28614, 28615, 28616, 28617, 28618, 28619, 28620, 28621, 28622, 28623, 28624, 28625, 28626, 28627, 28628, 28629, 28630, 28631, 28632, 28633, 28634, 28635, 28636, 28637, 28638, 28639, 28640, 28641, 28642, 28643) )
Put them in a table, create an index on it and then inner join to it instead of using IN
I am trying to parse data from the Minnesota DNR page and it says they are using JSON. I want to build a script to download the data tables from many different pages but am focusing on one first. I have tried rvest, JSONIO, and many other packages to no avail. The most frustrating error I am getting is:
Error in UseMethod("xml_find_first") :
no applicable method for 'xml_find_first' applied to an object of class "list"
Here is my code:
library(rvest)
kk<-read_html("http://www.dnr.state.mn.us/lakefind/showreport.html?downum=56003100")
node <- "table.table_colors:nth-child(1) > tbody:nth-child(1)"
html_table(node, fill=TRUE)
head(kk)
How do I get this table to download with the headers in tact???
Just get the actual data that goes into making the table. It's JSON and not too complex:
library(httr)
res <- GET("http://maps2.dnr.state.mn.us/cgi-bin/lakefinder/detail.cgi",
query=list(type="lake_survey", id="56003100"))
str(content(res))
This lets you get metadata by county name:
get_lake_metadata <- function(county_name) {
require(httr)
require(dplyr)
require(jsonlite)
xlate_df <- data_frame(
id = c(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L,
11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L,
24L, 25L, 26L, 27L, 28L, 29L, 30L, 31L, 32L, 33L, 34L, 35L, 36L,
37L, 38L, 39L, 40L, 41L, 42L, 44L, 45L, 46L, 43L, 47L, 48L, 49L,
50L, 51L, 52L, 53L, 54L, 55L, 56L, 57L, 58L, 59L, 60L, 61L, 62L,
63L, 64L, 65L, 66L, 67L, 68L, 70L, 71L, 72L, 69L, 73L, 74L, 75L,
76L, 77L, 78L, 79L, 80L, 81L, 82L, 83L, 84L, 85L, 86L, 87L),
county = c("Aitkin", "Anoka", "Becker", "Beltrami", "Benton",
"Big Stone", "Blue Earth", "Brown", "Carlton", "Carver",
"Cass", "Chippewa", "Chisago", "Clay", "Clearwater", "Cook",
"Cottonwood", "Crow Wing", "Dakota", "Dodge", "Douglas",
"Faribault", "Fillmore", "Freeborn", "Goodhue", "Grant",
"Hennepin", "Houston", "Hubbard", "Isanti", "Itasca", "Jackson",
"Kanabec", "Kandiyohi", "Kittson", "Koochiching", "Lac Qui Parle",
"Lake", "Lake of the Woods", "Le Sueur", "Lincoln", "Lyon",
"Mahnomen", "Marshall", "Martin", "McLeod", "Meeker", "Mille Lacs",
"Morrison", "Mower", "Murray", "Nicollet", "Nobles", "Norman",
"Olmsted", "Otter Tail", "Pennington", "Pine", "Pipestone",
"Polk", "Pope", "Ramsey", "Red Lake", "Redwood", "Renville",
"Rice", "Rock", "Roseau", "Scott", "Sherburne", "Sibley",
"St. Louis", "Stearns", "Steele", "Stevens", "Swift", "Todd",
"Traverse", "Wabasha", "Wadena", "Waseca", "Washington",
"Watonwan", "Wilkin", "Winona", "Wright", "Yellow Medicine"))
target <- filter(xlate_df, tolower(county) == tolower(county_name))
if (nrow(target) == 1) {
res <- GET("http://maps2.dnr.state.mn.us/cgi-bin/lakefinder_json.cgi",
query=list(county=target$id))
jsonlite::fromJSON(content(res, as="parsed"))
} else {
message("County not found")
}
}
get_lake_metadata("Anoka")
get_lake_metadata("Steele")
I am not sure what's going wrong, cause it looks pretty straight forward to me. I have few things in my mind about appending the JSON response, but for that when I am trying to apply forEach loop to my Json , no data is shown in the alert
Angular code
mainApp.controller('MultiCarouselController', function($scope, $http) {
$scope.products = [];
$http.get("/get_broad_category_products/?BroadCategory=BroadCategory3")
.success(function (response) {
$scope.products = response;
angular.forEach(products,function(value,key){
alert(key+'----'+value); // can see this alerts
});
}).error(function(){
console.log('Error happened ... ');
});
});
My JSON response is some junk data of Products
[
{
"sku": "d1cd71a8-9dc5-4724-b269-473ede28a1d7",
"selectedQtyOptions": [],
"selectedSize": "",
"description": "foolish interpolates trumpet monographs ferried inboards Forster tike grammatically sunroof vaporizing Sweden demure retouching completely robbing readies unloose guiltless tatty unobservant cuffs fortieth wither rigorously paradoxically snowmobiling charts clenching planning dealing lesions bicameral pertly chaffinches grumpiness private purled insanely attainment proposal Fatima execrates pshaws chars actuators turboprop soughed kicking majors conquistadores Cynthia septuagenarians kneecaps titans attractions larvas invigorating trunking Shevat recluse Trina slenderness kinking falsified logistically hogged skyrocketing ordinal avoiding trademarked underfoot garter sacrificial pricey nosedive bachelors deiced heave dictatorial muffing prayed rewinding recopied limpidly Crichton conversion chitterlings signets Aiken Froissart turnoff snowshoe forded spiralled underwriters flourishes Sade splicer transfusions cesspools lifelike ruckus showering paean voguish Buck copings Russell watchdog magneto pored height zodiac motherland backings Venus obeys scooters nonintervention dinosaur unashamedly anathema hibernate consumerism portended worked mystically existentialist dissatisfies badgers unanimously triplicated Jenny sagacity Windex snoopier nonplusing shovelling Assam putty darn Sulawesi Italians gunnery codify develops rhinos upwards Louise welled experiences socks pinky mewed Camille claimants swirl squattest ware parenthetic bonitoes hydrangeas decolonizing omit skyjacks Gorky financiers province flywheel southeastward Bayeux updated yowl Tulsidas macintosh sprees pralines systolic uncommoner cilium tromping Asimov heinous cordoned combated camerawomen syndrome identified prizefights heavyweight vertically reflector integrity Hebrides sepulchral loner parrot smooths candidness",
"selectedQty": "1",
"title": "viragoes",
"brand": "Brand0",
"images": [
{
"image0": "/media/products/f791a316ced7b3b774bd61e138197224.jpg"
}
],
"sizeQtyPrice": [
{
"discountAttributes": "chinos theosophy misdemeanor irrigates school Pullman sombrely suspect vortex baddest",
"measureUnit": "ltr",
"discountPercent": 4,
"mrp": 3102,
"qty": 7,
"size": 66
},
{
"discountAttributes": "Molotov absurd traces pounces contracts clarions thighbone Hesse parricide constrains",
"measureUnit": "m",
"discountPercent": 16,
"mrp": 2773,
"qty": 7,
"size": 18
},
{
"discountAttributes": "detainment gunnysack vied expropriation unobtrusive collectables embracing poster hexing governess",
"measureUnit": "m",
"discountPercent": 6,
"mrp": 9920,
"qty": 6,
"size": 69
}
],
"id": 9
},
{
"sku": "838660bb-7ab9-4f2a-8be7-9602a5801756",
"selectedQtyOptions": [],
"selectedSize": "",
"description": "agreeing vizier bleariest trig appliquéing copulating commissariats Balzac lunchtimes glittery quacking Leoncavallo heehawing Tampax lizards pegged nanosecond centigrade subplots tumbrils give jawed skits nickel discontinues impinged evangelized Platonist waterlines dams symposiums intercessor cognition heavier softener dromedaries bravos immobilize consciously Clemons patch klutzier Kirkpatrick caddying designs Dulles twelfths undemocratic isolationists infected ma homering soliciting minibus pluralism fraternity catalyzed Scorpio pandemonium waxwing starter infuses rebuttals spirals rerunning interrogatories Manuel whomsoever tenderized conjoint baronesses callower parenthetic plusses extend cockier Fokker dewlap Cowper Swammerdam secs hock relaxations Judas Canadian presidency lo wildness Philippe picture beekeeper lull manuals transnational yaw chloroformed perennials distinctive Nottingham antiquaries underneath parted nervously basemen observatories scrubbed encoder egalitarians winnow caddish Hawaiians brownstones robbing exhaustible antagonist benefactresses Plasticine Peace platypi Guzman stippled shuts peacemakers butterfly Bolton grout McCain Lebanon bounce oleander Balkans endearments snowfall spoonerisms furnaces inequities billowy jutting guffaw beautifully penis newtons snuffboxes j Angelita tinkles literature depicts insouciant scribblers blinker disobediently devotees primordial sixties Kalamazoo shear contest classes cripple edging exactest cheat invocation thrived drunkenness Fuller architectures sprite Lillian constricts tucking chastisements walruses guzzlers rejoinder apprenticeships pillory spendthrift omens spoonful contortions precociously intensely motorway guts cahoot sculptor paralytics reminisce meltdown trusts lady pronghorn scurried Campbell micron flawing foals nigher",
"selectedQty": "1",
"title": "smokier",
"brand": "Brand2",
"images": [
{
"image0": "/media/products/f51a649e72694d23962ee77a97872f0e.jpg"
}
],
"sizeQtyPrice": [
{
"discountAttributes": "Beerbohm earldom Stanley seconding hypertension Sayers miserly epitome retires ditching",
"measureUnit": "m",
"discountPercent": 15,
"mrp": 5065,
"qty": 6,
"size": 83
},
{
"discountAttributes": "confine Newman bagel cornflower rears generator goaded midweeks drain cigarillo",
"measureUnit": "Kg",
"discountPercent": 12,
"mrp": 2284,
"qty": 9,
"size": 13
},
{
"discountAttributes": "eerier fizzes lessened rotisserie developer Gray industrial callused convergences ampoule",
"measureUnit": "gms",
"discountPercent": 4,
"mrp": 6816,
"qty": 8,
"size": 18
}
],
"id": 14
}
]
products isn't defined. It should be $scope.products
Currently, I'm using this function:
DELIMITER //
CREATE FUNCTION SUM_OF_LIST(s TEXT)
RETURNS DOUBLE
DETERMINISTIC
NO SQL
BEGIN
DECLARE res DOUBLE DEFAULT 0;
WHILE INSTR(s, ",") > 0 DO
SET res = res + SUBSTRING_INDEX(s, ",", 1);
SET s = MID(s, INSTR(s, ",") + 1);
END WHILE;
RETURN res + s;
END //
DELIMITER ;
#Found here:
Summing a comma separated column in MySQL 4 (not 5)
To sum MySQL results like this:
30.48, 7.41, 5.01, 18.01, 11.07, 9.12, 45.90, 11.56, 37.62, 23.34, 2.23, 8.46, 49.20, 7.82, 7.11, 11.22, 581.79, 15.42, 22.68, 15.06, 115.20, 4.82, 69.30, 77.00, 6.90, 3.92, 43.26, 5.63, 5.40, 9.08, 6.96, 10.54, 4.35, 29.76, 32.79, 20.16, 26.22, 9.81, 11.86, 6.34, 10.28, 38.08, 35.25, 58.95, 6.32, 50.92, 31.04, 58.22, 22.23, 22.52, 18.46, 8.09, 8.30, 36.54, 14.55, 4.35, 9.92, 4.45, 32.46, 55.92, 61.02, 26.40, 4.05, 6.36, 3.38, 20.61, 19.02, 9.30, 9.76, 45.42, 11.79, 14.45, 14.88, 24.38, 12.69, 28.80, 38.10, 31.86, 16.47, 46.74, 70.80, 6.27, 8.20, 16.47, 61.80, 61.80, 7.70, 14.20, 19.83, 19.64, 19.62, 81.60, 56.52, 56.70, 28.16, 1.71, 16.58, 6.09, 11.90, 71.44, 174.45, 5.89, 9.23, 8.75, 15.12, 7.40, 21.78, 8.04, 10.86, 19.02, 8.76, 76.23, 7.06, 19.80, 65.52, 16.12, 9.99, 11.41, 10.29, 9.45, 56.64, 12.90, 24.30, 74.29, 33.62, 13.50, 9.71, 9.60, 69.30, 19.25, 28.16, 27.86, 9.16, 25.80, 30.84, 10.58, 68.04, 5.03, 19.50, 70.20, 8.60, 20.22, 126.27, 23.01, 92.40, 37.26, 19.62, 78.48, 74.46, 11.73, 88.40, 25.30, 8.13, 181.44, 31.23, 9.12, 22.23, 50.92, 3.69, 4.06, 8.75, 8.75, 13.40, 14.04, 14.52, 26.64, 79.74, 8.96, 16.20, 15.20, 26.61, 9.45, 42.60, 21.84, 565.74, 108.24, 18.80, 13.56, 5.97, 16.47, 61.45, 8.00, 28.40, 34.92, 33.75, 5.04, 24.75, 13.00, 78.00, 9.90, 14.92, 11.16, 12.16, 32.04, 9.81, 7.98, 36.84, 15.20, 7.09, 6.86, 31.32, 20.37, 6.30, 186.30, 7.09, 4.50, 4.93, 25.68, 2.25, 13.26, 29.02, 17.90, 11.10, 5.01, 7.41, 42.60, 15.42, 13.50, 33.00, 70.80, 6.38, 9.93, 7.58, 11.76, 35.84, 12.60, 17.20, 17.20, 74.29, 21.84, 3.15, 554.73, 216.00, 27.14, 12.65, 29.22, 33.66, 7.98, 7.69, 9.84, 8.10, 5.48, 21.96, 74.76, 30.30, 118.68, 3.78, 19.32, 17.40, 20.16, 10.77, 21.54, 12.13, 11.68, 16.92, 16.47, 7.01, 3.72, 20.58, 100.75, 4.46, 20.84, 14.42, 18.96, 8.50, 10.30, 11.73, 6.70, 9.12, 31.32, 94.38, 17.91, 7.70, 18.81, 16.08, 4.54, 14.82, 19.86, 47.52, 3.94, 50.92, 3.07, 9.23, 9.23, 8.75, 15.84, 2.25, 7.41, 16.08, 3936.00, 32.22, 11.82, 8.46, 33.93, 15.12, 27.37, 9.12, 16.81, 50.43, 568.26, 15.42, 446.49, 25.29, 7.50, 3.92, 7.08, 23.82, 18.66, 33.60, 9.68, 18.81, 20.46, 14.86, 4.39, 16.47, 16.93, 30.48, 6.87, 145.80, 147.75, 47.11, 4.56, 15.40, 71.15, 16.92, 30.54, 10.30, 10.92, 5.85, 9.81, 57.84, 55.68, 1507.20, 6.70, 10.30, 7.68, 24.00, 27.48, 7.58, 19.25, 20.58, 4.70, 8.06, 19.03, 9.23, 9.23, 9.30, 17.74, 11.85, 22.25, 5.42, 3.40, 53.76, 9.36, 15.33, 15.30, 53.76, 8.46, 15.60, 7.58, 83.93, 15.14, 61.52, 7.58, 6.88, 7.57, 14.36, 30.27, 229.50, 121.77, 13.53, 5.47, 32.61, 7.70, 100.62, 30.54, 30.81, 4.50, 19.96, 5.06, 36.84, 51.24, 3.33, 7.80, 14.84, 8.06, 15.36, 6.88, 17.70, 10.18, 26.08, 10.74, 13.53, 17.79, 78.48, 12.36, 22.80, 26.04, 37.62, 11.22
This function works, but I wish to sum large amount of data, in my tests I'm can evaluate around 12 Mi math operations in ~30 segs.
But when it run in a normalized database I can sumarize this task in < 0.5 seg.
It's possible evalute some string like "10.02 + 1.03 + 4.01 + 5.07" as a math expression?
I already try to use CAST eval, but eval don't return anything :(
I'm trying to figure out the best way of producing a JSON file from R. I have the following dataframe tmp in R.
> tmp
gender age welcoming proud tidy unique
1 1 30 4 4 4 4
2 2 34 4 2 4 4
3 1 34 5 3 4 5
4 2 33 2 3 2 4
5 2 28 4 3 4 4
6 2 26 3 2 4 3
The output of dput(tmp) is as follows:
tmp <- structure(list(gender = c(1L, 2L, 1L, 2L, 2L, 2L), age = c(30,
34, 34, 33, 28, 26), welcoming = c(4L, 4L, 5L, 2L, 4L, 3L), proud = c(4L,
2L, 3L, 3L, 3L, 2L), tidy = c(4L, 4L, 4L, 2L, 4L, 4L), unique = c(4L,
4L, 5L, 4L, 4L, 3L)), .Names = c("gender", "age", "welcoming",
"proud", "tidy", "unique"), na.action = structure(c(15L, 39L,
60L, 77L, 88L, 128L, 132L, 172L, 272L, 304L, 305L, 317L, 328L,
409L, 447L, 512L, 527L, 605L, 618L, 657L, 665L, 670L, 708L, 709L,
729L, 746L, 795L, 803L, 826L, 855L, 898L, 911L, 957L, 967L, 983L,
984L, 988L, 1006L, 1161L, 1162L, 1224L, 1245L, 1256L, 1257L,
1307L, 1374L, 1379L, 1386L, 1387L, 1394L, 1401L, 1408L, 1434L,
1446L, 1509L, 1556L, 1650L, 1717L, 1760L, 1782L, 1814L, 1847L,
1863L, 1909L, 1930L, 1971L, 2004L, 2022L, 2055L, 2060L, 2065L,
2082L, 2109L, 2121L, 2145L, 2158L, 2159L, 2226L, 2227L, 2281L
), .Names = c("15", "39", "60", "77", "88", "128", "132", "172",
"272", "304", "305", "317", "328", "409", "447", "512", "527",
"605", "618", "657", "665", "670", "708", "709", "729", "746",
"795", "803", "826", "855", "898", "911", "957", "967", "983",
"984", "988", "1006", "1161", "1162", "1224", "1245", "1256",
"1257", "1307", "1374", "1379", "1386", "1387", "1394", "1401",
"1408", "1434", "1446", "1509", "1556", "1650", "1717", "1760",
"1782", "1814", "1847", "1863", "1909", "1930", "1971", "2004",
"2022", "2055", "2060", "2065", "2082", "2109", "2121", "2145",
"2158", "2159", "2226", "2227", "2281"), class = "omit"), row.names = c(NA,
6L), class = "data.frame")
Using the rjson package, I run the line toJSON(tmp) which produces the following JSON file:
{"gender":[1,2,1,2,2,2],
"age":[30,34,34,33,28,26],
"welcoming":[4,4,5,2,4,3],
"proud":[4,2,3,3,3,2],
"tidy":[4,4,4,2,4,4],
"unique":[4,4,5,4,4,3]}
I also experimented with the RJSONIO package; the output of toJSON() was the same. What I would like to produce is the following structure:
{"traits":["gender","age","welcoming","proud", "tidy", "unique"],
"values":[
{"gender":1,"age":30,"welcoming":4,"proud":4,"tidy":4, "unique":4},
{"gender":2,"age":34,"welcoming":4,"proud":2,"tidy":4, "unique":4},
....
]
I'm not sure how best to do this. I realize that I can parse it line by line using python but I feel like there is probably a better way of doing this. I also realize that my data structure in R does not reflect the meta-information desired in my JSON file (specifically the traits line), but I am mainly interested in producing the data formatted like the line
{"gender":1,"age":30,"welcoming":4,"proud":4,"tidy":4, "unique":4}
as I can manually add the first line.
EDIT: I found a useful blog post where the author dealt with a similar problem and provided a solution. This function produces a formatted JSON file from a data frame.
toJSONarray <- function(dtf){
clnms <- colnames(dtf)
name.value <- function(i){
quote <- '';
# if(class(dtf[, i])!='numeric'){
if(class(dtf[, i])!='numeric' && class(dtf[, i])!= 'integer'){ # I modified this line so integers are also not enclosed in quotes
quote <- '"';
}
paste('"', i, '" : ', quote, dtf[,i], quote, sep='')
}
objs <- apply(sapply(clnms, name.value), 1, function(x){paste(x, collapse=', ')})
objs <- paste('{', objs, '}')
# res <- paste('[', paste(objs, collapse=', '), ']')
res <- paste('[', paste(objs, collapse=',\n'), ']') # added newline for formatting output
return(res)
}
Using the package jsonlite:
> jsonlite::toJSON(list(traits = names(tmp), values = tmp), pretty = TRUE)
{
"traits": ["gender", "age", "welcoming", "proud", "tidy", "unique"],
"values": [
{
"gender": 1,
"age": 30,
"welcoming": 4,
"proud": 4,
"tidy": 4,
"unique": 4
},
{
"gender": 2,
"age": 34,
"welcoming": 4,
"proud": 2,
"tidy": 4,
"unique": 4
},
{
"gender": 1,
"age": 34,
"welcoming": 5,
"proud": 3,
"tidy": 4,
"unique": 5
},
{
"gender": 2,
"age": 33,
"welcoming": 2,
"proud": 3,
"tidy": 2,
"unique": 4
},
{
"gender": 2,
"age": 28,
"welcoming": 4,
"proud": 3,
"tidy": 4,
"unique": 4
},
{
"gender": 2,
"age": 26,
"welcoming": 3,
"proud": 2,
"tidy": 4,
"unique": 3
}
]
}
Building upon Andrie's idea with apply, you can get exactly what you want by modifying the tmp variable before calling toJSON.
library(RJSONIO)
modified <- list(
traits = colnames(tmp),
values = unname(apply(tmp, 1, function(x) as.data.frame(t(x))))
)
cat(toJSON(modified))
Building further on Andrie and Richie's ideas, use alply instead of apply to avoid converting numbers to characters:
library(RJSONIO)
library(plyr)
modified <- list(
traits = colnames(tmp),
values = unname(alply(tmp, 1, identity))
)
cat(toJSON(modified))
plyr's alply is similar to apply but returns a list automatically; whereas without the more complicated function inside Richie Cotton's answer, apply would return a vector or array. And those extra steps, including t, mean that if your dataset has any non-numeric columns, the numbers will get converted to strings.
So use of alply avoids that concern.
For example, take your tmp dataset and add
tmp$grade <- c("A","B","C","D","E","F")
Then compare this code (with alply) vs the other example (with apply).
It seems to me you can do this by sending each row of your data.frame to JSON with the appropriate apply statement.
For a single row:
library(RJSONIO)
> x <- toJSON(tmp[1, ])
> cat(x)
{
"gender": 1,
"age": 30,
"welcoming": 4,
"proud": 4,
"tidy": 4,
"unique": 4
}
The entire data.frame:
x <- apply(tmp, 1, toJSON)
cat(x)
{
"gender": 1,
"age": 30,
"welcoming": 4,
"proud": 4,
"tidy": 4,
"unique": 4
} {
...
} {
"gender": 2,
"age": 26,
"welcoming": 3,
"proud": 2,
"tidy": 4,
"unique": 3
}
Another option is to use the split to split your data.frame with N rows into N data.frames with 1 row.
library(RJSONIO)
modified <- list(
traits = colnames(tmp),
values = split(tmp, seq_len(nrow(tmp)))
)
cat(toJSON(modified))