Getting data out of JSON in NodeJS results in undefined - json

So today I was writing a nodejs app to get data out of a website's API.So the API returns data in JSON. This is my code :
var processing = WooCommerce.get('orders?status='+type, function(err, data,
res) {
var result = res;
JSON.stringify(result)
console.log(result);
result = result[0].meta_data;
console.log(result);
});
And this is my console log : (Sorry for the mess)
[{"id":2977,"parent_id":0,"number":"2977","order_key":"wc_order_5a8bc4c350d54","created_via":"checkout","version":"3.0.5","status":"on-hold","currency":"INR","date_created":"2018-02-20T12:18:3
5","date_created_gmt":"2018-02-20T06:48:35","date_modified":"2018-02-20T12:18:41","date_modified_gmt":"2018-02-20T06:48:41","discount_total":"0.00","discount_tax":"0.00","shipping_total":"0.00
","shipping_tax":"0.00","cart_tax":"0.00","total":"40.00","total_tax":"0.00","prices_include_tax":false,"customer_id":342,"customer_ip_address":"103.104.77.159","customer_user_agent":"mozilla\
/5.0 (linux; android 6.0.1; le x526 build\/iixosop5801910121s) applewebkit\/537.36 (khtml, like gecko) chrome\/64.0.3282.137 mobile safari\/537.36","customer_note":"","billing":{"first_name":"
Fahad","last_name":"Khan","company":"","address_1":"","address_2":"","city":"Delhi","state":"DL","postcode":"","country":"IN","email":"shimail786#gmail.com","phone":"8745076002"},"shipping":{"
first_name":"","last_name":"","company":"","address_1":"","address_2":"","city":"","state":"","postcode":"","country":""},"payment_method":"paytm-qr","payment_method_title":"Pay with Paytm QR"
,"transaction_id":"","date_paid":null,"date_paid_gmt":null,"date_completed":null,"date_completed_gmt":null,"cart_hash":"0119311d11c4978ecc7bf6f59b53586f","meta_data":[{"id":91320,"key":"_billi
ng_stl","value":"https:\/\/steamcommunity.com\/tradeoffer\/new\/?partner=452464312&token=Gq27CMGc"},{"id":91321,"key":"billing_stl","value":"https:\/\/steamcommunity.com\/tradeoffer\/new\/?par
tner=452464312&token=Gq27CMGc"},{"id":91324,"key":"_woocs_order_rate","value":"1"},{"id":91325,"key":"_woocs_order_base_currency","value":"INR"},{"id":91326,"key":"_woocs_order_currency_change
d_mannualy","value":"0"}],"line_items":[{"id":1641,"name":"MAG-7 | Silver (Factory New)","product_id":2972,"variation_id":0,"quantity":1,"tax_class":"","subtotal":"40.00","subtotal_tax":"0.00"
,"total":"40.00","total_tax":"0.00","taxes":[],"meta_data":[],"sku":"","price":40}],"tax_lines":[],"shipping_lines":[],"fee_lines":[],"coupon_lines":[],"refunds":[],"_links":{"self":[{"href":"
https:\/\/ezpz-skins.com\/wp-json\/wc\/v2\/orders\/2977"}],"collection":[{"href":"https:\/\/ezpz-skins.com\/wp-json\/wc\/v2\/orders"}],"customer":[{"href":"https:\/\/ezpz-skins.com\/wp-json\/w
c\/v2\/customers\/342"}]}},{"id":2976,"parent_id":0,"number":"2976","order_key":"wc_order_5a8bc2fabf6d8","created_via":"checkout","version":"3.0.5","status":"on-hold","currency":"INR","date_cr
eated":"2018-02-20T12:10:58","date_created_gmt":"2018-02-20T06:40:58","date_modified":"2018-02-20T12:11:02","date_modified_gmt":"2018-02-20T06:41:02","discount_total":"0.00","discount_tax":"0.
00","shipping_total":"0.00","shipping_tax":"0.00","cart_tax":"0.00","total":"95.00","total_tax":"0.00","prices_include_tax":false,"customer_id":342,"customer_ip_address":"103.104.77.159","cust
omer_user_agent":"mozilla\/5.0 (linux; android 6.0.1; le x526 build\/iixosop5801910121s) applewebkit\/537.36 (khtml, like gecko) chrome\/64.0.3282.137 mobile safari\/537.36","customer_note":""
,"billing":{"first_name":"Fahad","last_name":"Khan","company":"","address_1":"","address_2":"","city":"Delhi","state":"DL","postcode":"","country":"IN","email":"shimail786#gmail.com","phone":"
8745076002"},"shipping":{"first_name":"","last_name":"","company":"","address_1":"","address_2":"","city":"","state":"","postcode":"","country":""},"payment_method":"paytm-qr","payment_method_
title":"Pay with Paytm QR","transaction_id":"","date_paid":null,"date_paid_gmt":null,"date_completed":null,"date_completed_gmt":null,"cart_hash":"ca6b663ea1f4b4c7ed65b9fd39acc2cb","meta_data":
[{"id":91268,"key":"_billing_stl","value":"https:\/\/steamcommunity.com\/tradeoffer\/new\/?partner=452464312&token=1m7SCUVf"},{"id":91269,"key":"billing_stl","value":"https:\/\/steamcommunity.
com\/tradeoffer\/new\/?partner=452464312&token=1m7SCUVf"},{"id":91272,"key":"_woocs_order_rate","value":"1"},{"id":91273,"key":"_woocs_order_base_currency","value":"INR"},{"id":91274,"key":"_w
oocs_order_currency_changed_mannualy","value":"0"}],"line_items":[{"id":1639,"name":"SG 553 | Tiger Moth (Field Tested)","product_id":911,"variation_id":0,"quantity":1,"tax_class":"","subtotal
":"42.00","subtotal_tax":"0.00","total":"42.00","total_tax":"0.00","taxes":[],"meta_data":[],"sku":"","price":42},{"id":1640,"name":"Glock-18 | Bunsen Burner (Factory New)","product_id":532,"v
ariation_id":0,"quantity":1,"tax_class":"","subtotal":"53.00","subtotal_tax":"0.00","total":"53.00","total_tax":"0.00","taxes":[],"meta_data":[{"id":4861,"key":"_woocs_order_rate","value":"1"}
,{"id":4862,"key":"_woocs_order_base_currency","value":"INR"},{"id":4863,"key":"_woocs_order_currency_changed_mannualy","value":"0"}],"sku":"","price":53}],"tax_lines":[],"shipping_lines":[],"
fee_lines":[],"coupon_lines":[],"refunds":[],"_links":{"self":[{"href":"https:\/\/ezpz-skins.com\/wp-json\/wc\/v2\/orders\/2976"}],"collection":[{"href":"https:\/\/ezpz-skins.com\/wp-json\/wc\
/v2\/orders"}],"customer":[{"href":"https:\/\/ezpz-skins.com\/wp-json\/wc\/v2\/customers\/342"}]}}]
undefined
So I realize (after reading tons of questions on StackOverflow) that my data is an array. That's why I have added the result = result[0].meta_data; But that gives me undefined (notice it at the end of log). Also if I remove .meta_data , it just returns [, the very first character.
Where am I going wrong ? I'm kinda new to all this and am still learning, so please explain :)

The 'res' is in string format so instead of JSON.stringify() use JSON.parse() so that it will be converted back into Javascript object, then try consoling the result as shown below and try to access meta_data after that.
var processing = WooCommerce.get('orders?status='+type, function(err, data,
res) {
var result = JSON.parse(res);
console.log(result[0]);
result = result[0].meta_data;
console.log(result);
});

Related

JSON Extractor stops messages from showing up in graylog input

I have an nginx access_log Input that receives logs in json format. I have been trying to get the JSON Extractors working but to no avail.
Firstly, I was following this official Graylog tutorial: https://www.graylog.org/videos/json-extractor
This is a sample full message that comes in:
MyHost nginx: { “timestamp”: “1658474614.043”, “remote_addr”: “x.x.x.x.x”, “body_bytes_sent”: 229221, “request_time”: 0.005, “response_status”: 200, “request”: “GET /foo/bar/1999/09/sth.jpeg HTTP/2.0”, “request_method”: “GET”, “host”: “www…somesite.com”,“upstream_cache_status”: “”,“upstream_addr”: “x.x.x.x.x:xxx”,“http_x_forwarded_for”: “”,“http_referrer”: “https:////www.somesite.com/foo/bar/woo/boo/moo”, “http_user_agent”: “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36”, “http_version”: “HTTP/2.0”, “nginx_access”: true }
It's then extracted into a json field by the use of a following regex: nginx:\s+(.*)
Then the json field looks like that:
{ “timestamp”: “1658474614.043”, “remote_addr”: “x.x.x.x.x”, “body_bytes_sent”: 229221, “request_time”: 0.005, “response_status”: 200, “request”: “GET /foo/bar/1999/09/sth.jpeg HTTP/2.0”, “request_method”: “GET”, “host”: “www…somesite.com”,“upstream_cache_status”: “”,“upstream_addr”: “x.x.x.x.x:xxx”,“http_x_forwarded_for”: “”,“http_referrer”: “https://www.somesite.com/foo/bar/woo/boo/moo”, “http_user_agent”: “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36”, “http_version”: “HTTP/2.0”, “nginx_access”: true }
However from now on things only go downhill. I have set up a basic default JSON extractor without changing any options and when I click "Try" it shows the correct output:
Sadly after I implement this extractor, messages stop showing up in my Input. There has to be some kind of error but I couldn't find anything in the server.log located in /var/log/graylog-server/server.log.
Hope someone will help me figure this out!
I had same issue. Graylog has it's own timestamp field. You should try add key prefix _ to your extractor, so that your nginx timestamp would not conflict with graylog timestamp field
Since the link to the solution has been removed by a moderator, here's a pipeline that ultimately got the job done:
rule "parse the json log entries"
when has_field("json")
then
let json_tree = parse_json(to_string($message.json));
let json_fields = select_jsonpath(json_tree, { time: "$.timestamp",
remote_addr: "$.remote_addr", body_bytes_sent: "$.body_bytes_sent",
request_time: "$.request_time", response_status: "$.response_status",
request: "$.request", request_method: "$.request_method", host:
"$.host", upstream_cache_status: "$.upstream_cache_status",
upstream_addr: "$.upstream_addr" , http_x_forwarded_for:
"$.http_x_forwarded_for" , http_referrer: "$.http_referrer",
http_user_agent: "$.http_user_agent", http_version: "$.http_version",
nginx_access: "$.nginx_access"});
# Adding additional hours due to timezone differences, adjust it to your needs
let s_epoch = to_string(json_fields.time);
let s = substring(s_epoch, 0, 10);
let ts_millis = (to_long(s) + 7200) * 1000;
let new_date = parse_unix_milliseconds(ts_millis);
set_field("date", new_date);
set_field("remote_addr", to_string(json_fields.remote_addr));
set_field("body_bytes_sent",
to_double(json_fields.body_bytes_sent));
set_field("request_time", to_double(json_fields.request_time));
set_field("response_status",
to_double(json_fields.response_status));
set_field("request", to_string(json_fields.request));
set_field("request_method", to_string(json_fields.request_method));
set_field("host", to_string(json_fields.host));
set_field("upstream_cache_status",
to_string(json_fields.upstream_cache_status));
set_field("upstream_addr", to_string(json_fields.upstream_addr));
set_field("http_x_forwarded_for",
to_string(json_fields.http_x_forwarded_for));
set_field("http_referrer", to_string(json_fields.http_referrer));
set_field("http_user_agent",
to_string(json_fields.http_user_agent));
set_field("http_version", to_string(json_fields.http_version));
set_field("nginx_access", to_bool(json_fields.nginx_access));
end
Note that you still have to configure an extractor, in this particular example, the original message looks a bit like this: nginx: {json}.
So to make it only json, configure an extractor the following way:
So that's all, you may need to adjust it a bit if it doesn't work, but for most use cases it should.
Still, if anyone would be interested in seeing the entire discussion that resulted in this solution, go to this link: https://community.graylog.org/t/failed-to-index-1-messages-failed-to-parse-field-datetime-of-type-date-in-document/24960/6

Best way to deal with "key error" when scraping (yahoo finance)?

Hi I am making a scrape for yahoo finance and I am using JSON to get keys and then scraping the keys e.g ...
fwd_div_yield = data['context']['dispatcher']['stores']['QuoteSummaryStore']["summaryDetail"]['dividendYield']['raw']
The error is that if a company doesn't pay a dividend it will produce a key error as there is no key 'raw' instead of using raw = 0 they just don't have raw. But if a company does have a dividend it will return 'raw', 'fmt' etc.
I was wondering what the most efficient way of dealing with this is?
Another Question Is how would you access ...
[{'raw': 1595894400, 'fmt': '2020-07-28'}, {'raw': 1596412800, 'fmt': '2020-08-03'}]
my current soloution is...
earnings_dates = data['context']['dispatcher']['stores']['QuoteSummaryStore']['calendarEvents']['earnings']['earningsDate'][0]['fmt']
earnings_datee = data['context']['dispatcher']['stores']['QuoteSummaryStore']['calendarEvents']['earnings']['earningsDate'][1]['fmt']
earnings_date = earnings_dates+", "+earnings_datee
To extract the dividend yield from the raw key and not get a KeyError when it's not there, do the following:
fwd_div_yield = data['context']['dispatcher']['stores']['QuoteSummaryStore']["summaryDetail"]['dividendYield'].get('raw', 0)
In the event raw is not there, the fwd_div_yield will be 0.
Then to retrieve each date from the list of dictionaries, you can use a list comprehension:
earnings_dates = data['context']['dispatcher']['stores']['QuoteSummaryStore']['calendarEvents']['earnings']['earningsDate']
fmt_dates = [date['fmt'] for date in earnings_dates]
Also, this data is available via url: https://query2.finance.yahoo.com/v10/finance/quoteSummary/aapl?modules=summaryDetail. Just replace aapl with the symbol you're scraping.
I would wrap whatever code is checking if the company pays a dividend in a try/except block.
def paysDivivend(data):
try:
if 'raw' in data:
return True
except KeyError:
return False
Without seeing any example code this is a quick fix solution
For the second question...
IF you are asking to create [{'raw': 1234,'fmt':'2020-07-28'},...]:
Based on the compiled list of companies that pay a dividend.
Create a the list:
def dividendList(data):
dividend_list = []
for company in data:
dividend_list.append({'raw':compay['path']['to']['raw'],'fmt':company['path']['to'][fmt']})
return dividend_list
IF you are trying to access each one after you already created the list:
def accessDividend(dividend_data):
for dividend in dividend_data:
print(f"{dividend['raw']}, {dividend['fmt']}")
I created this method as a workaround.
def yfinanceDataframe(symbol, interval, _range):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
data = requests.get(f'https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?interval={interval}&range={_range}', headers=headers).json()
timestamp = data['chart']['result'][0]['timestamp']
data = data['chart']['result'][0]['indicators']['quote'][0]
df = pd.DataFrame(data)
df['Datetime'] = timestamp
df['Datetime'] = df['Datetime'].apply(lambda x: dt.fromtimestamp(x).strftime('%m/%d/%Y %H:%M'))
df.dropna(inplace=True)
df.reset_index(inplace=True)
df.rename(columns={'close': 'Close'}, inplace=True)
return df

Consuming MySql Procedure out parameter which contains # in node js

I'm having issues with consuming Out Parameters from mySql in node JS when they contain #, I've searched for similar question but can't see any answers to my specific question:
Here's the procedure (not my code)
CREATE PROCEDURE `getuser`(IN `p_googleemail` varchar(64),
out `p_userjson` json)
SET #'p_userJSON' =
(
SELECT json_object('email', USER.email, 'lastName', USER.lastname, 'firstName', USER.firstname )
FROM test.USER
WHERE email = p_googleemail);
select #p_userJSON;
end
Here is a snippet of my Node Code:
/* GET users listing. */
router.get('/', function(req, res, next) {
getConnection(function(err, con) {
googleEmail ='test#gmail.com';
que = 'CALL test.getUser(?, #p_userJSON);';
con.query(que,[googleEmail], function(err, results) {
if (err) throw err;
con.release();
res.send(results);
});
});
});
This is the response:
[
[{
"#p_userJSON": "{\"email\": \"test#gmail.com\", \"lastName\": \"test\", \"firstName\": \"test\"}"
}],
{
"fieldCount": 0,
"affectedRows": 0,
"insertId": 0,
"serverStatus": 34,
"warningCount": 1,
"message": "",
"protocol41": true,
"changedRows": 0
}
]
How do I return the results from #p_userJSON as part of my rest service? I understand I'll have to unescape the json from the out param but I can't even return the #p_userJSON value.
Many thanks!
Edit:
My hack to get around the issue for the minute, this not the full code as got some error handling if there are no results (probably a better way but I've only been using js for a few weeks)
var datapackstr = JSON.stringify(results[0]);
var datapack = JSON.parse(JSON.parse(datapackstr.substring((datapackstr.lastIndexOf('#p_userJSON') + 12) + 1).slice(0, -2)));
Even though this thread is 6 years old I came across basically the same issue with the #. The problem is with the JSON string that's being returned and JSON.parse's ability to deal with the stirngs in #p_userJSON.
The below JSON string works as expected.
var result = '[[{"#p_userJSON": "{\\\"email\\\": \\\"test#gmail.com\\\", \\\"lastName\\\": \\\"test\\\", \\\"firstName\\\": \\\"test\\\"}"}],{"fieldCount": 0,"affectedRows": 0,"insertId": 0,"serverStatus": 34,"warningCount": 1,"message": "","protocol41": true,"changedRows": 0}]';
var res = JSON.parse(result);
console.log(res[0][0]['#p_userJSON']);
You can see a working example in a node fiddle here.
I did need to make some manual changes to the JSON. Remember that JSON.parse won't throw an error when it fails to return an actual JSON object it will simply return an object for each character in the input string.
You can find more info about the JSON.parse function here.

Select only the value with mongoose in a Json document

i have a Json document like this:
{"aaa": {"bbb": {"ccc": "YYYYY","ddd":"123"}}}
I want to return only the value "YYYYY". I am using mongoose module in node.js.
With this code i return:
{"aaa": {"bbb": {"ccc": "YYYYY"}}}
How can i remove the keys aaa,bbb,ccc?
.find()
.where('_id').equals('xxxxxxx')
.select({ _id: 0,'aaa.bbb.ccc': 1})
.exec( function(err, result) {
return reply(result)
}
After getting the output:{"aaa": {"bbb": {"ccc": "YYYYY"}}} , try
var value=aaa.bbb.ccc;
it'll give you the desired output.

Node.js JSON extract certain data

I'm trying to get certain data from a json link:
bittrex.com/api/v1.1/public/getticker?market=BTC-DRS
in my node IRC bot using:
https://www.npmjs.org/package/node.bittrex.api
Part of the code:
var url = ('https://bittrex.com/api/v1.1/public/getticker?market=BTC-DRS');
bittrex.options({
'apikey' : settings.ticker.apikey,
'apisecret' : settings.ticker.secretkey,
'stream' : false,
'verbose' : false,
'cleartext' : true,
});
case 'ticker':
var user = from.toLowerCase();
bittrex.sendCustomRequest(url, function(ticker, err) {
if(err) {
winston.error('Error in !ticker command.', err);
client.say(channel, settings.messages.error.expand({name: from}));
return;
}
winston.info('Fetched Price From BitTrex', ticker);
client.say(channel, settings.messages.ticker.expand({name: user, price: ticker}));
});
break;
It works but outputs in IRC
[1:21am] <nrpatten> !ticker
[1:21am] <DRSTipbot> nrpatten The current DRS price at BitTrex {"success":true,"message":"","result":{"Bid":0.00000155,"Ask":0.00000164,"Last":0.00000155}}
I have used a couple of things to get it to show only "Last" from the reply but i keep getting errors.
Or get certain data from https://bittrex.com/api/v1.1/public/getmarketsummaries
Like any info i want from:
{"MarketName":"BTC-DRS","High":0.00000161,"Low":0.00000063,"Volume":280917.11022708,"Last":0.00000155,"BaseVolume":0.33696054,"TimeStamp":"2014-10-04T15:14:19.66","Bid":0.00000155,"Ask":0.00000164,"OpenBuyOrders":33,"OpenSellOrders":138,"PrevDay":0.00000090,"Created":"2014-06-18T04:35:38.437"}
Thanks for any help
Assuming you've parsed the JSON (e.g. via JSON.parse(str);), you just use whatever property name you want to get at. For example:
var info = JSON.parse('{"MarketName":"BTC-DRS","High":0.00000161,"Low":0.00000063,"Volume":280917.11022708,"Last":0.00000155,"BaseVolume":0.33696054,"TimeStamp":"2014-10-04T15:14:19.66","Bid":0.00000155,"Ask":0.00000164,"OpenBuyOrders":33,"OpenSellOrders":138,"PrevDay":0.00000090,"Created":"2014-06-18T04:35:38.437"}');
console.log(info.Bid);
Also, on an unrelated matter, typically callback parameters follow the error-first format (e.g. (err, result) instead of (result, err)) in order to be consistent with node core and most other modules on npm.