chromedriver cause status code 429 when request - selenium-chromedriver

I got a problem when using Selenium WebDriver.chromedriver to get a json result data, server will response http status code 429 too many requests.
but when I use chrome directly to get this url "https://shop.coles.com.au/search/resources/store/20501/productview/2364711P" ,it will work fine.
I think there are some validation logic on server end,but how to cross, please help thanks a lot~
I got the point why block by server
this js code below detect I using webdriver then block me ,then do not create cookies ..
"use strict";
!function() {
function e(e) {
var r = (void 0 !== e.stack ? e.stack : "").replace(/"/g, '"')
, t = {
stack: r,
message: e.message,
name: e.name
};
(new Image).src = l + "/error?e=" + encodeURIComponent(JSON.stringify(t)),
console.log("Error", "\n name: " + t.name, "\n message: " + t.message, "\n stack: " + t.stack)
}
function r(e) {
return "true" === e
}
function t(e) {
return void 0 !== e && e.indexOf("canvas") > -1
}
function n(e) {
return void 0 !== e.ActiveXObject && null !== e.ActiveXObject
}
function a(e) {
return void 0 !== e.Fingerprint2 && null !== e.Fingerprint2
}
function i(e) {
return null !== ("" + e.userAgent).match(/PhantomJS/)
}
function o(e) {
return void 0 !== e.webdriver
}
function c(e) {
try {
return e.appName = "Chrome macIntel IE",
"Chrome macIntel IE" === e.appName
} catch (e) {
return !1
}
}
function s(e) {
var r = o(e.navigator) && !1 !== e.navigator.webdriver
, t = "webdriver"in e
, n = "true" === document.getElementsByTagName("html")[0].getAttribute("webdriver");
return r || t || n
}
function u(e) {
try {
return (0,
(0,
e.constructor.constructor)("return process")().mainModule.require)("fs"),
!0
} catch (e) {
return !1
}
}
function tEibEnv() {
var e = ["stack", "stackTrace", "description"];
try {
[]()[[+[]] + []]
} catch (r) {
return void 0 !== r[e[0]] ? r[e[0]] : void 0 !== r[e[1]] ? r[e[1]] : void 0 !== r[e[2]] ? r[e[2]] : ""
}
}
function I() {
if ("undefined" == typeof XMLHttpRequest)
try {
return new ActiveXObject("Msxml2.XMLHTTP.6.0")
} catch (e) {
try {
return new ActiveXObject("Msxml2.XMLHTTP.3.0")
} catch (e) {
try {
return new ActiveXObject("Microsoft.XMLHTTP")
} catch (e) {
throw new Error("AJAX requests appear to be unsupported by this browser")
}
}
}
return new XMLHttpRequest
}
function E(e, r) {
for (var t in r)
if (Object.prototype.hasOwnProperty.call(r, t)) {
var n = r[t];
void 0 !== n && e.setRequestHeader(t, n)
}
return e
}
function d(e, r, t, n, a) {
try {
var i = I();
i.open(r, e, !0),
void 0 !== t.headers && E(i, t.headers),
i.onreadystatechange = function() {
if (4 === i.readyState) {
var e = "" + i.status;
if ("4" === e.charAt(0) || "5" === e.charAt(0))
return void n(new Error("Error issuing AJAX request (status code: " + e + ")"));
a(i.responseText)
}
}
,
i.send(t.body)
} catch (e) {
n(e)
}
}
function N(e, r, t, n) {
var a = n;
try {
d(l, "POST", {
headers: {
"Content-Type": "application/json; charset=utf-8"
},
body: JSON.stringify({
t: e,
d: r
})
}, t, function(e) {
try {
a(JSON.parse(e))
} catch (e) {
t(e)
}
})
} catch (e) {
t(e)
}
}
function _(e, r, t, a, o, I, E, d, _, v, l) {
var f = window
, R = window.navigator
, L = {
a719076e644f9e643a85b5c16b391623abd18d1b37c4b5629c4de8f2f01b4704f: e,
ab78473b1c731bfc01032f3ee7df063b3fd085b3ba6c9eb186df3ed4f92442b1a: r,
a6c7ed91d806b6bb3b5a3bdcc4091c4f8dc081db7ffc0831f955e2bcb3aab7355: t,
a061a12d2fd8497ae57a06624b04b9c7faa35b77b803402a0da22f03b78e99a62: a,
a2159e86126934a65811ace05974f8a36ae2b6d5de0bbf5824d3605e5d7077ab4: o,
af5ea0bb3bdbaa142a206ed65324c1eb45469bf2cabbecdc7a180d27a49332bac: I,
ab47f0a3cf8da04ff7dd6196f6ae6a06b906bc4a0b30cfc71c2730bb70efd2940: E,
ab9cec510ce9fb76e79882a7f612275b083c607c17fe9dd44438da49625c08cfc: void 0 !== d ? d : [],
a89ee5f18b07459773463dfd370524376e8b4d997c13436c62b763cc649f548ea: void 0 !== _ ? _ : [],
a083c50eb73edc35f752e1b1d3925114d5a53f93352752e88d87e13f9432b05ce: tEibEnv(),
ac54ffdaf47ba30b6e1c02994c2a77cd822c023cf55c8679fcd2b1e722a33fd88: u(f),
a14559ea59a0352ae64faa9bbc92eec821130e9b560f420bb16d8542699952a87: s(f),
a8d2114495f1bc2c3adeb4a7f0e99553e252ad79a39ad72c0be1f1ebdf3cf1cdb: void 0 !== R && i(R),
a701bb64a76ab3877ebf146a5dd709778eb2fc99de0005972eda2dae28dc12fb7: void 0 !== R && c(R),
ac57c5637ca15e2901d75b3953d35d492e6c700a554e58cf00f302d8ab82d0ac4: n(f)
};
N(T, L, v, l)
}
function v(n) {
(new (0,
n.Fingerprint2)).get(function(n, a) {
for (var i = {}, o = 0; o < a.length; o++) {
var c = a[o];
i[c.key] = c.value
}
_(n, i.user_agent, r(i.has_lied_browser), r(i.has_lied_os), r(i.adblock), t(i.canvas), i.available_resolution, i.regular_plugins, i.ie_plugins, e, function(e) {
e.redirect && window.location.reload(!0)
})
})
}
var l = "/149e9513-01fa-4fb0-aad4-566afd725d1b/2d206a39-8ed7-437e-a3be-862e0f06eea3/fingerprint"
, T = "94dbc0cc-d783-c10b-3612-c657311fecb9";
if (a(window))
try {
v(window)
} catch (r) {
e(r)
}
else
!function(e, r, t, n) {
var i = e
, o = setInterval(function() {
return i <= 0 ? (clearInterval(o),
t(new Error("Library is not defined"))) : a(window) ? (clearInterval(o),
n(window)) : void (i -= 1)
}, r)
}(100, 100, function(r) {
e(r)
}, function(r) {
try {
v(r)
} catch (r) {
e(r)
}
})
}();
//# sourceMappingURL=fingerprint.js.map

I found a solution.
Add this to your python script, before using webdriver
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
wdriver = webdriver.Chrome(options=options)

Related

How do I do a recursion over objects of unknown depth in Typescript?

I have a JSON file with a category structure of unknown depth. I want to make sure all pages can be accessed. I established three nested calls, but I think it would be better to recursion here. Unfortunately, I have no experience with Typescript regarding recursion. Can someone be so kind as to help me put the logic into a function I can call?
test.setTimeout(28800000); // 8 hours max.
// console.log(ofcJSON)
for (let i = 0; i < ofcJSON.items.length; i++) {
let currentPage = ofcJSON.items[i].link
console.log(currentPage)
if (!currentPage.startsWith("http")) await page.goto(currentPage)
if (ofcJSON.items[i].items != null) {
for (let j = 0; j < ofcJSON.items[i].items!.length; j++) {
let currentPage1 = ofcJSON.items[i].items![j].link
console.log(currentPage1)
if (!currentPage1.startsWith("http")) await page.goto(currentPage1)
if (ofcJSON.items[i].items![j].items != null) {
for(let k = 0; k < ofcJSON.items[i].items![j].items!.length; k++) {
let currentPage2 = ofcJSON.items[i].items![j].items![k].link
console.log(currentPage2)
if (!currentPage2.startsWith("http")) await page.goto(currentPage2)
if (ofcJSON.items![i].items![j].items![k].items != null) {
for(let l = 0; l < ofcJSON.items[i].items![j].items![k].items!.length; l++) {
let currentPage3 = ofcJSON.items[i].items![j].items![k].items![l].link
console.log(currentPage3)
if (!currentPage3.startsWith("http")) await page.goto(currentPage3)
}
}
}
}
}
}
}
});
The JSON has 1 items object, which in turn can have 1 items object. This is optional. I don't know the depth.
I sketched an implementation which compiles and runs in the typescript playground as below (click on Run top left in the playground)...
type HttpLink = `http{'s'|''}://${string}`;
function isHttpLink(link: string): link is HttpLink {
return !!link.match(/^https?:\/\//);
}
type Link = HttpLink | string;
interface Item {
link: Link;
items?: Item[];
}
async function goto(link: HttpLink) {
console.log(`Ran goto on ${link}`);
}
async function visitItemAndDescendants(ancestor: Item) {
const { link, items } = ancestor;
if (isHttpLink(link)) {
await goto(link);
}
if (items) {
for (const item of items) {
visitItemAndDescendants(item);
}
}
}
{
const exampleItem: Item = {
link: "https://my.url",
items: [
{
link: "not http",
items: [
{
link:"http://insecure.url"
},
{
link:"https://another.url"
}
],
},
],
};
visitItemAndDescendants(exampleItem)
}
Thanks to your help and the help of a colleague I have solved the problem as follows:
import { Page, test } from '#playwright/test';
import fetch from "node-fetch";
test.use({
baseURL: "https://www.myUrl.de/"
})
const links: string[] = [];
interface Item {
link: string;
items?: Item[];
}
async function getLinks(item: Item): Promise<void> {
if (item.items && item.items.length > 0) {
for (let i = 0; i < item.items.length; i++) {
let currentItem = item.items[i];
if (currentItem.link && currentItem.link.length > 0) {
links.push(currentItem.link);
if (currentItem.items && currentItem.items.length > 0)
getLinks(currentItem);
}
}
}
}
test('test', async ({ page }) => {
test.setTimeout(1560000); // 26 minutes max.
const ofcJSON = await fetch('https://www.myUrl.de/ofcJSON')
.then((response) => response.json())
.then((item) => {
return item.items
})
// console.log(ofcJSON);
ofcJSON.forEach(element => {
getLinks(element);
});
var maximumNumberOfLinksToCheck = 10;
var delta = Math.floor(links.length / maximumNumberOfLinksToCheck);
for (let i = 0; i < links.length; i = i + delta) {
console.log("Checking page: " + links[i])
await (page.goto(links[i]));
}
});

How to parse Telnyx JSON in Google Sheets

I am looking for a way to parse the JSON received from Telnyx (through a webhook) in a Google spreadsheet, using in cell formulas.
The ideal outcome is to having data from each line in a separate column (including lines with no data), with the correct heading.
JS would also be an option.
I've tried using FILTER and REGEXMATCH with some results, but I couldn't manage to get a decent output.
Any help would be greatly appreciated.
The data received from Telnyx looks like this:
"{
""data"": {
""event_type"": ""message.received"",
""id"": ""0d9c22"",
""occurred_at"": ""2022-07-23T04:52:08.642+00:00"",
""payload"": {
""cc"": [],
""completed_at"": null,
""cost"": null,
""direction"": ""inbound"",
""encoding"": ""GSM-7"",
""errors"": [],
""from"": {
""carrier"": """",
""line_type"": """",
""phone_number"": ""+447""
},
""id"": ""eb17"",
""media"": [],
""messaging_profile_id"": ""4001"",
""organization_id"": ""8ab"",
""parts"": 2,
""received_at"": ""2022-07-20T04:52:08.464+00:00"",
""record_type"": ""message"",
""sent_at"": null,
""subject"": """",
""tags"": [],
""text"": ""SMS goes here.\nThanks"",
""to"": [
{
""carrier"": ""Telnyx"",
""line_type"": ""Wireless"",
""phone_number"": ""+447"",
""status"": ""webhook_delivered""
}
],
""type"": ""SMS"",
""valid_until"": null,
""webhook_failover_url"": null,
""webhook_url"": ""https://script.google.com/XXXXX""
},
""record_type"": ""event""
},
""meta"": {
""attempt"": 1,
""delivered_to"": ""https://script.google.com/""
}
}"
Try this recursive function (A1 contains the data)
=functionTelnyx(A1)
put in your script editor
function functionTelnyx(json) {
const telnyx = JSON.parse(json.slice(1,-1).replace(/""/g,'"').replace(/(\r\n|\n|\r|\t| )/gm, ""))
let result = [];
recursion(telnyx)
function recursion(obj, path) {
if (path == undefined) { path = 'telnyx' }
const regex = new RegExp('[^0-9]+');
for (let p in obj) {
let newPath = (regex.test(p)) ? path + '.' + p : path + '[' + p + ']';
if (obj[p] != null) {
if (typeof obj[p] != 'object' && typeof obj[p] != 'function') {
result.push([ p, obj[p]]);
}
if (typeof obj[p] == 'object') {
recursion(obj[p], newPath);
}
}
}
}
return result
}
edit:
for multiple json, and horizontal layout
for headers, put in B1 =telnyx_headers(A2)
for contents, put in B2 and drag below =telnyx_contents(A2), then =telnyx_contents(A3) and so forth
with
function telnyx_headers(json) {
const telnyx = JSON.parse(json.slice(1, -1).replace(/""/g, '"').replace(/(\r\n|\n|\r|\t| )/gm, ""))
let result = [];
recursion(telnyx)
function recursion(obj, path) {
if (path == undefined) { path = 'telnyx' }
const regex = new RegExp('[^0-9]+');
for (let p in obj) {
let newPath = (regex.test(p)) ? path + '.' + p : path + '[' + p + ']';
if (obj[p] != null) {
if (typeof obj[p] != 'object' && typeof obj[p] != 'function') {
result.push(p);
}
if (typeof obj[p] == 'object') {
recursion(obj[p], newPath);
}
}
}
}
return [result]
}
function telnyx_contents(json) {
const telnyx = JSON.parse(json.slice(1, -1).replace(/""/g, '"').replace(/(\r\n|\n|\r|\t| )/gm, ""))
let result = [];
recursion(telnyx)
function recursion(obj, path) {
if (path == undefined) { path = 'telnyx' }
const regex = new RegExp('[^0-9]+');
for (let p in obj) {
let newPath = (regex.test(p)) ? path + '.' + p : path + '[' + p + ']';
if (obj[p] != null) {
if (typeof obj[p] != 'object' && typeof obj[p] != 'function') {
result.push(obj[p]);
}
if (typeof obj[p] == 'object') {
recursion(obj[p], newPath);
}
}
}
}
return [result]
}
optional: array-enabled version
To process a larger number of JSON strings in the column A2:A, use this formula:
=telnyx(A2:A)
The telnyx() function uses telnyx_headers() and telnyx_contents() while observing custom function best practices.
/**
* Converts one or more JSON strings to a horizontal table.
* Row 1 will show field names and subsequent rows the
* data in each field.
*
* #param {A2:A} json The JSON strings to convert to a table.
* #customfunction
*/
function telnyx(json) {
'use strict';
if (!Array.isArray(json)) {
json = [json];
}
json = json.flat();
let result = telnyx_headers(json[0]);
json.forEach(j => result = result.concat(j ? telnyx_contents(j) : [null]));
return result;
}

Logstash: Flatten nested JSON, combine fields inside array

I have a JSON looking like this:
{
"foo": {
"bar": {
"type": "someType",
"id": "ga241ghs"
},
"tags": [
{
"#tagId": "123",
"tagAttributes": {
"attr1": "AAA",
"attr2": "111"
}
},
{
"#tagId": "456",
"tagAttributes": {
"attr1": "BBB",
"attr2": "222"
}
}
]
},
"text": "My text"
}
Actually it's not split to multiple lines (just did it to give a better overview), so it's looking like this:
{"foo":{"bar":{"type":"someType","id":"ga241ghs"},"tags":[{"#tagId":"123","tagAttributes":{"attr1":404,"attr2":416}},{"#tagId":"456","tagAttributes":{"attr1":1096,"attr2":1103}}]},"text":"My text"}
I want to insert this JSON with Logstash to an Elasticsearch index. However, I want to insert a flattened JSON with the fields in the array combined like this:
"foo.bar.tags.tagId": ["123", "456"]
"foo.tags.tagAttributs.attr1": ["AAA", "BBB"]
"foo.tags.tagAttributs.attr2": ["111", "222"]
In total, the data inserted to Elasticsearch should look like this:
"foo.bar.type": "someType"
"foo.bar.id": "ga241ghs"
"foo.tags.tagId": ["123", "456"]
"foo.tags.tagAttributs.attr1": ["AAA", "BBB"]
"foo.tags.tagAttributs.attr2": ["111", "222"]
"foo.text": "My text"
This is my current Logstash .conf; I am able to split the "tags" array, but now I am getting 2 entries as a result.
How can I now join all tagIds to one field, attr1 values of the array to one field, and all attr2 values to another?
input {
file {
codec => json
path => ["/path/to/my/data/*.json"]
mode => "read"
file_completed_action => "log"
file_completed_log_path => ["/path/to/my/logfile"]
sincedb_path => "/dev/null"
}
}
filter {
split {
field => "[foo][tags]"
}
}
output {
stdout { codec => rubydebug }
}
Thanks a lot!
Nice example for my JSON iterator IIFE - no need for complex algos, just pick DepthFirst, sligthly modified path (new "raw" version) and that is it.
In case you like this JS answer, mind ticking accept flag under voting buttons.
In case you want different language, have also C# parser with similar iterators on same GitHub.
var src = {"foo":{"bar":{"type":"someType","id":"ga241ghs"},"tags":[{"#tagId":"123","tagAttributes":{"attr1":"AAA","attr2":"111"}},{"#tagId":"456","tagAttributes":{"attr1":"BBB","attr2":"222"}}],"text":"My text"}};
//console.log(JSON.stringify(src, null, 2));
function traverse(it) {
var dest = {};
var i=0;
do {
if (it.Current().HasStringValue()) {
var pathKey = it.Path(true).join('.');
var check = dest[pathKey];
if (check) {
if (!(check instanceof Array)) dest[pathKey] = [check];
dest[pathKey].push(it.Value());
} else {
dest[pathKey] = it.Value();
}
}
//console.log(it.Level + '\t' + it.Path(1).join('.') + '\t' + it.KeyDots(), (it.Value() instanceof Object) ? "-" : it.Value());
} while (it.DepthFirst());
console.log(JSON.stringify(dest, null, 2));
return dest;
}
/*
* https://github.com/eltomjan/ETEhomeTools/blob/master/HTM_HTA/JSON_Iterator_IIFE.js
* +new raw Path feature
*/
'use strict';
var JNode = (function (jsNode) {
function JNode(_parent, _pred, _key, _value) {
this.parent = _parent;
this.pred = _pred;
this.node = null;
this.next = null;
this.key = _key;
this.value = _value;
}
JNode.prototype.HasOwnKey = function () { return this.key && (typeof this.key != "number"); }
JNode.prototype.HasStringValue = function () { return !(this.value instanceof Object); }
return JNode;
})();
var JIterator = (function (json) {
var root, current, maxLevel = -1;
function JIterator(json, parent) {
if (parent === undefined) parent = null;
var pred = null, localCurrent;
for (var child in json) {
var obj = json[child] instanceof Object;
if (json instanceof Array) child = parseInt(child); // non-associative array
if (!root) root = localCurrent = new JNode(parent, null, child, json[child]);
else {
localCurrent = new JNode(parent, pred, child, obj ? ((json[child] instanceof Array) ? [] : {}) : json[child]);
}
if (pred) pred.next = localCurrent;
if (parent && parent.node == null) parent.node = localCurrent;
pred = localCurrent;
if (obj) {
var memPred = pred;
JIterator(json[child], pred);
pred = memPred;
}
}
if (this) {
current = root;
this.Level = 0;
}
}
JIterator.prototype.Current = function () { return current; }
JIterator.prototype.SetCurrent = function (newCurrent) {
current = newCurrent;
this.Level = 0;
while(newCurrent = newCurrent.parent) this.Level++;
}
JIterator.prototype.Parent = function () {
var retVal = current.parent;
if (retVal == null) return false;
this.Level--;
return current = retVal;
}
JIterator.prototype.Pred = function () {
var retVal = current.pred;
if (retVal == null) return false;
return current = retVal;
}
JIterator.prototype.Node = function () {
var retVal = current.node;
if (retVal == null) return false;
this.Level++;
return current = retVal;
}
JIterator.prototype.Next = function () {
var retVal = current.next;
if (retVal == null) return false;
return current = retVal;
}
JIterator.prototype.Key = function () { return current.key; }
JIterator.prototype.KeyDots = function () { return (typeof (current.key) == "number") ? "" : (current.key + ':'); }
JIterator.prototype.Value = function () { return current.value; }
JIterator.prototype.Reset = function () {
current = root;
this.Level = 0;
}
JIterator.prototype.RawPath = function () {
var steps = [], level = current;
do {
if (level != null && level.value instanceof Object) {
steps.push(level.key + (level.value instanceof Array ? "[]" : "{}"));
} else {
if (level != null) steps.push(level.key);
else break;
}
level = level.parent;
} while (level != null);
var retVal = "";
retVal = steps.reverse();
return retVal;
}
JIterator.prototype.Path = function (raw) {
var steps = [], level = current;
do {
if (level != null && level.value instanceof Object) {
var size = 0;
var items = level.node;
if (typeof (level.key) == "number" && !raw) steps.push('[' + level.key + ']');
else {
if(raw) {
if (typeof (level.key) != "number") steps.push(level.key);
} else {
while (items) {
size++;
items = items.next;
}
var type = (level.value instanceof Array ? "[]" : "{}");
var prev = steps[steps.length - 1];
if (prev && prev[0] == '[') {
var last = prev.length - 1;
if (prev[last] == ']') {
last--;
if (!isNaN(prev.substr(1, last))) {
steps.pop();
size += '.' + prev.substr(1, last);
}
}
}
steps.push(level.key + type[0] + size + type[1]);
}
}
} else {
if (level != null) {
if (typeof (level.key) == "number") steps.push('[' + level.key + ']');
else steps.push(level.key);
}
else break;
}
level = level.parent;
} while (level != null);
var retVal = "";
retVal = steps.reverse();
return retVal;
}
JIterator.prototype.DepthFirst = function () {
if (current == null) return 0; // exit sign
if (current.node != null) {
current = current.node;
this.Level++;
if (maxLevel < this.Level) maxLevel = this.Level;
return 1; // moved down
} else if (current.next != null) {
current = current.next;
return 2; // moved right
} else {
while (current != null) {
if (current.next != null) {
current = current.next;
return 3; // returned up & moved next
}
this.Level--;
current = current.parent;
}
}
return 0; // exit sign
}
JIterator.prototype.BreadthFirst = function () {
if (current == null) return 0; // exit sign
if (current.next) {
current = current.next;
return 1; // moved right
} else if (current.parent) {
var level = this.Level, point = current;
while (this.DepthFirst() && level != this.Level);
if (current) return 2; // returned up & moved next
do {
this.Reset();
level++;
while (this.DepthFirst() && level != this.Level);
if (current) return 3; // returned up & moved next
} while (maxLevel >= level);
return current != null ? 3 : 0;
} else if (current.node) {
current = current.node;
return 3;
} else if (current.pred) {
while (current.pred) current = current.pred;
while (current && !current.node) current = current.next;
if (!current) return null;
else return this.DepthFirst();
}
}
JIterator.prototype.ReadArray = function () {
var retVal = {};
var item = current;
do {
if (item.value instanceof Object) {
if (item.value.length == 0) retVal[item.key] = item.node;
else retVal[item.key] = item;
} else retVal[item.key] = item.value;
item = item.next;
} while (item != null);
return retVal;
}
JIterator.prototype.FindKey = function (key) {
var pos = current;
while (current && current.key != key) this.DepthFirst();
if (current.key == key) {
var retVal = current;
current = pos;
return retVal;
} else {
current = pos;
return null;
}
}
return JIterator;
})();
traverse(new JIterator(src));
Your short JSON version was different, now using this one, which looks like your required results (attrs changed and text moved from root under foo):
{
"foo": {
"bar": {
"type": "someType",
"id": "ga241ghs"
},
"tags": [
{
"#tagId": "123",
"tagAttributes": {
"attr1": "AAA",
"attr2": "111"
}
},
{
"#tagId": "456",
"tagAttributes": {
"attr1": "BBB",
"attr2": "222"
}
}
],
"text": "My text"
}
}
Figured it out how to do it with a Ruby filter directly in Logstash - for all searching for this in future, here is one example on how to do it for #tagId:
filter {
ruby { code => '
i = 0
tagId_array = Array.new
while i < event.get( "[foo][tags]" ).length do
tagId_array = tagId_array.push(event.get( "[foo][tags][" + i.to_s + "][#tagId]" ))
i += 1
end
event.set( "foo.tags.tagId", tagId_array )
'
}
}

How to convert Firestore REST API response to normal json?

I am trying to use Firestore from Unity by using REST API of it. Everything is working as expected till now.
When reading document from Firestore, it returns json in different format.
Like this.
{
"name": "projects/firestore-unity-demo-87998/databases/(default)/documents/test/panara",
"fields": {
"health": {
"integerValue": "1008"
},
"name": {
"stringValue": "Bhavin Panara"
},
"birthday": {
"timestampValue": "1992-10-08T04:40:10Z"
},
"alive": {
"booleanValue": true
},
"floatingPointNumber": {
"testFloat": 100.1
}
},
"createTime": "2019-07-30T13:27:09.599079Z",
"updateTime": "2019-07-31T11:41:10.637712Z"
}
How can I convert this kind of json to a normal json like this.
{
"health":1008,
"name":"Bhavin Panara",
"birthday" : "1992-10-08T04:40:10Z",
"alive":true,
"floatingPointNumber":100.1
}
I used the code from firestore-parser
const getFireStoreProp = (value) => {
const props = {
arrayValue: 1,
bytesValue: 1,
booleanValue: 1,
doubleValue: 1,
geoPointValue: 1,
integerValue: 1,
mapValue: 1,
nullValue: 1,
referenceValue: 1,
stringValue: 1,
timestampValue: 1,
};
return Object.keys(value).find(k => props[k] === 1);
};
export const fireStoreParser = (value) => {
let newVal = value;
// You can use this part to avoid mutating original values
// let newVal;
// if (typeof value === 'object') {
// newVal = { ...value };
// } else if (value instanceof Array) {
// newVal = value.slice(0);
// } else {
// newVal = value;
// }
const prop = getFireStoreProp(newVal);
if (prop === 'doubleValue' || prop === 'integerValue') {
newVal = Number(newVal[prop]);
} else if (prop === 'arrayValue') {
newVal = ((newVal[prop] && newVal[prop].values) || []).map(v => fireStoreParser(v));
} else if (prop === 'mapValue') {
newVal = fireStoreParser((newVal[prop] && newVal[prop].fields) || {});
} else if (prop === 'geoPointValue') {
newVal = { latitude: 0, longitude: 0, ...newVal[prop] };
} else if (prop) {
newVal = newVal[prop];
} else if (typeof newVal === 'object') {
Object.keys(newVal).forEach((k) => { newVal[k] = fireStoreParser(newVal[k]); });
}
return newVal;
};
function toValue(field) {
return "integerValue" in field
? Number(field.integerValue)
: "doubleValue" in field
? Number(field.doubleValue)
: "arrayValue" in field
? field.arrayValue.values.map(toValue)
: "mapValue" in field
? toJSON(field.mapValue)
: Object.entries(field)[0][1];
}
function toJSON(doc) {
return Object.fromEntries(
Object.entries(doc.fields ?? {}).map(([key, field]) => [key, toValue(field)])
);
}

TypeScript Call Signature Error Only When Setting Value Twice

I am trying to change the value of an HTML input based upon if a checkbox is checked. It works totally fine if I only reset one input, but if I try to reset two inputs at the same time I get the error
Cannot invoke an expression whose type lacks a call signature. Type 'String' has no compatible call signatures.
My code is as follows
if((<HTMLInputElement>unavailableInputs[i]).checked){
(<HTMLInputElement>qtyShippedInputs[i]).value = ""
(<HTMLInputElement>trackingNumberInputs[i]).value = ""
}
If I only reset one of the values, regardless of which one, it throws no errors and works totally fine. As soon as I do both qtyShipped and trackingNumber I get the error. In addition in the browser I get the error
"" is not a function
I appreciate any and all help.
As requested here is the whole code with a work around that solves the problem
disableFulfillment(shipment) {
const unavailableInputs = document.getElementsByClassName('unavailable-qty')
const qtyShippedInputs = document.getElementsByClassName('qty-shipped')
const requestedQtyFields = document.getElementsByClassName('requested-qty')
const trackingNumberInputs = document.getElementsByClassName('tracking-number')
const reasonCodeInputs = document.getElementsByClassName('reason-codes')
const reasonCodeValues = []
const lines = []
const lineCompleted = []
let i
for (i = 0; i < trackingNumberInputs.length; i++) {
if ((<HTMLInputElement>unavailableInputs[i]).checked) {
(<HTMLInputElement>qtyShippedInputs[i]).value = ''
}
if ((<HTMLInputElement>unavailableInputs[i]).checked) {
(<HTMLInputElement>trackingNumberInputs[i]).value = ''
}
if (!(<HTMLInputElement>unavailableInputs[i]).checked) {
if ((<HTMLInputElement>unavailableInputs[i]).nextElementSibling) {
(<HTMLInputElement>unavailableInputs[i].nextElementSibling.firstChild).value = ''
reasonCodeValues[i] = (<HTMLInputElement>reasonCodeInputs[i]).value
} else {
reasonCodeValues[i] = 0
}
} else {
if ((<HTMLInputElement>unavailableInputs[i]).nextElementSibling.firstElementChild) {
reasonCodeValues[i] = (<HTMLInputElement>unavailableInputs[i].nextElementSibling.firstElementChild).value
} else {
reasonCodeValues[i] = 0
}
}
const inputs = {
unavailable: (<HTMLInputElement>unavailableInputs[i]).checked,
qtyShipped: (<HTMLInputElement>qtyShippedInputs[i]).value,
requestedQty: (<HTMLInputElement>requestedQtyFields[i]).innerText,
reasonCodeInputs: reasonCodeValues[i],
trackingNumber: (<HTMLInputElement>trackingNumberInputs[i]).value
}
lines.push(inputs)
}
lines.forEach(line => {
if (line.unavailable === true && (line.reasonCodeInputs === 'CUSTOMERCANCEL' || line.reasonCodeInputs === 'UNAVAILABLE')) {
lineCompleted.push(true)
} else if (line.qtyShipped === line.requestedQty && line.trackingNumber.length >= 9) {
lineCompleted.push(true)
} else {
lineCompleted.push(false)
}
})
return !lineCompleted.every(function(e) {
return e === true
})
}
You will see the only change is that it is broken into two identical if statements. Which works no problem.
Here is the same code with the two setters placed inside the same if statement which breaks.
disableFulfillment(shipment) {
const unavailableInputs = document.getElementsByClassName('unavailable-qty')
const qtyShippedInputs = document.getElementsByClassName('qty-shipped')
const requestedQtyFields = document.getElementsByClassName('requested-qty')
const trackingNumberInputs = document.getElementsByClassName('tracking-number')
const reasonCodeInputs = document.getElementsByClassName('reason-codes')
const reasonCodeValues = []
const lines = []
const lineCompleted = []
let i
for (i = 0; i < trackingNumberInputs.length; i++) {
if ((<HTMLInputElement>unavailableInputs[i]).checked) {
(<HTMLInputElement>qtyShippedInputs[i]).value = ''
(<HTMLInputElement>trackingNumberInputs[i]).value = ''
}
if (!(<HTMLInputElement>unavailableInputs[i]).checked) {
if ((<HTMLInputElement>unavailableInputs[i]).nextElementSibling) {
(<HTMLInputElement>unavailableInputs[i].nextElementSibling.firstChild).value = ''
reasonCodeValues[i] = (<HTMLInputElement>reasonCodeInputs[i]).value
} else {
reasonCodeValues[i] = 0
}
} else {
if ((<HTMLInputElement>unavailableInputs[i]).nextElementSibling.firstElementChild) {
reasonCodeValues[i] = (<HTMLInputElement>unavailableInputs[i].nextElementSibling.firstElementChild).value
} else {
reasonCodeValues[i] = 0
}
}
const inputs = {
unavailable: (<HTMLInputElement>unavailableInputs[i]).checked,
qtyShipped: (<HTMLInputElement>qtyShippedInputs[i]).value,
requestedQty: (<HTMLInputElement>requestedQtyFields[i]).innerText,
reasonCodeInputs: reasonCodeValues[i],
trackingNumber: (<HTMLInputElement>trackingNumberInputs[i]).value
}
lines.push(inputs)
}
lines.forEach(line => {
if (line.unavailable === true && (line.reasonCodeInputs === 'CUSTOMERCANCEL' || line.reasonCodeInputs === 'UNAVAILABLE')) {
lineCompleted.push(true)
} else if (line.qtyShipped === line.requestedQty && line.trackingNumber.length >= 9) {
lineCompleted.push(true)
} else {
lineCompleted.push(false)
}
})
return !lineCompleted.every(function(e) {
return e === true
})
}