Spreadsheet display JSON data into columns and rows - json

I'd like to handle a bunch of data in Google Spreadsheets so I can easily create charts and all that stuff. The data come in Json format from an URL. I can successfully get those data in a Spreadsheet but I cannot figure out an easy way to distribute those data into columns of the sheet.
This is a sample of the JSON structure:
Is there any easy way to achieve my goal?
Thanks a lot.

How about this sample script? This sample script retrieves all elements in JSON data using the recursive call, and import them to Spreadsheet.
In order to use this script, please set as follows.
1. Use this script as a bound script for Spreadsheet.
1. Define json data in main() or as a global variable.
Sample JSON :
var json = {
keyA: {
ArrayA1: ["ArrayA1_val1", "ArrayA1_val2", "ArrayA1_val3"],
keyA1: "keyA1_val",
keyA2: "keyA2_val",
},
keyB: {
ArrayB1: ["ArrayB1_val1", "ArrayB1_val2", "ArrayB1_val3"],
keyB1: "keyA1_val",
keyB2: "keyA2_val",
keyB3: {
ArrayB31: ["ArrayB31_val1", "ArrayB31_val2", "ArrayB31_val3"],
ArrayB32: ["ArrayB32_val1", "ArrayB32_val2", "ArrayB32_val3"],
}
}
};
Sample Script :
function getElements(v, callback) {
for (var k in v) {
callback(
k,
Object.prototype.toString.call(v[k]).slice(8, -1).toLowerCase() === "object"
? JSON.stringify(v[k], null, "\t") : v[k],
Object.prototype.toString.call(v[k]).slice(8, -1).toLowerCase()
);
if (typeof v[k] === "object") {
getElements(v[k], callback);
}
}
}
function main() {
var ar = [];
getElements(json, function(key, value, type) {ar.push([key, value, type])});
var ss = SpreadsheetApp.getActiveSheet();
ss.getRange(ss.getLastRow() + 1, 1, ar.length, ar[0].length).setValues(ar);
}
Result :
If I misunderstand your question, I'm sorry.

Related

FetchingURL Using JSON on Google Sheets v2

I have a code in Google Apps Scripts that takes a Spotify URL and pastes the amount of followers in the cell where you reference the url. =SAMPLE(A2) (where A2 holds the URL)
It seems that the specification has changed again. And I'm not sure how to fix it. Any guidance or help would be greatly appreciated!
Here's the original post = FetchingURL Using JSON on Google Sheets
Here's the code I have currently in Google Apps Scripts. (courtesy of #Tanaike)
function SAMPLE(url) {
const res = UrlFetchApp.fetch(url).getContentText();
const v = res.match(/"followers":({[\s\S\w]+?})/);
return v && v.length == 2 ? JSON.parse(v[1].trim()).total : "Value cannot be
retrieved.";
}
Thanks!
In the current stage, it seems that the JSON data is put as base64 data, and also, the structure of the object was changed. So, as the current sample script, how about the following sample script?
Sample script:
function SAMPLE(url) {
const res = UrlFetchApp.fetch(url).getContentText();
const v = res.match(/<script id\="initial-state" type\="text\/plain">([\s\S\w]+?)<\//);
if (!v || v.length != 2) return "Value cannot be retrieved.";
const obj = JSON.parse(Utilities.newBlob(Utilities.base64Decode(v[1].trim())).getDataAsString());
const value = Object.entries(obj.entities.items).reduce((n, [k, o]) => {
if (k.includes("spotify:playlist:")) n.push(o.followers.total);
return n;
}, []);
return value.length > 0 ? value[0] : "Value cannot be retrieved.";
}
Note:
This sample script is for the current HTML data. So, I think that this might be changed in the future update. So, I also would like to recommend using the API for your future work as it has already been mentioned in the comment.

Extract a JSON DATA table in html using VBA; converting Apps Script into VBA

I want to retrieve a table from the URL of https://s.cafef.vn/screener.aspx#data using VBA. This task is difficult because the table contains JSON data embedded in an html file, but it was so kind of Tanaike, an GAS expert who helped me to create a custom function for Google Apps Scripts.
(IMPORTHTML() doesn't work in this webpage structure)
The function looks like:
function SAMPLE(url) {
const res = UrlFetchApp.fetch(url, {muteHttpExceptions: true});
const html = res.getContentText().match(/var jsonData \=([\S\s\w]+\}\])/);
if (!html) return "No tables. Please confirm URL again.";
const table = JSON.parse(html[1].replace(/\n/g, ""));
const header = ["", "FullName", "Symbol", "CenterName", "ChangePrice", "VonHoa", "ChangeVolume", "EPS", "PE", "Beta", "Price"];
return table.reduce((ar, e, i) => {
const temp = header.map(f => f == "" ? i + 1 : e[f]);
ar.push(temp);
return ar;
}, [header]); }
This function works perfect in Google Sheets environment, but my goal now is to convert it into VBA, or in other words, writing a VBA module which can get the table at https://s.cafef.vn/screener.aspx#data.
Many thanks for any help or suggestions
Cao Doremi

Google Sheets Scraping Options Chain from Yahoo Finance, Incomplete Results [duplicate]

This question already has answers here:
Scraping data to Google Sheets from a website that uses JavaScript
(2 answers)
Closed last month.
I'm attempting to scrape options pricing data from Yahoo Finance in Google Sheets. Although I'm able to pull the options chain just fine, i.e.
=IMPORTHTML("https://finance.yahoo.com/quote/TCOM/options?date=1610668800","table",2)
I find that it's returning results that don't completely match what's actually shown on Yahoo Finance. Specifically, the scraped results are incomplete - they're missing some strikes. i.e. the first 5 rows of the chart may match, but then it will start returning only every other strike (aka skipping every other strike).
Why would IMPORTHTML be returning "abbreviated" results, which don't match what's actually shown on the page? And more importantly, is there some way to scrape complete data (i.e. that doesn't skip some portion of the available strikes)?
In Yahoo finance, all data are available in a big json called root.App.main. So to get the complete set of data, proceed as following
var source = UrlFetchApp.fetch(url).getContentText()
var jsonString = source.match(/(?<=root.App.main = ).*(?=}}}})/g) + '}}}}'
var data = JSON.parse(jsonString)
You can then choose to fetch the informations you need. Take a copy of this example https://docs.google.com/spreadsheets/d/1sTA71PhpxI_QdGKXVAtb0Rc3cmvPLgzvXKXXTmiec7k/copy
edit
if you want to get a full list of available data, you can retrieve it by this simple script
// mike.steelson
let result = [];
function getAllDataJSON(url = 'https://finance.yahoo.com/quote/TCOM/options?date=1610668800') {
var source = UrlFetchApp.fetch(url).getContentText()
var jsonString = source.match(/(?<=root.App.main = ).*(?=}}}})/g) + '}}}}'
var data = JSON.parse(jsonString)
getAllData(eval(data),'data')
var sh = SpreadsheetApp.getActiveSpreadsheet().getActiveSheet()
sh.getRange(1, 1, result.length, result[0].length).setValues(result);
}
function getAllData(obj,id) {
const regex = new RegExp('[^0-9]+');
for (let p in obj) {
var newid = (regex.test(p)) ? id + '["' + p + '"]' : id + '[' + p + ']';
if (obj[p]!=null){
if (typeof obj[p] != 'object' && typeof obj[p] != 'function'){
result.push([newid, obj[p]]);
}
if (typeof obj[p] == 'object') {
getAllData(obj[p], newid );
}
}
}
}
Here's a simpler way to get the last market price of a given option. Add this function to you Google Sheets Script Editor.
function OPTION(ticker) {
var ticker = ticker+"";
var URL = "finance.yahoo.com/quote/"+ticker;
var html = UrlFetchApp.fetch(URL).getContentText();
var count = (html.match(/regularMarketPrice/g) || []).length;
var query = "regularMarketPrice";
var loc = 0;
var n = parseInt(count)-2;
for(i = 0; i<n; i++) {
loc = html.indexOf(query,loc+1);
}
var value = html.substring(loc+query.length+9, html.indexOf(",", loc+query.length+9));
return value*100;
}
In your google sheets input the Yahoo Finance option ticker like below
=OPTION("AAPL210430C00060000")
I believe your goal as follows.
You want to retrieve the complete table from the URL of https://finance.yahoo.com/quote/TCOM/options?date=1610668800, and want to put it to the Spreadsheet.
Issue and workaround:
I could replicate your issue. When I saw the HTML data, unfortunately, I couldn't find the difference of HTML between the showing rows and the not showing rows. And also, I could confirm that the complete table is included in the HTML data. By the way, when I tested it using =IMPORTXML(A1,"//section[2]//tr"), the same result of IMPORTHTML occurs. So I thought that in this case, IMPORTHTML and IMPORTXML might not be able to retrieve the complete table.
So, in this answer, as a workaround, I would like to propose to put the complete table parsed using Sheets API. In this case, Google Apps Script is used. By this, I could confirm that the complete table can be retrieved by parsing the HTML table with Sheet API.
Sample script:
Please copy and paste the following script to the script editor of Spreadsheet, and please enable Sheets API at Advanced Google services. And, please run the function of myFunction at the script editor. By this, the retrieved table is put to the sheet of sheetName.
function myFunction() {
// Please set the following variables.
const url ="https://finance.yahoo.com/quote/TCOM/options?date=1610668800";
const sheetName = "Sheet1"; // Please set the destination sheet name.
const sessionNumber = 2; // Please set the number of session. In this case, the table of 2nd session is retrieved.
const html = UrlFetchApp.fetch(url).getContentText();
const section = [...html.matchAll(/<section[\s\S\w]+?<\/section>/g)];
if (section.length >= sessionNumber) {
if (section[sessionNumber].length == 1) {
const table = section[sessionNumber][0].match(/<table[\s\S\w]+?<\/table>/);
if (table) {
const ss = SpreadsheetApp.getActiveSpreadsheet();
const body = {requests: [{pasteData: {html: true, data: table[0], coordinate: {sheetId: ss.getSheetByName(sheetName).getSheetId()}}}]};
Sheets.Spreadsheets.batchUpdate(body, ss.getId());
}
} else {
throw new Error("No table.");
}
} else {
throw new Error("No table.");
}
}
const sessionNumber = 2; means that 2 of =IMPORTHTML("https://finance.yahoo.com/quote/TCOM/options?date=1610668800","table",2).
References:
Method: spreadsheets.batchUpdate
PasteDataRequest

gsheet importjson query for parsing JSONSchema

Using ImportJSON to parse JSONSchema documents and load into GSheet.
I have JSON documents with paths as in the snip below.
I want to output the names of properties in one column and the type in another.
Wanted to see if someone has done this already before i start hacking about with parseJSON or the defaultTransform functions of ImportJSON.
Added example GSheet here
Shows source, currently parsed output and what i need in terms of required output
/data/schema/properties/plan_id/type
/data/schema/properties/plan_id/maxLength
/data/schema/properties/plan_name/type
/data/schema/properties/plan_name/maxLength
/data/schema/properties/type/type
/data/schema/properties/type/maxLength
/data/schema/properties/quantity_ranges/type
/data/schema/properties/quantity_ranges/maximum
/data/schema/properties/quantity_ranges/minimum
/data/schema/properties/pricing_option/type
/data/schema/properties/pricing_option/maxLength
/data/schema/properties/currency/type
/data/schema/properties/currency/enum
/data/schema/properties/value/type
/data/schema/properties/value/maximum
/data/schema/properties/value/minimum
Thanks in advance!
You want to achieve the following situation.
From
To
You want to achieve this using Google Apps Script.
I understood like above. If my understanding is correct, how about this answer? Please think of this as just one of several possible answers.
Sample script:
When you use this sample script, please put =parseObject("SourceJSON!A1") to a cell in your shared Spreadsheet.
function parseObject(range) {
var range = SpreadsheetApp.getActiveSpreadsheet().getRange(range);
var value = range.getValue();
var object = JSON.parse(value);
var res = [];
var headers = ["type", ["maxLength", "maximum"], "minimum", "enum"];
// var headers = ["type", "maxLength", "maximum", "minimum", "enum"];
for (var i in object.data.schema.properties) {
var obj = object.data.schema.properties[i];
for (var j = 0; j < headers.length; j++) {
var temp = [object.data.id, object.data.version];
if (Array.isArray(headers[j])) {
for (var k = 0; k < headers[j].length; k++) {
if (obj[headers[j][k]]) res.push(temp.concat([i, "",obj[headers[j][k]],"",""]));
}
} else {
if (obj[headers[j]]) {
var ar = [i, "","","",""];
ar.splice(j + 1, 1, Array.isArray(obj[headers[j]]) ? obj[headers[j]].join(",") : obj[headers[j]]);
res.push(temp.concat(ar));
}
}
}
}
return res;
}
Result:
Note:
This sample script retrieves the data from the Spreadsheet.
In your DesiredOutput, the values of "maxLength" and "maximum" in the data are put to the same column. At above sample script, the result is the same with it. If you want to separate the values of "maxLength" and "maximum", please modify var headers = ["type", ["maxLength", "maximum"], "minimum", "enum"]; to var headers = ["type", "maxLength", "maximum", "minimum", "enum"];.
This sample script is for the value in your shared Spreadsheet. So when you use this for the data with other structure, an error might occur and/or the result you don't want might be returned. Please be careful this.

Remove all grouped rows / columns in a spreadsheet

I have a Google Sheet that does dynamic grouping with a script. I am looking for a function that gets rid of all those Groups in the sheet again.
Similar to expandAllColumnGroups, I would like to have a function called removeAllColumnGroups - but it seems there is no such function available.
My current approach is very slow and cumbersome. I did quite some research but could not even find a way to get all the columnGroups or at least the start-column-IDs in a sheet, so I iterate over every column and literally try to remove the group if there is one, as there is no way to tell if a group exits. Unfortunately for about 90 columns this takes ages (minutes, really)...
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sh = ss.getActiveSheet();
//remove all column-groups
var group = null;
var maxCol = sh.getMaxColumns();
for(var c = 1; c <= maxCol; c++) {
try {
group = sh.getColumnGroup(c, 1); // Try to get group
group.remove(); // Remove the group
} catch (e) {
//Logger.log("No Group at ColIndex "+c);
}
}
Any ideas would be really appreciated. Thanks in advance!
Expanding on my comment re: use of the Google Sheets REST API to access and modify row/column groupings:
Row/Column groups are metadata associated with a Sheet resource, and as such can be obtained for all sheets in a workbook with a single HTTP request to spreadsheets.get, with the appropriate fields specification:
GET https://sheets.googleapis.com/v4/spreadsheets/{YOUR_SPREADSHEET_ID}?fields=sheets(columnGroups%2Cproperties(sheetId%2Ctitle)%2CrowGroups)&key={YOUR_API_KEY}
Demo link
The above request returns an object with a sheets property, which is an array of objects (1 per sheet in the spreadsheet) having 3 properties: rowGroups, columnGroups, and properties. The group properties are arrays of DimensionGroup objects, while the properties object contains the sheet's gridId/sheetId, which you need for identifying the sheet in various API requests, and its name (which may be useful for your own script logic).
To delete every row/column group, you need to issue as many DeleteDimensionGroupRequests to do so as the maximum depth returned in your query's groups. If you do not specify the indices of the DimensionRange in your request, this is interpreted as the whole range of the spreadsheet (all rows / all columns, depending on direction).
An example request (requires OAuth authentication, not just API key):
POST https://sheets.googleapis.com/v4/spreadsheets/{YOUR SPREADSHEET ID}:batchUpdate?fields=replies%2FdeleteDimensionGroup
{
"requests": [
{
"deleteDimensionGroup": {
"range": {
"sheetId": "{SHEET 1 ID}",
"dimension": "COLUMNS"
}
}
},
{
"deleteDimensionGroup": {
"range": {
"sheetId": "{SHEET 2 ID}"
"dimension": "COLUMNS",
}
}
},
...
]
}
Demo link
Each delete request has a reply response, and that response will be very similar to the initial response you got for the row/column groups from the initial query. If you knew the gridIds beforehand, you could forgo the initial query and use a while loop to keep sending delete requests while the response contains a dimension group.
To use these methods with Google Apps Script, you can either use UrlFetchApp with raw URL resources, or take advantage of the available "advanced service" client library Sheets (which must first be enabled). Both methods require you to enable use of the Sheets API from your script's Google Cloud Platform project page.
An example using the enabled client library Sheets:
function removeAllGroups() {
const wb = SpreadsheetApp.getActive(),
wbId = wb.getId();
const initial = Sheets.Spreadsheets.get(wbId, {
fields: "sheets(columnGroups,properties(sheetId,title),rowGroups)"
});
// Determine the maximum depth of row & column groups on each sheet in the workbook.
const maxDepths = {row: {}, col: {}};
initial.sheets.forEach(function (s) {
// if (s.properties.title ... (could write logic to do this only for certain sheets)
var sId = s.properties.sheetId;
if (s.columnGroups && s.columnGroups.length)
maxDepths.col[sId] = s.columnGroups.reduce(dgMaxDepth_, 0);
if (s.rowGroups && s.rowGroups.length)
maxDepths.row[sId] = s.rowGroups.reduce(dgMaxDepth_, 0);
});
// Add all delete requests to an array
const rqs = [];
for (var rqType in maxDepths) {
for (var sheetId in maxDepths[rqType]) {
addDeleteDGRequests_(rqs, rqType, sheetId, maxDepths[rqType][sheetId]);
}
}
// Send all requests.
if (rqs.length) {
const replies = Sheets.Spreadsheets.batchUpdate({requests: rqs}, wbId);
console.log({message: "Batch response", response: replies});
}
}
// Callback for Array#reduce
function dgMaxDepth_(val, dg, i, allDGs) {
return Math.max(val, dg.depth);
}
function addDeleteDGRequests_(requests, rqType, sheetId, num) {
const dim = rqType === "col" ? "COLUMNS" : "ROWS";
while (num > 0) {
var rq = {
deleteDimensionGroup: {
range: { sheetId: sheetId,
dimension: dim }
}
};
requests.push(rq);
--num;
}
}
Resources:
Google APIs Explorer - Sheets API
Google Sheets REST API
Enable Advanced Services
Array#reduce
Array#forEach
Here's a simple solution.
function removeAllGroupsFromSheet() {
let sheet = SpreadsheetApp.getActiveSheet();
let lastRow = sheet.getDataRange().getLastRow();
for (let row = 1; row < lastRow; row++) {
let depth = sheet.getRowGroupDepth(row);
if (depth < 1) continue;
sheet.getRowGroup(row, depth).remove();
}
}