Importing Importing API data via importJSON - json

This is my code:
function pricesV2(){
var url='https://prices.runescape.wiki/api/v1/osrs/mapping'
var data = JSON.parse(UrlFetchApp.fetch(url).getContentText())
let myItems = new Map()
let myItem = new Map1()
json=eval('data')
json.forEach(function(elem){myItems.set(elem.id.toString(),elem.name)})
json.forEach(function(elem){myItem.set(elem.id.toString(),elem.examine)})
var url='https://prices.runescape.wiki/api/v1/osrs/latest'
var data = JSON.parse(UrlFetchApp.fetch(url).getContentText())
var result = []
result.push(['#','name','examine','high','low','lowTime', 'highTime'])
for (let p in eval('data.data')) {
try{result.push([p,myItems.get(p),myItem.get(p),data.data.item(p).high,data.data.item(p).low,convertTimestamp(data.data.item(p).lowTime),convertTimestamp(data.data.item(p).highTime)])}catch(e){}
}
return result
}
This is maybe important to know the variables of the API:
function prices(url){
//var url='https://prices.runescape.wiki/api/v1/osrs/latest'
var data = JSON.parse(UrlFetchApp.fetch(url).getContentText())
var result = []
result.push(['#','high','low','highTime','lowTime'])
for (let p in eval('data.data')) {
try{result.push([p,data.data.item(p).high,data.data.item(p).low,data.data.item(p).lowTime, ,data.data.item(p).highTime])}catch(e){}
}
return result
}
function naming(url){
//var url='https://prices.runescape.wiki/api/v1/osrs/mapping'
var data = JSON.parse(UrlFetchApp.fetch(url).getContentText())
var result = []
result.push(["id","name","examine","members","lowalch","limit","value","highalch"])
json=eval('data')
json.forEach(function(elem){
result.push([elem.id.toString(),elem.name,elem.examine,elem.members,elem.lowalch,elem.limit,elem.value,elem.highalch])
})
return result
}
These are 2 API combined (Importing API data via importJSON, solution did work out for 1 element, (element.name)). But when I want to add more from mapping it is giving an error. Could someone help me out? I want to combine all results in one table.

I believe your goal is as follows.
You want to integrate 2 returned data (JSON data) with the value of id.
From your reply of The colums doesn't need in this specific order., you are not required to check the order of columns.
You want to run the script as a custom function.
From your showing script, I thought that you might have wanted to use this script as a custom function.
In this case, how about the following sample script?
Sample script:
Please copy and paste the following script to the script editor of Spreadsheet. And, please put a custom function =SAMPLE() to a cell. By this, the script is run.
function SAMPLE() {
const url1 = "https://prices.runescape.wiki/api/v1/osrs/mapping";
const url2 = "https://prices.runescape.wiki/api/v1/osrs/latest";
const [res1, res2] = [url1, url2].map(url => JSON.parse(UrlFetchApp.fetch(url).getContentText()));
const head = [...Object.keys(res1[0]), ...Object.keys(res2.data[Object.keys(res2.data)[0]])];
const obj1 = res1.reduce((o, e) => (o[e.id] = e, o), {});
const obj2 = Object.entries(res2.data).reduce((o, [k, v]) => (o[k] = v, o), {});
const keys = Object.keys(obj1).map(e => Number(e)).sort((a, b) => a - b);
const values = [head, ...keys.map(k => {
const o = Object.assign(obj1[k], obj2[k]);
return head.map(h => o[h] || "");
})];
return values;
}
Testing:
When this script is run, the following result is obtained.
Note:
If you want to set the specific order of the columns, please modify head in the above script.
When the custom function of =SAMPLE() is put to a cell, if an error occurs, please reopen Spreadsheet and test it again.
If you want to directly put the values to the Spreadsheet instead of the custom function, please modify the script.
References:
Custom Functions in Google Sheets
map()
reduce()
Added:
From the following 3 new questions,
Now how can I change like the top row to- > id, name, examine, members, lowalch, highalch, limit, high, low, lowtime, hightime? How can this be done in the function head, can't edit them individualy?
And also how can I format/convert highTime and lowTime to time (hh:mm:ss)?
From The colums doesn't need in this specific order., I didn't check the order of the column. In that case, as I have already mentioned in my answer, please modify head as follows. About your 2nd new question, in this case, please parse the unix time as follows.
So, when these new 2 questions are reflected in my sample script, it becomes as follows.
Sample script:
function SAMPLE() {
const url1 = "https://prices.runescape.wiki/api/v1/osrs/mapping";
const url2 = "https://prices.runescape.wiki/api/v1/osrs/latest";
const [res1, res2] = [url1, url2].map(url => JSON.parse(UrlFetchApp.fetch(url).getContentText()));
const head = ['id', 'name', 'examine', 'members', 'lowalch', 'highalch', 'limit', 'high', 'low', 'lowTime', 'highTime'];
const obj1 = res1.reduce((o, e) => (o[e.id] = e, o), {});
const obj2 = Object.entries(res2.data).reduce((o, [k, v]) => (o[k] = v, o), {});
const keys = Object.keys(obj1).map(e => Number(e)).sort((a, b) => a - b);
const timeZone = Session.getScriptTimeZone();
const values = [head, ...keys.map(k => {
const o = Object.assign(obj1[k], obj2[k]);
return head.map(h => o[h] ? (['lowTime', 'highTime'].includes(h) ? Utilities.formatDate(new Date(o[h] * 1000), timeZone, "HH:mm:ss") : o[h]) : "");
})];
return values;
}
Note:
About your following 3rd question,
How can this database also be added <prices.runescape.wiki/api/v1/osrs/volumes>?
I think that this is a new question. In this case, please post it as a new question.

Related

Error: Malformed UTF-8 data stringify (while scraping yahoo finance historical)

I have a script to scrape Yahoo Historical Data but it looks like the decrypt service stopped working.
function Scrapeyahoo(symbol) {
//modificación del 27/1/23 hecha por Tanaike
// https://stackoverflow.com/questions/75250562/google-apps-script-stopped-scraping-data-from-yahoo-finance/75253348#75253348
const s = encodeURI(symbol);
const url = 'https://finance.yahoo.com/quote/' +s +'/history?p=' +s;
var html = UrlFetchApp.fetch(url).getContentText().match(/root.App.main = ([\s\S\w]+?);\n/);
if (!html || html.length == 1) return;
var obj = JSON.parse(html[1].trim());
var key = [...new Map(Object.entries(obj).filter(([k]) => !["context", "plugins"].includes(k)).splice(-4)).values()].join("");
if (!key) return;
const cdnjs = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js";
eval(UrlFetchApp.fetch(cdnjs).getContentText());
const obj1 = JSON.parse(CryptoJS.enc.Utf8.stringify(CryptoJS.AES.decrypt(obj.context.dispatcher.stores, key)));
const header = ["date", "open", "high", "low", "close", "adjclose", "volume"];
const ar = obj1.HistoricalPriceStore.prices.map(o => header.map(h => h == "date" ? new Date(o[h] * 1000) : (o[h] || "")));
return ar
}
I get the error Malformed UTF-8 data stringify in the line
JSON.parse(CryptoJS.enc.Utf8.stringify(CryptoJS.AES.decrypt(obj.context.dispatcher.stores, key)));
A few weeks ago #Tanaike solved here a similar issue, but it looks like there has been new changes.
I ask for help with this problem.
Thanks in advance.
It seems that the specification for retrieving the key has been changed. In this case, vvar key = [...new Map(Object.entries(obj).filter(([k]) => !["context", "plugins"].includes(k)).splice(-4)).values()].join(""); doesn't return the correct key. And also, it seems that the logic for retrieving the valid key has been changed. But, unfortunately, I cannot still find the correct logic. So, in this answer, I would like to refer to this thread. In this thread, the valid keys are listed in a text file. When this is reflected in your script, it becomes as follows.
Modified script:
function Scrapeyahoo(symbol) {
const s = encodeURI(symbol);
const url = 'https://finance.yahoo.com/quote/' + s + '/history?p=' + s;
var html = UrlFetchApp.fetch(url).getContentText().match(/root.App.main = ([\s\S\w]+?);\n/);
if (!html || html.length == 1) return;
var obj = JSON.parse(html[1].trim());
const cdnjs = "https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js";
eval(UrlFetchApp.fetch(cdnjs).getContentText());
const keyFile = "https://github.com/ranaroussi/yfinance/raw/main/yfinance/scrapers/yahoo-keys.txt";
const res = UrlFetchApp.fetch(keyFile);
const keys = res.getContentText().split("\n").filter(String);
let obj1 = keys.reduce((ar, key) => {
try {
const o = JSON.parse(CryptoJS.enc.Utf8.stringify(CryptoJS.AES.decrypt(obj.context.dispatcher.stores, key.trim())));
ar.push(o);
} catch (e) {
// console.log(e.message)
}
return ar;
}, []);
if (obj1.length == 0) {
throw new Error("Specification at the server side might be changed. Please check it.");
}
obj1 = obj1[0];
const header = ["date", "open", "high", "low", "close", "adjclose", "volume"];
const ar = obj1.HistoricalPriceStore.prices.map(o => header.map(h => h == "date" ? new Date(o[h] * 1000) : (o[h] || "")));
return ar
}
When I tested this script with a sample value of CL=F as symbol, I confirmed that the script worked.
Note:
In this sample, in order to load crypto-js, eval(UrlFetchApp.fetch(cdnjs).getContentText()) is used. But, if you don’t want to use it, you can also use this script by copying and pasting the script of https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js to the script editor of Google Apps Script. By this, the process cost can be reduced.
I can confirm that this method can be used for the current situation (February 15, 2023). But, when the specification in the data and HTML is changed in the future update on the server side, this script might not be able to be used. Please be careful about this.
Reference:
crypto-js

Google Apps script large data set transfer [duplicate]

I have a dataset which contains images in col C loaded via formula =IMAGE("") and the need is to refresh the data and have these formulas load the images at destination.
I tried the Spreadsheet API, but handling the data the way it's needed it still far to me - knowledge wise.
I try with the script below, but the column C shows as blank at destination:
function getOrdersData() {
const srcFile = SpreadsheetApp.openById('XXXXXXXXXXX');
const srcSht = srcFile.getSheetByName('Orders');
let srcData = srcSht.getRange(1, 1, srcSht.getLastRow(),
srcSht.getLastColumn()).getValues();
const orderHeaders = srcData[4]; //Colunm headers are actually in row 05
const imgCol = orderHeaders.indexOf('Image');//Whish is where the formulas loading the imgs are
const imgFormulas = srcSht.getRange(1, imgCol + 1, srcSht.getLastRow(), 1).getFormulas();
srcData.forEach(function (row) {
row.splice(imgCol, 1, imgFormulas);
});
const dstFile = SpreadsheetApp.openById('XXXXXXXXXXXXXXX');
const dstSht = dstFile.getSheetByName('Orders v2');
const dstShtLr = dstSht.getLastRow();
if (dstShtLr > 0) {
dstSht.getRange(1, 1, dstShtLr, dstSht.getLastColumn()).clearContent();
}
dstSht.getRange(1, 1, srcData.length, srcData[0].length).setValues(srcData);
}
What can I try next?
In your script, imgFormulas is a 2-dimensional array. In this case, by srcData.forEach(,,,), srcData is not 2 dimensional array. I thought that this might be the reason for your issue. When your script is modified, how about the following modification?
From:
srcData.forEach(function (row) {
row.splice(imgCol, 1, imgFormulas);
});
To:
srcData.forEach(function (row, i) {
if (i > 4) row.splice(imgCol, 1, imgFormulas[i][0]);
});
if (i > 4) was used for considering Colunm headers are actually in row 05.
Note:
In your situation, when Sheets API is used, the sample script is as follows. In this case, please enable Sheets API at Advanced Google services. When the number of cells are large, this might be useful.
function sample() {
const srcSSId = '###'; // Please set source Spreadsheet ID.
const dstSSId = '###'; // Please set destination Spreadsheet ID.
const srcSheetName = 'Orders';
const dstSheetName = 'Orders v2';
const srcValues = Sheets.Spreadsheets.Values.get(srcSSId, srcSheetName).values;
const srcFormulas = Sheets.Spreadsheets.Values.get(srcSSId, srcSheetName, { valueRenderOption: "FORMULA" }).values;
const data = [{ range: dstSheetName, values: srcValues }, { range: dstSheetName, values: srcFormulas }];
Sheets.Spreadsheets.Values.batchUpdate({ valueInputOption: "USER_ENTERED", data }, dstSSId);
}
References:
Method: spreadsheets.values.get
Method: spreadsheets.values.batchUpdate

Finding ImportJSON Query Path in Google Sheets

Using Google Sheets with the custom IMPORTJSON function, I'm trying to parse the results of this URL so that it returns only the nodes with "highs".
Using the below formula, where cell A1 is the above URL, returns a #REF! error:
=ImportJSON(A1, "/data//highs", "noInherit, noTruncate")
I've also tried other variations of the /data//highs query (which would usually work with the IMPORTXML function) in the formula with the same result.
DESIRED OUTPUT
The query should result in displaying the records under "highs", i.e., both /data/nasdaq/highs and /data/nyse/highs.
The below image is the output for /data/nasdaq/highs. I'm looking to combine that query result with that for /data/nyse/highs in a single call.
Using ImportJSON:
One option could be to just separate both queries by a comma:
=ImportJSON(A1, "/data/nasdaq/highs,/data/nyse/highs", "noInherit, noTruncate")
This has several potential downsides, though: it won't automatically detect any additional ticket that has highs, but just nasdaq and nyse. If you wanted others, you'd have to edit your query.
Also, it will return nasdaq and nyse values in multiple columns, which I assume is not what you want.
Writing an alternative function:
Alternatively, since importJson cannot handle queries like //data/*/highs, I'd suggest writing a different custom function to handle this. To do this, select Extensions > Apps Script and copy the following function (see inline comments):
function altImportJson(url, query) {
const jsondata = UrlFetchApp.fetch(url); // Get data
let object = JSON.parse(jsondata.getContentText());
const levels = query.split("/").filter(String); // Get query levels
for (let i = 0; i < levels.length; i++) { // Iterate through query levels
const current = levels[i];
const last = levels[i-1];
const next = levels[i+1];
if (current != "*" && last != "*") object = object[current]; // Regular level, not "*"
else if (current == "*") { // Handle "*"
object = Object.values(object)
.filter(o => o[next])
.map(o => o[next])
.flat();
}
};
const headers = [...new Set(object.map(item => Object.keys(item)).flat())];
const data = object.map(item => { // Transform object to 2D array
let row = new Array(headers.length);
const entries = Object.entries(item);
entries.forEach(entry => {
const columnIndex = headers.indexOf(entry[0]);
if (columnIndex > -1) row[columnIndex] = entry[1];
});
return row;
});
data.unshift(headers);
return data;
}
After copying this function and saving your script, you can now call it:
=altImportJson(A1, "//data/*/highs")

Failed to use map function in the right way within google apps script

I can't use map function in the right way within google apps script while scraping two fields—movie name and image— from a webpage.
function scrapeMovies() {
const URL = "https://yts.am/browse-movies";
const response = UrlFetchApp.fetch(URL);
const $ = Cheerio.load(response.getContentText());
const container = $(".browse-movie-wrap");
const result = container.map(item => {
const movieName = $(item).find('a.browse-movie-title').text();
const movieImage = $(item).find('img.img-responsive').attr('src');
console.log(movieName,movieImage);
});
}
When I execute the script, all I get is undefined as result.
You can still use map but you need to change the way you access the element.
The reason it is undefined is because you were trying to do a find on the index value. Upon testing, container on each element returns [index, item] instead of [item, index]. Specifying you want the 2nd element will fix the issue.
const result = container.map((index, item) => {
const movieName = $(item).find('a.browse-movie-title').text();
const movieImage = $(item).find('img.img-responsive').attr('src');
console.log(movieName, movieImage);
});
But since you aren't returning anything, just use each as mentioned by Sysix.
Note:
For some reason, execution doesn't end if I return both values into result when using map and trying to log result.
I tested another way to store the data and the script below worked.
var result = [];
container.each((index, item) => {
const movieName = $(item).find('a.browse-movie-title').text();
const movieImage = $(item).find('img.img-responsive').attr('src');
result.push([movieName, movieImage]);
});
console.log(result);

Google form: create two rows with different information

Im trying to create a google form with 2 fields, one for "item" and the other for "quantity"
Since user might need to send miltiple items I want to create 1 form only and sort the information.
my Google form
So far I have managed to add a script that splits the information submitted in "item" into many rows, however, Im not able to do the same with the field "quantity"
I got this information from this post
This is my script:
function onFormSubmit(e) {
var ss = SpreadsheetApp.openByUrl("URL_here");
var sheet = ss.getSheetByName("FormResponses");
// Form Response retrieved from the event object
const formResponse = e.response;
var itemResponses = formResponse.getItemResponses();
// Add responses comma-separated included
var rowData = itemResponses.map(item => item.getResponse().toString());
rowData.splice(0, 0, formResponse.getTimestamp());
// Split into different rows afterwards
if (rowData[1].includes(',')) {
rowData[1].split(',').forEach(instanceName => {
let tmpRow = rowData.map(data => data);
tmpRow[1] = instanceName;
sheet.appendRow(tmpRow);
// Append to the sheet
});
}
else {
sheet.appendRow(rowData); // Append to the sheet
}
Current results:
Click here to see image
What I want to get:
Click here to see image
Thanks
When I saw your script, only the 2nd element of rowData is split with ,. I thought that this might be the reason for your issue. And, when appendRow is used in a loop, the process cost will become high. So, in your situation, how about the following modification?
From:
rowData.splice(0, 0, formResponse.getTimestamp());
// Split into different rows afterwards
if (rowData[1].includes(',')) {
rowData[1].split(',').forEach(instanceName => {
let tmpRow = rowData.map(data => data);
tmpRow[1] = instanceName;
sheet.appendRow(tmpRow);
// Append to the sheet
});
}
else {
sheet.appendRow(rowData); // Append to the sheet
}
To:
var date = formResponse.getTimestamp();
var values = rowData.map(v => v.includes(',') ? v.split(",") : [v]);
var res = values[0].map((_, c) => [date, ...values.map(r => r[c] || "")]);
sheet.getRange(sheet.getLastRow() + 1, 1, res.length, res[0].length).setValues(res);
Reference:
map()