considering the website: https://www.coindesk.com/price/bitcoin/
I'm trying to extract the value $23,073.15 (that changes continuosly) of the following span class.
span class="typography__StyledTypography-owin6q-0 fZpnIj">$23,073.15</span
I tried the following code with Google script using Cheerio without success:
var URL = "https://www.coindesk.com/price/bitcoin/";
var response = UrlFetchApp.fetch(URL);
var $ = Cheerio.load(response.getContentText());
var itemsOfInterest = $('span.typography__StyledTypography-owin6q-0 fZpnIj').text();```
any help about?
When I saw the HTML data from your URL using Google Apps Script, unfortunately, it seems that the data is retrieved from an API by Javascript. By this, in this case, your expected value cannot be retrieved from your URL with cheerio. So, in this answer, I would like to propose retrieving your expected value using the API. The sample script is as follows.
Sample script:
function sample() {
const url = "https://production.api.coindesk.com/v2/tb/price/ticker?assets=BTC";
const res = UrlFetchApp.fetch(url);
const obj = JSON.parse(res.getContentText());
const value = obj.data.BTC.ohlc.l;
console.log(value)
}
I think that when this script is run, you can see your expected value in the log.
Related
I am trying to use Google Apps Script to scrape the first ten results of this page: https://www.tenderboard.gov.bh/Tenders/Opened%20Bids/.
Currently, I have written the POST request to the website. However, the response is coming back in a format that I am unsure how to process. The code I have written so far looks like this:
function fetchData() {
var url = "https://www.tenderboard.gov.bh/Templates/TenderBoardWebService.aspx/GetOpenedTenderF";
var formData = {
"tenderNumber":"",
"ministry":"0",
"category":"0",
"openedDate_filter":"",
"sortingType":"0",
"pageIndex":"2"
};
var payload = JSON.stringify(formData);
var params = {
"method":"post",
"contentType":"application/json; charset=UTF-8",
"payload":payload
};
var response = UrlFetchApp.fetch(url,params);
To illustrate its format, a sample of the response is:
{"d":"\u003cNewDataSet\u003e\r\n \u003cPager\u003e\r\n \u003cPageIndex\u003e1\u003c/PageIndex\u003e\r\n \u003cPageSize\u003e10\u003c/PageSize\u003e\r\n \u003cRecordCount\u003e3799\u003c/RecordCount\u003e\r\n \u003cContent\u003e\u0026lt;div class=\u0027table-head\u0027 style=\u0027display: table-header-group; opacity: 1;\u0027\u0026gt;\u0026lt;div class=\"column\" data-label=\"No.\"\u0026gt;No.\u0026lt;/div\u0026gt;\u0026lt;div class=\"column\"
How would I go about extracting the contents of this response to build a 2D array containing the same information as the web page?
Many thanks and apologies for what I am sure is a dense question.
I'm working on a small project using Google App Script now. What I need to do is make both GET and POST request to API. I'm working on a GET request for now and wrote the code below.
var myApiKey = "123456789"
const requestServer = (path, method, params = {}) => {
if (method.toUpperCase() === "GET"){
var url = "https://api.xyz.com/" + path + "apikey=" + myApiKey;
var response = UrlFetchApp.fetch(url);
var json = response.getContentText();
var data = JSON.parse(json);
} else if (method.toUpperCase() === "POST"){
}
}
I used fake url and apikey for asking this question.
My question is if there is a place or a way to store myApiKey in Google App Script, like .env file?
If so how should I store and make my key hidden...
In order to achieve your goal, how about the following 3 patterns.
Pattern 1:
In this pattern, how about using Properties Service? Ref When the script is run, the value can be put and retrieved using Properties Service.
In this case, at first, it is required to input the value to Properties Service.
Sample script:
var scriptProperties = PropertiesService.getScriptProperties();
// Put value.
scriptProperties.setProperty("key", "value");
// Get value.
var value = scriptProperties.getProperty("key");
Pattern 2:
In this pattern, how about using the custom file properties? Ref This can be put and retrieve the value using Drive API.
In this case, at first, it is required to input the value to the custom file properties.
Sample script:
var fileId = ScriptApp.getScriptId();
// Put value.
Drive.Files.update({properties: [{key: "key", value: "value"}]}, fileId);
// Get value.
var savedKey = "key";
var obj = Drive.Files.get(fileId).properties.filter(({key}) => key == savedKey);
if (obj.length == 0) throw new Error("No property for the inputted key");
var value = obj[0].value;
Pattern 3:
In this pattern, how about using a temporal file? In this case, this can be used like .env file. At first, the value is put to the file (in this case, I think that the text file and Google Docs file can be used.). And, when the script is run, the value is retrieved from the file and use it in the script.
Sample script:
// Put value.
DriveApp.createFile("sample.txt", "value");
// Get value.
var value = DriveApp.getFileById("###fileId###").getBlob().getDataAsString().trim();
References:
Properties Service
Add custom file properties
I make a request to Go to Webinar API in Apps Script and the response is something like this:
{"joinUrl":"https://example.com","asset":true,"registrantKey":3669516009270899211,"status":"APPROVED"}
When I make a JSON.parse i get something like this:
{joinUrl=https://example.com, asset=true, registrantKey=3.6695160092708992E18, status=APPROVED}
The RegistrantKey changes, I dont know why.
Thats my code:
try{
var resp = UrlFetchApp.fetch(url,options)
var json=JSON.parse(resp);
return json
}catch(err){
Logger.log(err);
return err;
}
How about this answer? Please think of this as just one of several possible answers.
Issue and workaround:
I think that the reason of your issue is that 3669516009270899211 of "registrantKey":3669516009270899211 is used as the number. In Javascript, the maximum integer value is 9007199254740991. Ref So when 3669516009270899211 is converted to the number, it becomes 3.6695160092708992E18 and when this is converted to the string, it becomes 3669516009270899000.
So in this case, as one of several workaround, how about enclosing 3669516009270899211 by the double quotes like "registrantKey":"3669516009270899211"? By this, "3669516009270899211" is used ad the string, and you can retrieve the value of 3669516009270899211.
Modified script:
When your script is modified, how about the following modification?
From:
var resp = UrlFetchApp.fetch(url,options)
var json=JSON.parse(resp);
return json
To:
var resp = UrlFetchApp.fetch(url,options);
resp = resp.getContentText().replace(/registrantKey":(\d.+),/, "registrantKey\":\"$1\","); // Added
var json=JSON.parse(resp);
return json
Note:
As the test script, you can also use the following script.
var str = '{"joinUrl":"https://example.com","asset":true,"registrantKey":3669516009270899211,"status":"APPROVED"}';
str = str.replace(/registrantKey":(\d.+),/, "registrantKey\":\"$1\",");
var obj = JSON.parse(str);
Logger.log(obj)
Reference:
Number.MAX_SAFE_INTEGER
If I misunderstood your question and this was not the direction you want, I apologize.
I would like to return the number of tables in a HTML page to a google sheet. The code below can get me the number of tables in the chrome console.
var i = 1; [].forEach.call(document.getElementsByTagName("table"),
function(x) { (i++, x); });
console.log (i)
But I dont know how to get this result (i) in Google App Script so I can return it to my sheet. Something on the lines of
function doGet() {
var html = UrlFetchApp.fetch('http://allqs.saqa.org.za/showUnitStandard.php?id=7743').getContentText();
var table = getElementsByClassName(html, 'table')[0];
var i = 1; [].forEach.call(document.getElementsByTagName("table"),
(i++, x);
console.log (i)
You want to retrieve the number of tags of <table> fronm the URL of http://allqs.saqa.org.za/showUnitStandard.php?id=7743 using GAS. If my understanding is correct, how about this modification?
Modification points :
In the native GAS, getElementsByTagName() can't be used.
In this answer, the number of <table> was retrieved using regex.
Modified script :
var url = "http://allqs.saqa.org.za/showUnitStandard.php?id=7743";
var res = UrlFetchApp.fetch(url).getContentText();
var numberOfTables = res.match(/<table/g).length;
Logger.log(numberOfTables) // 96 is retrieved.
If I misunderstand your question, I'm sorry.
I keep finding forum results that refer to the google visualiser for displaying my query results. But it just seems to dump the data in a pre-made table. I haven't found a way to write my own custom table dynamically.
In the past, when I have hacked together PHP to make use of mySQL DB, I would simply connect to my DB, run a query into an array, then cycle through the array and write the table in HTML. I could do IF statements in the middle for formatting or extra tweaks to the displayed data, etc. But I am struggling to find documentation on how to do something similar in a google script. What is the equivalent work flow? Can someone point me to a tutorial that will start me down this path?
I just want a simple HTML page with text box and submit button that runs a query on my Google sheet (back in the .gs file) and displays the results in a table back on the HTML page.
Maybe my understanding that GAS/google sheets is an alternative to PHP/mySQL is where I'm going wrong? Am I trying to make a smoothie with a toaster instead of a blender?
Any help appreciated
Welcome David. Ruben is right, it's cool to post up some code you have tried. But I spent many months getting my head around Apps-script and love to share what I know. There are several ways to get the data out of a Google sheet. There is a very well document Spreadsheet Service For GAS. There is also a client API..
There is the approach you mention. I suggest converting the incoming data to JSON so you can do with it as you like.
Unauthenticated frontend queries are also possible. Which needs the spreadsheet to be published and set to anyone with the link can view, and uses the Google visualisation API
var sql = 'SELECT A,B,C,D,E,F,G where A = true order by A DESC LIMIT 10 offset '
var queryString = encodeURIComponent(sql);
var query = new google.visualization.Query('https://docs.google.com/spreadsheets/d/'+ spreadsheetId +'/gviz/tq?tq=' + queryString);
query.send(handleSampleDataQueryResponse);
function handleSampleDataQueryResponseTotal(responsetotal) {
var myData = responsetotal.getDataTable();
var myObject = JSON.parse(myData.toJSON());
console.log(myObject)
}
Other Approaches
In your GAS back end this can get all of your data in columns A to C as an array of arrays ([[row1],[row2],[row3]]).
function getData(query){
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName('Sheet1');
var range = sheet.getRange('A1:C');
var data = range.getValues();
// return data after some query
}
In your GAS front end this can call to your backend.
var data = google.script.run.withSuccessHandler(success).getData(query);
var success = (e) => {
console.log(e)
}
In your GAS backend this can add data to your sheet. The client side API will also add data, but is more complex and needs authentication.
function getData(data){
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName('Sheet1');
var range = sheet.getRange('A1:C');
var data = range.setValues([[row1],[row2],[row3]]);
return 'success!'
}
In your GAS frontend this can send data to your backend.
var updateData = [[row1],[row2],[row3]]
var data = google.script.run.withSuccessHandler(success).getData(updateData);
var success = (e) => {
console.log(e)
}
Finally
Or you can get everything from the sheet as JSON and do the query in the client. This works okay if you manipulate the data in the sheet as you will need it. This also needs the spreadsheet to be published and set to anyone with the link can view.
var firstSheet = function(){
var spreadsheetID = "SOME_ID";
var url = "https://spreadsheets.google.com/feeds/list/" + spreadsheetID +"/1/public/values?alt=json";
return new Promise((resolve,reject)=>{
$.getJSON(url, (data)=>{
let result = data.feed.entry
resolve(result)
});
})
}
firstSheet().then(function(data){
console.log(data)
})