I am trying to do an urlfetchapp on a website, the website contains HTML but thats good enough for me.
For example I want this url.
The problem of this method is it won't show me any data until I first went to this url.
I tried to do 2 urlfetchapp after each other. But this didn't work.
How can I solve this problem?
Below is my test code, if there is data then objTable should get data.
function loadXML()
{
try
{
var URL = "http://www.knzb.nl/verenigingen/wedstrijdsport/waterpolo/uitslagen_waterpolo/iframe/verenigingen/KNZB-DISTRICT-OOST/";
var response = UrlFetchApp.fetch(URL);
URL = "http://www.knzb.nl/verenigingen/wedstrijdsport/waterpolo/uitslagen_waterpolo/iframe/verenigingen/poule/1178/2074/DHS289F/";
response = UrlFetchApp.fetch(URL).getContentText();
Logger.log(response);
var doc = Xml.parse(response, true);
var objTable = doc.html.body.form.div[1].div.div.div.div.div.div;
}
catch(err)
{
var objTable = "Load failed";
Logger.log("Loading data failed: " + err);
}
return objTable;
}
I already found the answer to my own question.
Needed to resend the cookie. The script below is working as it should be.
function loadXML()
{
try
{
var URL = "http://www.knzb.nl/verenigingen/wedstrijdsport/waterpolo/uitslagen_waterpolo/iframe/verenigingen/KNZB-DISTRICT-OOST/";
var response1 = UrlFetchApp.fetch(URL);
Logger.log(response1);
var cookieStr = response1.getHeaders()["Set-Cookie"];
var params = {method: "POST", headers:{"Cookie": cookieStr}};
URL = "http://www.knzb.nl/verenigingen/wedstrijdsport/waterpolo/uitslagen_waterpolo/iframe/verenigingen/poule/1178/2074/DHS289F/";
response = UrlFetchApp.fetch(URL, params);
Logger.log(response.getAllHeaders());
response = response.getContentText();
var doc = Xml.parse(response, true);
var objTable = doc.html.body.form.div[1].div.div.div.div.div.div;
}
catch(err)
{
var objTable = "Load failed";
Logger.log("Loading data failed: " + err);
}
return objTable;
}
Related
Hi this question was already asked in a previous post and it was answer by Tanaike but when I'm using the suggested script I'm always getting the same error "Login information disallowed" the same problem from post (72164203) I tested this Get line
"https://APIKEY:x#mydomain.salesbinder.com/api/2.0/documents/.json?documentNumber=8006&contextId=5" in postman and it works. Here's the code I'm using but it just wont work for me any help would be greatly appreciated
function fetching() {
var USERNAME = 'API KEY';
var PASSWORD = 'x';
var url = 'https://API KEY:x#myDomain.salesbinder.com/api/2.0/documents/.json?documentNumber=8006&contextId=5"
var headers = {
"Authorization": "Basic " + Utilities.base64Encode(USERNAME + ':' + PASSWORD)
};
let response = UrlFetchApp.fetch(url, { headers });
Logger.log(response.getContentText());
}
Salesbinder API documentation available from this link"https://www.salesbinder.com/api/documents/"
Thanks
With this code I was able to get invoice data from Salesbinder API
function fetching() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName('Sheet1');
var InvNumber = sheet.getRange('A1').getValue();
var USERNAME = 'APIKey';
var PASSWORD = 'x';
var url = 'https://mydomain.salesbinder.com/api/2.0/documents.json?documentNumber='+InvNumber+'&contextId=5';
var headers = {
"Authorization": "Basic " + Utilities.base64Encode(USERNAME + ':' + PASSWORD)
};
let response = UrlFetchApp.fetch(url, { headers });
Logger.log(response.getContentText());
}
I am stucked. I can log in to the website and now after i logged in, i want to get the source code of the url2. Any ideas how i can continue here?
function login() {
var url = "https://www.erevollution.com/en/login";
var payload = {
"email":"test#gmail.com",
"password":"testpassword",
"remember":"on"
};
var options = {
"payload":payload,
"method":"post",
"followRedirects" : false
};
var response = UrlFetchApp.fetch(url, options);
if ( response.getResponseCode() == 200 ) { //could not log in.
var result = "Couldn't login. Username/password is incorrect.";
}
else if ( response.getResponseCode() == 302 ) { //login was successful
var result = "Logged in successfully";
var cookie = response.getAllHeaders()['Set-Cookie'];
var header = { "Cookie":cookie[0] };
var options2 = { "headers": header };
var url2 = "https://www.erevollution.com/tr/market/1/1/1/1";
var response2 = UrlFetchApp.fetch(url2, options2);
}
Logger.log(result);
Logger.log(response2);
}
Issue:
You should be including the cookie in the property headers of the object you are passing as parameter of your call, as you can see in the docs.
Code sample:
var header = { "Cookie": cookie[1] };
var options = { "headers": header };
var url = "https://www.erevollution.com/tr/market/1/1/1/1";
var response = UrlFetchApp.fetch(url, options);
Reference:
UrlFetchApp.fetch(url, params)
I solved the problem by taking my response headers cookies and sorting them as the request headers cookies of the next page (url2) wanted. So i got the right order of cookie (newcookie) for the next page.
function Price() {
var url = "https://www.erevollution.com/en/login";
var payload = {
"email":"test#gmail.com",
"password":"testpassword",
"remember":"on"
};
var options = {
"payload":payload,
"method":"post",
"followRedirects" : false
};
var response = UrlFetchApp.fetch(url, options);
if ( response.getResponseCode() == 200 ) { //could not log in.
var result = "Couldn't login. Username/password is incorrect.";
}
else if ( response.getResponseCode() == 302 ) { //login was successful
var result = "Logged in successfully";
var cookie = response.getAllHeaders()['Set-Cookie'];
for(m=0;m<5;m++){
cookie[m]=cookie[m].substring(0,cookie[m].indexOf(";"));
}
var newcookie=cookie[4]+"; "+cookie[1]+"; "+cookie[2]+";"+cookie[3]+"; "+cookie[0];
var header = { "Cookie":newcookie };
var options2 = { "headers": header };
var url2 = "https://www.erevollution.com/tr/market/1/1/1/1";
var response2 = UrlFetchApp.fetch(url2, options2);
var content = response2.getContentText();
I am working on verifying 301 redirects are functioning properly. I have a Google Sheet of URLs and have a Google script that checks the status being returned:
function getStatusCode(url){
var options = {
'muteHttpExceptions': true,
'followRedirects': false
};
var url_trimmed = url.trim();
var response = UrlFetchApp.fetch(url_trimmed, options);
return response.getResponseCode();
}
Next I am wanting to verify that the URL is being redirected correctly. This is where I'm getting stuck. I tried using a variation of the above code, but I can just return the URL being passed in, not the URL being redirected to, or I just get an error on my Google Sheet. Here is the last bit I tried using (that returns an error).
function getReturnedURL(url) {
var options = {
'method': 'GET',
'muteHttpExceptions': true,
'followRedirects': true
};
var url_trimmed = url.trim();
var returnedUrl = ScriptApp.getService().getUrl(url_trimmed);
var response = UrlFetchApp.fetch(returnedUrl, options);
return response;
}
Any ideas? Or, is this even possible?
So for my situation, I was wanting to pull a URL from a spreadsheet and return the redirected URL in a different column of said spreadsheet. #carlesgg97 put me on the right path, here is what ended up working for me:
function urlProtocol(url){
return URI(url).protocol()
}
function urlHostname(url){
return URI(url).hostname()
}
function getRedirects(url) {
eval(UrlFetchApp.fetch('https://rawgit.com/medialize/URI.js/gh-pages/src/URI.js').getContentText());
var params = {
'followRedirects': false,
'muteHttpExceptions': true
};
var baseUrl = urlProtocol(url) + "://" + urlHostname(url),
response = UrlFetchApp.fetch(url, params),
responseCode = response.getResponseCode();
if(response.getHeaders()['Location']){
var redirectedUrl = getRedirects(baseUrl + response.getHeaders()['Location']);
return redirectedUrl;
} else {
return url;
}
}
You cannot know what the final destination of your request is only from the HTTPresponse obtained.
However, you can use the getRedirects function below to get the URLs that your request would follow upon calling fetch with followRedirects set as true:
function getRedirects(url) {
var params = {
'followRedirects': false,
'muteHttpExceptions': true
};
var followedUrls = [url];
while (true) {
var res = UrlFetchApp.fetch(url, params);
if (res.getResponseCode() < 300 || res.getResponseCode() > 399) {
return followedUrls;
}
var url = res.getHeaders()['Location'];
followedUrls.push(url);
}
}
function test() {
var followedUrls = getRedirects('http://mail.google.com/mail/u/0/#inbox/');
Logger.log(followedUrls);
}
In this case, running test will return:
[http://mail.google.com/mail/u/0/#inbox/,
https://mail.google.com/mail/u/0/,
https://accounts.google.com/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/mail/&ss=1&scc=1<mpl=default<mplcache=2&emr=1&osid=1#]
There was a slight edit needed on Billy's answer (https://stackoverflow.com/a/58801197/9758091) to make it work (remove "baseUrl + " from 6th last Line):
function urlProtocol(url){
return URI(url).protocol()
}
function urlHostname(url){
return URI(url).hostname()
}
function getRedirects(url) {
eval(UrlFetchApp.fetch('https://rawgit.com/medialize/URI.js/gh-pages/src/URI.js').getContentText());
var params = {
'followRedirects': false,
'muteHttpExceptions': true
};
var baseUrl = urlProtocol(url) + "://" + urlHostname(url),
response = UrlFetchApp.fetch(url, params),
responseCode = response.getResponseCode();
if(response.getHeaders()['Location']){
var redirectedUrl = getRedirects(response.getHeaders()['Location']);
return redirectedUrl;
} else {
return url;
}
}
I have setup a bot on telegram bot and connected it with google spreadsheets via apps script by following this tutorial. Here is the code:
var token = ""; // FILL IN YOUR OWN TOKEN
var telegramUrl = "https://api.telegram.org/bot" + token;
var webAppUrl = ""; // FILL IN YOUR GOOGLE WEB APP ADDRESS
var ssId = ""; // FILL IN THE ID OF YOUR SPREADSHEET
function getMe() {
var url = telegramUrl + "/getMe";
var response = UrlFetchApp.fetch(url);
Logger.log(response.getContentText());
}
function setWebhook() {
var url = telegramUrl + "/setWebhook?url=" + webAppUrl;
var response = UrlFetchApp.fetch(url);
Logger.log(response.getContentText());
}
function sendText(id,text) {
var url = telegramUrl + "/sendMessage?chat_id=" + id + "&text=" + text;
var response = UrlFetchApp.fetch(url);
Logger.log(response.getContentText());
}
function doGet(e) {
return HtmlService.createHtmlOutput("Hi there");
}
function doPost(e) {
// this is where telegram works
var data = JSON.parse(e.postData.contents);
var text = data.message.text;
var id = data.message.chat.id;
var name = data.message.chat.first_name + " " + data.message.chat.last_name;
var answer = "Hi " + name + ", thank you for your comment " + text;
sendText(id,answer);
SpreadsheetApp.openById(ssId).getSheets()[0].appendRow([new Date(),id,name,text,answer]);
if(/^#/.test(text)) {
var sheetName = text.slice(1).split(" ")[0];
var sheet = SpreadsheetApp.openById(ssId).getSheetByName(sheetName) ? SpreadsheetApp.openById(ssId).getSheetByName(sheetName) : SpreadsheetApp.openById(ssId).insertSheet(sheetName);
var comment = text.split(" ").slice(1).join(" ");
sheet.appendRow([new Date(),id,name,comment,answer]);
}
}
Now I encountered the following issue; I use my bot to store messages from my home automation system. Therefore I send those messages from the system to telegram bot via HTTP GET request:
https://api.telegram.org/bot[BOT_API_KEY]/sendMessage?chat_id=[MY_CHANNEL_NAME]&text=[MY_MESSAGE_TEXT]
Currently these messages sent through http get request seem to be ignored by the script. Does anyoene know how I can solve this issue?
Judging from your question and comments, it seems you are struggling with sending info from your script to your bot on Telegram. Here are the steps to do that:
1.- Create a bot: on Telegram's search look for #BotFather. Click start, write /newbot, give it a name and a username. You should get a token to access the HTTP API. Save this token.
2.- Find your bot on Telegram with its username. Write something to it e.g. 'test'. This will come in handy later.
3.- Test access to the bot from your code
var token = "123456:kioASDdjicOljd_ijsdf"; // Fill this in with your token
var telegramUrl = "https://api.telegram.org/bot" + token;
function getMe() {
var url = telegramUrl + "/getMe";
var response = UrlFetchApp.fetch(url);
Logger.log(response.getContentText());
}
You should get something resembling this:
{"ok":true,"result":{"id":<somenumber>,"is_bot":true,"first_name":"<name of your bot>","username":"<username of your bot>","can_join_groups":true,"can_read_all_group_messages":false,"supports_inline_queries":false}}
4.- Write the sendMessage function
function sendMessage(chat_id,text) {
var url = telegramUrl + "/sendMessage?chat_id=" + chat_id + "&text=" + text;
var response = UrlFetchApp.fetch(url);
Logger.log(response.getContentText());
}
Known is the text you want to send e.g. 'testing bot', but chat_id is unknown. Where do we get this?
5.- Find the chat_id. Before running this function, make sure that you have at least written one message to your bot on Telegram (step 2)
function getChat_id(){
var res = UrlFetchApp.fetch(telegramUrl+"/getUpdates").getContentText();
var res = JSON.parse(res);
Logger.log(res.result[0].message.chat.id.toString());
}
6.- Run sendMessage with the chat_id you found in step 5 and the message you want to send.
I am using the GAS_Service_Account library.
I was able to do all the .p12 key decoding. But now I have:
myJwt.generateJWT().requestToken();
var token=myJwt.getToken();
Now I would like to know a simple way to use it to make an API call with
the service account to an api like drive to get a specific user's files.
Service accounts are powerful tools. The library GersonLobos mentions is: MJ5317VIFJyKpi9HCkXOfS0MLm9v2IJHf
Lets give some back ground. See the first half of this video on how to set up a service account in your domain:
https://www.youtube.com/watch?v=EDmEzZEGSts
With the library enabled in your project you can generate the token with the following:
// If userEmail is null the service account's token is returned
function tokenService(userEmail){
var userEmail = userEmail || ""
var jsonKey = JSON.parse(PropertiesService.getScriptProperties().getProperty("jsonKey"));
var privateKey = jsonKey.private_key;
var serviceAccountEmail = jsonKey.client_email;
if(!userEmail){userEmail = serviceAccountEmail};
var sa = GSApp.init(privateKey, ['https://www.googleapis.com/auth/drive'], serviceAccountEmail).addUser(userEmail);
var tokenObj = JSON.parse(PropertiesService.getScriptProperties().getProperty(userEmail)) || {};
return function(){
var nowTime = parseInt((Date.now()/1000).toString().substr(0,10));
if(!("token" in tokenObj) || tokenObj.expire < nowTime){
var newToken = sa.requestToken().getToken(userEmail);
PropertiesService.getScriptProperties().setProperty(userEmail, JSON.stringify(newToken));
tokenObj.token = newToken.token;
tokenObj.expire = newToken.expire;
}
return tokenObj.token;
}
}
Good now you have your token. I even show how to cache it as the requests take several seconds. Now you can make some calls. As requested by Gerson here are some drive calls:
function transferFileToUser(fileId, transferToEmail, token){
var url = "https://www.googleapis.com/drive/v2/files/"+fileId+"/permissions?sendNotificationEmails=false";
var payload = {"role":"owner","value":transferToEmail,"type":"user"};
var params ={method:"POST",
contentType:'application/json',
headers:{Authorization: "Bearer " + token},
payload:JSON.stringify(payload),
muteHttpExceptions:true
};
var results = UrlFetchApp.fetch(url, params);
return JSON.parse(results.getContentText());
}
function getAllFolders(token){
var query = "mimeType = 'application/vnd.google-apps.folder'";
return driveList(query, token);
}
function getFilesInFolder(folderId, token){
var query = "'"+folderId+"' in parents and mimeType != 'application/vnd.google-apps.folder'";
return driveList(query, token);
}
// genereic call to drive just pass the query
function driveList(query, token){
var filesArray = [];
var pageToken = "";
var query = encodeURIComponent(query);
var params = {method:"GET",
contentType:'application/json',
headers:{Authorization:"Bearer "+token},
muteHttpExceptions:true
};
var url = "https://www.googleapis.com/drive/v2/files?q="+query;
do{
var results = UrlFetchApp.fetch(url,params);
if(results.getResponseCode() != 200){
Logger.log(results);
break;
}
var files = JSON.parse(results.getContentText());
url = "https://www.googleapis.com/drive/v2/files?q="+query;
for(var i in files.items){
filesArray.push({"name":files.items[i].title, "id":files.items[i].id})
}
pageToken = files.nextPageToken;
url += "&pageToken="+encodeURIComponent(pageToken);
}while(pageToken != undefined)
var filesObj = {};
filesObj["fileObjs"] = filesArray;
return filesObj;
}