I am trying to fetch data from an API. I am able to call the API from postman but when I try to call this from Google App Script I am getting no data. I don't get any response even after 1 min and ultimately it times out. Following is my code. Can anybody help me track the issue?
function myFunction() {
var opt ={
"method":"GET",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
"accept-language": "en-US,en;q=0.9",
"followRedirects": false,
'muteHttpExceptions': true,
'accept-encoding': 'gzip, deflate, br',
'cookie': '_ga=GA1.2.1001788100.1597808220; _gid=GA1.2.2063955289.1607778290; nsit=4RM_Oh6erYvr4YPS_n4CN7vk; ak_bmsc=CD2083BC920E78C3D86C7F69BAFFAAF06011B68D521400008E4DD65FD5A6905E~plJFNfSOzIP8W+3MutjCYfLJuMfo7+QwR5GXtkUmc8bTOwOxtK/SQ9l029moqlrX/JWk0ds0rYR5i6ZOQK+pIRy673it3PDgtnbOevcCB9VakNxo59IZSdaQmIn3A8osFVkH0LH1RYeeigds6/If10inS8OGRscyIVkQIkxQYIGmMvupoNX8UgZ+BphtwRFDqkOSOiCRe/TSZRH8AK9/meUk58bConk4VRQepynCxSqiexjGOk0Agd6qqobGhzD58W; nseappid=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhcGkubnNlIiwiYXVkIjoiYXBpLm5zZSIsImlhdCI6MTYwNzg4MzMwOCwiZXhwIjoxNjA3ODg2OTA4fQ.T48fqcbru4IWbjYh_oGJqXlRRHglM8TNfXI9-_n8iLw; _gat_UA-143761337-1=1; RT="z=1&dm=nseindia.com&si=ce7cafcc-d516-4a6c-a813-a3f417dc755e&ss=king44kw&sl=4&tt=3fn&bcn=%2F%2F684d0d3d.akstat.io%2F&ld=3cus"; bm_sv=57CE5C9C62EF8E9B2C9B39F1BBBB339F~Veo2CLd/zFZyQo+ewos9TTuguUlHl81MyE6n8wX4Byq1Fd7H8VQRzpFkbp+fh1Xn8kcWIPshipgCMHXiCtInyoacx+WNu0V1Ugoi0ATMpi7O+ny3ppkRXH9yGO2IrVc9dp1HgjXeNViJflQCRjpfdt4rXpo87STgHx0NP5xSgcw=',
'authority': 'www.nseindia.com',
'path': '/api/option-chain-indices?symbol=NIFTY',
'scheme':'https',
'dnt': '1',
'referer': 'https://www.nseindia.com/market-data/option-chain',
'sec-ch-ua-mobile': '?0',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin'
};
var url = "https://www.nseindia.com/api/option-chain-indices?symbol=NIFTY";
var response = UrlFetchApp.fetch(
url, opt);
var json = response.getContentText();
console.info(json);
}
Here is the code for the cUrl which works
curl -v --output test.gz -H User-Agent: -H Accept: -H "Connection: keep-alive" -H "Accept-Encoding: gzip, deflate, br" "https://www.nseindia.com/market-data/option-chain"
the output is stored in test.gz, extract the data from this gzip file.
Related
I am trying to login to a website using requests in Python. Firstly, I have recorded the network activity while logining in manually in Chrome and got this:
headers = {
'authority': 'api.stepn.com',
'accept': '*/*',
'accept-language': 'en-US,en;q=0.9,ru;q=0.8,fr;q=0.7',
'origin': 'https://m.stepn.com',
'referer': 'https://m.stepn.com/',
'sec-ch-ua': '".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
}
params = {
'account': 'account',
'password': 'password',
'type': '3',
'deviceInfo': 'web',
}
Then I go and try to login with requests:
response = requests.get('https://api.stepn.com/run/login', params=params, headers=headers)
However, I got the error message: "Empty login/password"
I tried to do the same with Sessions:
data = {
'account': 'account',
'password': 'password',
}
res = s.post('https://api.stepn.com/run/login',data=data)
However, the problem is the same. I also used encrypted and non-encrypted version of the password. What could be the issue?
I've been trying to log in to this website using my credentials in order to scrape my profile name using google apps script. The status code is 200 and I can see that the script is able to get cookies. However, I get Undefined as result instead of profile name.
This is how I'm trying:
function loginAndParseProfile() {
var link = 'https://stackoverflow.com/users/login?ssrc=head&returnurl=https%3a%2f%2fstackoverflow.com%2f';
var options = {
"method": "get",
"headers": {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36"
}
};
var res = UrlFetchApp.fetch(link, options);
var $ = Cheerio.load(res.getContentText());
var fkey = $("input[name='fkey']").first().attr('value');
var payload = {
'fkey': fkey,
'ssrc': 'head',
'email': 'emailaddress',
'password': 'password',
'oauth_version': '',
'oauth_server': ''
};
var options = {
"method" : "post",
'payload': payload,
'muteHttpExceptions': true,
"headers": {
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36",
}
};
var loginURL = "https://stackoverflow.com/users/login?ssrc=head&returnurl=https%3a%2f%2fstackoverflow.com%2f";
var resp = UrlFetchApp.fetch(loginURL,options);
console.log(resp.getResponseCode());
console.log(resp.getAllHeaders()['Set-Cookie']);
var $ = Cheerio.load(resp.getContentText());
var item = $('a.my-profile > [class^="gravatar-wrapper"]').first().attr('title');
console.log(item);
}
How can I make the script work?
Disable redirects by setting followRedirects to false:
var options = {
"method" : "post",
'payload': payload,
'muteHttpExceptions': true,
"headers": {
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36",
},
'followRedirects': false
};
Grab the acct cookie from the response to the POST /users/login request:
const acct = resp.getAllHeaders()['Set-Cookie']
.find(cookie => cookie.includes('acct=t='))
.match(/(acct=t=.*?)\s/)[1];
Make a GET / request supplying the acct cookie and grab your profile name:
const profileRequest = UrlFetchApp.fetch('https://stackoverflow.com', {
method: 'get',
headers: {
Cookie: acct
}
});
const $main = Cheerio.load(profileRequest.getContentText());
const myName = $main('a.my-profile > [class^="gravatar-wrapper"]').first().attr('title');
console.log(myName);
If your credentials are correct, this should output robots.txt.
I am very new to angularjs.
I am trying to make the http call and posting object to the api which only accept json.
'use strict';
// Declare app level module which depends on views, and components
var app = angular.module('ngShow', ['ngRoute','ngResource']);
app.
config(['$routeProvider', function($routeProvider) {
$routeProvider.when('/login', {templateUrl: 'partials/login.html' });
$routeProvider.otherwise({redirectTo: '/login'});
}]);
app.
config(['$httpProvider', function($httpProvider) {
delete $httpProvider.defaults.headers.common['X-Requested-With'];
$httpProvider.defaults.headers.common['Access-Control-Max-Age'] = '1728000';
$httpProvider.defaults.headers.common['Accept'] = 'application/json, text/javascript';
$httpProvider.defaults.headers.common['Content-Type'] = 'application/json; charset=utf-8';
$httpProvider.defaults.useXDomain = true;
}]);
but seems doesn't work from the request headers.
Request Headers 15:31:10.000
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0
Pragma: no-cache
Origin: http://localhost:63342
Host: localhost:8080
DNT: 1
Connection: keep-alive
Cache-Control: no-cache
Access-Control-Request-Method: POST
Access-Control-Request-Headers: access-control-max-age
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
UPDATE #001
Added my loginService.js - it is calling the api ok but just not the json.
app.factory('loginService', function($http){
return{
login:function(user){
console.log("enter login service");
var $promise=$http.post('http://localhost:8080/api/login',user); //send data to the api
$promise.then(function(msg){
if(msg.data=='succes') console.log('succes login');
else console.log('error login');
});
}
}
});
I've got following concerns here:
You use 2 config sections for one module. Not sure if this approach is expected.
Accordingly the documentation, such params are set in run section
Assuming that the user param in the login() method of your loginService factory is the object you want to send as your JSON payload, you can try the following:
var payload = JSON.stringify(user);
var promise = $http({
url: '/api/login',
method: 'POST',
data: payload,
headers: {'Content-Type': 'application/json'}
});
promise.then(function(...) {
...
});
UPDATE:
I would also consolidate the two .config() calls in your code.
app.config(['$routeProvider', '$httpProvider', function($routeProvider, $httpProvider) {
$routeProvider.when('/login', {templateUrl: 'partials/login.html' });
$routeProvider.otherwise({redirectTo: '/login'});
delete $httpProvider.defaults.headers.common['X-Requested-With'];
$httpProvider.defaults.headers.common['Access-Control-Max-Age'] = '1728000';
$httpProvider.defaults.headers.common['Accept'] = 'application/json, text/javascript';
$httpProvider.defaults.headers.common['Content-Type'] = 'application/json; charset=utf-8';
$httpProvider.defaults.useXDomain = true;
}]);
You're already setting the Content-Type header globally (i.e. for all requests) using $httpProvider.defaults.headers.common['Content-Type'], so the headers: {'Content-Type': 'application/json'} in my code above is redundant.
In order to send a file as part of my form, I changed the content type to "multipart/form-data", but when I check the console, it's still using application/json;charset=utf-8 And Node.js (with Sails.js framework) prints an error saying it's invalid JSON.
Here is the code:
$rootScope.objects.user.withHttpConfig({
transformRequest: angular.identity
}).customPUT($scope.objects.user, "", { // user object contains a file
'Content-Type': "multipart/form-data" // should change content-type. I've tried using `undefined` as well
}).then(function(resp) {
if (resp == "OK") {
$scope.successes = [{
msg: "Saved"
}];
}
}, function(err) {
console.log(err);
$scope.errors = err.data.errors;
});
The "Inspect Network Request" tab in Google Chrome reads:
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:30.0) Gecko/20100101 Firefox/30.0
Referer: http://localhost:1337/user/settings
Host: localhost:1337
DNT: 1
Content-Type: application/json;charset=utf-8
Content-Length: 505679
Connection: keep-alive
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Accept: application/json, text/plain, */*
As said in comment above and According to docs you should pass headers as fourth parameter. So your code should be
$rootScope.objects.user.withHttpConfig({
transformRequest: angular.identity
}).customPUT(
$scope.objects.user,
undefined,
undefined,
{
'Content-Type': "multipart/form-data"
}
).then(function(resp) {
if (resp == "OK") {
$scope.successes = [{
msg: "Saved"
}];
}
}, function(err) {
console.log(err);
$scope.errors = err.data.errors;
});
Reference url : https://github.com/mgonto/restangular#custom-methods
I am trying to connect to a table score that exits in a flash game.
When using the browser the header that i am sending it's this one
POST /game/json?h=c1234567890f HTTP/1.1
Host: pt3.forgeofempires.com
User-Agent: Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Cookie: sid=awsndax7nsow; req_page_info=game_v1; start_page_type=game; start_page_version=v1
Connection: keep-alive
Referer: http://cdn.pt.forgeofempires.com/swf/Main.swf?123456789
Content-type: application/x-www-form-urlencoded
Content-length: 161
34c4ceb6d8[{"requestClass":"ClanService","requestMethod":"getOwnClanData","requestData":[],"requestId":1,"__class__":"ServerRequest","clientVersionNumber":0.31}]
After looking for a while (my acknowledgement in JavaScript is not that height), i did find some examples and after understanding them i did make my own code.
function test(){
var header =
{
'Host': 'pt3.forgeofempires.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Cookie': 'sid=awsndax7nsow; req_page_info=game_v1; start_page_type=game; start_page_version=v1',
'Connection': 'keep-alive',
'Referer': 'http://cdn.pt.forgeofempires.com/swf/Main.swf?123456789',
'Content-type': 'application/x-www-form-urlencoded',
'Content-length': '161',
'': '96c4ceb6d9[{"requestClass":"ClanService","requestMethod":"getOwnClanData","requestData":[],"requestId":4,"__class__":"ServerRequest","clientVersionNumber":0.31}]'
}
var url = "http://pt3.forgeofempires.com/game/json?h=c1234567890f";
var h = {
'method' : "post",
'payload' : header
};
var response = UrlFetchApp.fetch(url, h);
return response;
}
When i run the code it goes always to the login page.
Not sure if this is happening because of any script error or simple because it's a different ip address accessing to my account.
No sure as well how to send the last part, because the last part it's a text data only.
Any help will be appreciated... thanks in advance
You are adding your headers as a payload. UrlFetchApp provides a way to add headers. Use that
var h = {
'method' : "post",
'headers' : header
};
var response = UrlFetchApp.fetch(url, h);
I'm not sure what your payload is. In the list of headers, there is a blank header - is this the payload ? If yes, then you have to add the payload separately.
var h = {
'method' : "post",
'headers' : header,
'payload' : '96c4ceb6d9[{"requestClass":"ClanService","requestMethod":"getOwnClanData","requestData":[],"requestId":4,"__class__":"ServerRequest","clientVersionNumber":0.31}]'
};
var response = UrlFetchApp.fetch(url, h);
Note: Ensure that you remove the blank header from the list of headers.