I want to get html from a web. But it show like that.
meta http-equiv=refresh content="0;url=http://www.skku.edu/errSkkuPage.jsp">
But when I use https://www.naver.com/ instead of https://www.skku.edu/skku/index.do, it works well.
I want to know the reason.
Here's my code.
var request = require('request');
const url = "https://www.skku.edu/skku/index.do";
request(url, function(error, response, body){
if (error) throw error;
console.log(body);
});
The website blocks the request that is coming from programmatic script checking User-Agent in the request header.
Pass the user-Agent that web-browser(eg: Google chrome) sends and it should work.
var request = require('request');
var options = {
'method': 'GET',
'url': 'https://www.skku.edu/skku/index.do',
'headers': {
'User-Agent': ' Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36'
}
};
request(options, function (error, response) {
if (error) throw new Error(error);
console.log(response.body);
});
I wouldn't recommend request module as it is not maintained for changes anymore. see it here - https://github.com/request/request/issues/3142
You could look for alternatives in form of got, axios etc which makes code much more readable and clear. And most important thing - Native support for promises and async/await The above code will look like
var got = require('got');
const url = "https://www.skku.edu/skku/index.do";
(async () => {
const response = await got(url);
console.log(response.body);
})();
Related
Hey there It's my first post so sorry if I am doing something wrong here but be patient with me ;)
I am trying to send some data in JSON format to my MySQL DB using Express but whenever I use something else besides app.get() it fails. I guess it is because the request methode shown in the browser is always GET but I dont know why.
What am I doing wrong? How can the request method be GET when I am using app.post()?
const express = require('express');
const mysql = require('mysql');
const bodyParser = require('body-parser');
const db = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'masterkey',
database: 'articelStorage'
});
const app = express();
app.use(bodyParser.json());
app.use(express.json());
//connect to db
db.connect((err) => {
if (err) {
throw err;
}
console.log('Myql connected...');
});
app.listen('3000', () => {
console.log('Server started and running on port 3000...');
});
app.get('/getOne/:code', (req, res) => {
let sql = "SELECT * FROM articels WHERE acode ='" +req.params.code+"'";
db.query(sql, (err, result) => {
if (err) throw err;
res.send(result);
});
});
app.delete('/deleteOne/:code', (req, res) => {
let sql = "DELETE FROM articels WHERE acode ='" +req.params.code+ "'";
db.query(sql, (err, result) => {
if (err) throw err;
res.send(result);
});
});
Here is the result I get:
Cannot GET /deleteOne/DE12345678
And the Headers:
General:
Request URL: http://localhost:3000/deleteOne/DE12345678
Request Method: GET
Status Code: 404 Not Found
Remote Address: [::1]:3000
Referrer Policy: no-referrer-when-downgrade
Response Headers:
Connection: keep-alive
Content-Length: 159
Content-Security-Policy: default-src 'self'
Content-Type: text/html; charset=utf-8
Date: Thu, 07 Mar 2019 13:12:36 GMT
X-Content-Type-Options: nosniff
X-Powered-By: Express
Request Headers:
Accept: text/html,application/xhtml+xml,application/xml;
q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Encoding: gzip, deflate, br
Accept-Language: de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7
Connection: keep-alive
Host: localhost:3000
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36
Thx in advance.
I think there is a misconception here.
When you are declaring app.get('/getOne/:code', (req, res)=>{..}) it means the route is accessible by HTTP GET. Similarly for app.delete the route is accessible by request method HTTP DELETE.
Now, when you open a url in browser it is ALWAYS a HTTP GET request. That's why the /getOne will work, but not the delete one.
You need to use Postman(or curl) like application to test your REST api.
If you are accessing an endpoint from Client, use axios or request or xmlhttprequest and set the request method what you want it to be.
For example,
axios({url:url, method:'delete', { data: { foo: "bar" } });// or axios.delete(..)
Im trying to request a Json file from a server different from mine but i cant set the right encoding.
I tried using HTTP module and failed.
Now im trying to do this using the 'Request' module.
The response i get is encoded to i dont know what. maybe utf 16 and is not readable at all.
Note: The json has some Hebrew chars in it.
I added the following to try and fix it but also failed:
headers: {'Content-Type': 'application/json; charset=utf-8'}
My code:
var http = require('http');
var request = require('request');
var express = require('express');
var app = express();
var url = 'http://www.oref.org.il/WarningMessages/alerts.json?v=1';
app.listen(process.env.PORT || 8080);
app.get('/', function(req,res){
res.send("Red color");
});
// get Alerts from web-service
app.get('/getAlerts', function(req,res){
request({
url: url,
json: true,
headers: {'Content-Type': 'application/json; charset=utf-8'}
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
console.log(response.headers) // Print the json response
res.set({
'content-type': 'application/json'
}).send(body);
}
})
});
That API returns a JSON response encoded in UTF-16-LE, so you'll have to tell request to use that encoding instead.
However, since you're trying to query Pikud Haoref's alerts API, check out pikud-haoref-api on npm to do the heavy lifting for you:
https://www.npmjs.com/package/pikud-haoref-api
I have the following route:
Loads.TestRoute = Ember.Route.extend({
model: function() {
return this.store.find('load');
}
});
This to the best of my knowledge will return all instances of load in the data store, which in this case can be anywhere from 1 to 100. For this application, I am using a local storage adapter.
My controller looks like this:
Loads.TestController = Ember.ArrayController.extend({
actions: {
test: function () {
var loads = this.get('model');
var driverId = getCookie("id");
this.store.find("driver", driverId).then(function (driver,loads) {
$.ajax({
type: "POST",
data: JSON.stringify({ Driver: driver, Loads: loads }),
url: "api/build",
contentType: "application/json",
success: function (message) {
alert(message);
}
});
});
}
}
});
What I'm trying to accomplish is to send all instances of the model 'load' along with a specific instance of the model driver as a JSON object to an API on my server that builds spreadsheets.
When I run this, I can see in the request payload that the Driver model object is turned into JSON, but the Loads are not. Here is what is in the payload:
Remote Address:[::1]:49438
Request URL:http://localhost:49438/api/build
Request Method:POST
Status Code:200 OK
Request Headersview source
Accept:*/*
Accept-Encoding:gzip, deflate
Accept-Language:en-US,en;q=0.8
Connection:keep-alive
Content-Length:66
Content-Type:application/json
Cookie:id=atcn4
Host:localhost:49438
Origin:http://localhost:49438
Referer:http://localhost:49438/
User-Agent:Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36
X-Requested-With:XMLHttpRequest
Request Payloadview source
{Driver: {firstName: "Ron", lastName: "Burgandy", truck: "12"}}
How can I update this to make it so both the driver and loads models are sent in the Payload?
Thanks for any help in advance!
You need to make sure both promises from your store are resolved before you send off your ajax request. Use Ember.RSVP.hash
Loads.TestController = Ember.ArrayController.extend({
actions: {
test: function () {
var driverId = getCookie("id");
Ember.RSVP.hash({
loads: this.store.find('load'),
driver: this.store.find('driver', driverId)
}).then(function(data) {
$.ajax({
type: "POST",
data: JSON.stringify({ Driver: data.driver, Loads: data.loads }),
url: "api/build",
contentType: "application/json",
success: function (message) {
alert(message);
}
});
});
}
}
});
I am very new to angularjs.
I am trying to make the http call and posting object to the api which only accept json.
'use strict';
// Declare app level module which depends on views, and components
var app = angular.module('ngShow', ['ngRoute','ngResource']);
app.
config(['$routeProvider', function($routeProvider) {
$routeProvider.when('/login', {templateUrl: 'partials/login.html' });
$routeProvider.otherwise({redirectTo: '/login'});
}]);
app.
config(['$httpProvider', function($httpProvider) {
delete $httpProvider.defaults.headers.common['X-Requested-With'];
$httpProvider.defaults.headers.common['Access-Control-Max-Age'] = '1728000';
$httpProvider.defaults.headers.common['Accept'] = 'application/json, text/javascript';
$httpProvider.defaults.headers.common['Content-Type'] = 'application/json; charset=utf-8';
$httpProvider.defaults.useXDomain = true;
}]);
but seems doesn't work from the request headers.
Request Headers 15:31:10.000
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0
Pragma: no-cache
Origin: http://localhost:63342
Host: localhost:8080
DNT: 1
Connection: keep-alive
Cache-Control: no-cache
Access-Control-Request-Method: POST
Access-Control-Request-Headers: access-control-max-age
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
UPDATE #001
Added my loginService.js - it is calling the api ok but just not the json.
app.factory('loginService', function($http){
return{
login:function(user){
console.log("enter login service");
var $promise=$http.post('http://localhost:8080/api/login',user); //send data to the api
$promise.then(function(msg){
if(msg.data=='succes') console.log('succes login');
else console.log('error login');
});
}
}
});
I've got following concerns here:
You use 2 config sections for one module. Not sure if this approach is expected.
Accordingly the documentation, such params are set in run section
Assuming that the user param in the login() method of your loginService factory is the object you want to send as your JSON payload, you can try the following:
var payload = JSON.stringify(user);
var promise = $http({
url: '/api/login',
method: 'POST',
data: payload,
headers: {'Content-Type': 'application/json'}
});
promise.then(function(...) {
...
});
UPDATE:
I would also consolidate the two .config() calls in your code.
app.config(['$routeProvider', '$httpProvider', function($routeProvider, $httpProvider) {
$routeProvider.when('/login', {templateUrl: 'partials/login.html' });
$routeProvider.otherwise({redirectTo: '/login'});
delete $httpProvider.defaults.headers.common['X-Requested-With'];
$httpProvider.defaults.headers.common['Access-Control-Max-Age'] = '1728000';
$httpProvider.defaults.headers.common['Accept'] = 'application/json, text/javascript';
$httpProvider.defaults.headers.common['Content-Type'] = 'application/json; charset=utf-8';
$httpProvider.defaults.useXDomain = true;
}]);
You're already setting the Content-Type header globally (i.e. for all requests) using $httpProvider.defaults.headers.common['Content-Type'], so the headers: {'Content-Type': 'application/json'} in my code above is redundant.
Angular $http.post method is not posting JSON to service (RESTFul service, node service).
Showing the following error :
XMLHttpRequest cannot load /some/service. Invalid HTTP status code 404
Here is the posted code
$http({method:'POST', url:'/some/service/',data:{"key1":"val1","key2":"val2"}}).success(function(result){
alert(result);
});
The same code is working with the old version of my chrome i.e, v29...* . I updated my chrome to V30...* . Now, it is not working. Not working in the Firefox as well. Is there any problem with chrome and Firefox?
Can anybody help?
I came across a similar issue after updating Chrome to version 30.0.1599.101 and it turned out to be a server problem.
My server is implemented using Express (http://expressjs.com/) and the code below allowing CORS (How to allow CORS?) works well:
var express = require("express");
var server = express();
var allowCrossDomain = function(req, res, next) {
res.header('Access-Control-Allow-Origin', req.headers.origin || "*");
res.header('Access-Control-Allow-Methods', 'GET,POST,PUT,HEAD,DELETE,OPTIONS');
res.header('Access-Control-Allow-Headers', 'content-Type,x-requested-with');
next();
}
server.configure(function () {
server.use(allowCrossDomain);
});
server.options('/*', function(req, res){
res.header('Access-Control-Allow-Origin', req.headers.origin || "*");
res.header('Access-Control-Allow-Methods', 'GET,POST,PUT,HEAD,DELETE,OPTIONS');
res.header('Access-Control-Allow-Headers', 'content-Type,x-requested-with');
res.send(200);
});
server.post('/some_service', function (req, res) {
res.header('Access-Control-Allow-Origin', req.headers.origin);
// stuff here
//example of a json response
res.contentType('json');
res.send(JSON.stringify({OK: true}));
});
The HTTP request looks like:
$http({
method: 'POST',
url: 'http://localhost/some_service',
data: JSON.stringify({
key1: "val1",
key2: "val2"
}),
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}).success(
function (data, status, headers, config) {
//do something
}
).error(
function (data, status, headers, config) {
//do something
}
);
As pointed out in here (https://stackoverflow.com/a/8572637/772020), the idea is to ensure that your server handles properly the OPTIONS request in order to enable CORS.
Well, a new chrome update was released a couple of days ago. Check the patch notes from that release if they changed anything security related.
My extension stopped working both in FF and Chrome a couple of days ago.