function getData(request){
try{
var options = {
'method' : 'post',
'contentType': 'application/json',
'payload' : JSON.stringify(request)
};
response=UrlFetchApp.fetch(getDataUrl, options);
resData = JSON.parse(response.getContentText())
return resData
}catch (e) {
e = (typeof e === 'string') ? new Error(e) : e;
Logger.log("Catch", e);
throw e;
}
}
The the above is my getData() function.
My isAdminUser() returns true.
When I try to visualize my data, I get the following error
Data Set Configuration Error
Data Studio cannot connect to your data set.
There was an error requesting data from the community connector. Please report the issue to the provider of this community connector if this issue persists.
Error ID: 3d11b88b
https://i.stack.imgur.com/x3Hki.png
The error code changes every time I refresh data and I can't find any dictionary to map the error id to an error
I tried debugging by logging the request parameter, response.getContentText() and resData variable to make sure I my data is formatted correctly.
Following are the logs printed in Stackdriver logs
request
{configParams={/Personal config data/}, fields=[{name=LASTNAME}]}
response.getContentText()
{"schema":[{"name":"LASTNAME","dataType":"STRING"}],"rows":[{"values":["test"]},{"values":["test"]},{"values":["Dummy"]},{"values":["One"]},{"values":["Nagargoje"]},{"values":[""]},{"values":[""]},{"values":[""]},{"values":[""]},{"values":[""]}],"filtersApplied":false}
resData
{rows=[{values=[test]}, {values=[test]}, {values=[Dummy]},
{values=[One]}, {values=[Nagargoje]}, {values=[]}, {values=[]},
{values=[]}, {values=[]}, {values=[]}], filtersApplied=false,
schema=[{name=LASTNAME, dataType=STRING}]}
I am not sure what is wrong with my getData() function.
The Object that I am returning seems to match the structure given here https://developers.google.com/datastudio/connector/reference#getdata
So there was no issue with my getData() function, the issue existed in the manifest file.
I was searching about passing parameter via URL and I stumbled upon a field called
dataStudio.useQueryConfig and added that to my manifest file and set its value to true.
Google Data studio was expecting me to return a query Config for getData().
But what I really wanted was this.
Anyways, I was able to debug it thanks to Matthias for suggesting me to take a look at Open-Source implementations
I implemented JSON connect which worked fine, so I Logged what it was returning in getData() and used that format/structure in my code, but my connector still didn't work.
My next assumption was maybe there is something wrong with my getSchema() return value. So I logged that as well and then copy pasted the hard coded value of both getData() and getSchema() return varaibles from JSON connect.
And even that didn't work, so my last bet was there must be something wrong with the manifest file, maybe the dummy links I added in it must be the issue. Then, after carrying out field by comparison I was finally able to get my community connector working.
This would have been easier to debug if the error messages were a bit helpful and didn't seem so generic.
First: You can always check out the Open-Source implementations that others did for custom Google Data Studio connectors. They are a great source if information. Fore more information checkout the documentation on Open Source Community Connectors.
Second: My implementation is for a time tracking system thus having confidential GDPR relevant data. That's why I can not just give you response messages. But I assembled this code. It contains authentifiction, HTTP GET data fetch and data conversions. Explanation is below the code. Again, checkout the open-source connectors if you need further assistance.
var cc = DataStudioApp.createCommunityConnector();
const URL_DATA = 'https://www.myverysecretdomain.com/api';
const URL_PING = 'https://www.myverysecretdomain.com/ping';
const AUTH_USER = 'auth.user'
const AUTH_KEY = 'auth.key';
const JSON_TAG = 'user';
String.prototype.format = function() {
// https://coderwall.com/p/flonoa/simple-string-format-in-javascript
a = this;
for (k in arguments) {
a = a.replace("{" + k + "}", arguments[k])
}
return a
}
function httpGet(user, token, url, params) {
try {
// this depends on the URL you are connecting to
var headers = {
'ApiUser': user,
'ApiToken': token,
'User-Agent': 'my super freaky Google Data Studio connector'
};
var options = {
headers: headers
};
if (params && Object.keys(params).length > 0) {
var params_ = [];
for (const [key, value] of Object.entries(params)) {
var value_ = value;
if (Array.isArray(value))
value_ = value.join(',');
params_.push('{0}={1}'.format(key, encodeURIComponent(value_)))
}
var query = params_.join('&');
url = '{0}?{1}'.format(url, query);
}
var response = UrlFetchApp.fetch(url, options);
return {
code: response.getResponseCode(),
json: JSON.parse(response.getContentText())
}
} catch (e) {
throwConnectorError(e);
}
}
function getCredentials() {
var userProperties = PropertiesService.getUserProperties();
return {
username: userProperties.getProperty(AUTH_USER),
token: userProperties.getProperty(AUTH_KEY)
}
}
function validateCredentials(user, token) {
if (!user || !token)
return false;
var response = httpGet(user, token, URL_PING);
if (response.code == 200)
console.log('API key for the user %s successfully validated', user);
else
console.error('API key for the user %s is invalid. Code: %s', user, response.code);
return response;
}
function getAuthType() {
var cc = DataStudioApp.createCommunityConnector();
return cc.newAuthTypeResponse()
.setAuthType(cc.AuthType.USER_TOKEN)
.setHelpUrl('https://www.myverysecretdomain.com/index.html#authentication')
.build();
}
function resetAuth() {
var userProperties = PropertiesService.getUserProperties();
userProperties.deleteProperty(AUTH_USER);
userProperties.deleteProperty(AUTH_KEY);
console.info('Credentials have been reset.');
}
function isAuthValid() {
var credentials = getCredentials()
if (credentials == null) {
console.info('No credentials found.');
return false;
}
var response = validateCredentials(credentials.username, credentials.token);
return (response != null && response.code == 200);
}
function setCredentials(request) {
var credentials = request.userToken;
var response = validateCredentials(credentials.username, credentials.token);
if (response == null || response.code != 200) return { errorCode: 'INVALID_CREDENTIALS' };
var userProperties = PropertiesService.getUserProperties();
userProperties.setProperty(AUTH_USER, credentials.username);
userProperties.setProperty(AUTH_KEY, credentials.token);
console.info('Credentials have been stored');
return {
errorCode: 'NONE'
};
}
function throwConnectorError(text) {
DataStudioApp.createCommunityConnector()
.newUserError()
.setDebugText(text)
.setText(text)
.throwException();
}
function getConfig(request) {
// ToDo: handle request.languageCode for different languages being displayed
console.log(request)
var params = request.configParams;
var config = cc.getConfig();
// ToDo: add your config if necessary
config.setDateRangeRequired(true);
return config.build();
}
function getDimensions() {
var types = cc.FieldType;
return [
{
id:'id',
name:'ID',
type:types.NUMBER
},
{
id:'name',
name:'Name',
isDefault:true,
type:types.TEXT
},
{
id:'email',
name:'Email',
type:types.TEXT
}
];
}
function getMetrics() {
return [];
}
function getFields(request) {
Logger.log(request)
var fields = cc.getFields();
var dimensions = this.getDimensions();
var metrics = this.getMetrics();
dimensions.forEach(dimension => fields.newDimension().setId(dimension.id).setName(dimension.name).setType(dimension.type));
metrics.forEach(metric => fields.newMetric().setId(metric.id).setName(metric.name).setType(metric.type).setAggregation(metric.aggregations));
var defaultDimension = dimensions.find(field => field.hasOwnProperty('isDefault') && field.isDefault == true);
var defaultMetric = metrics.find(field => field.hasOwnProperty('isDefault') && field.isDefault == true);
if (defaultDimension)
fields.setDefaultDimension(defaultDimension.id);
if (defaultMetric)
fields.setDefaultMetric(defaultMetric.id);
return fields;
}
function getSchema(request) {
var fields = getFields(request).build();
return { schema: fields };
}
function convertValue(value, id) {
// ToDo: add special conversion if necessary
switch(id) {
default:
// value will be converted automatically
return value[id];
}
}
function entriesToDicts(schema, data, converter, tag) {
return data.map(function(element) {
var entry = element[tag];
var row = {};
schema.forEach(function(field) {
// field has same name in connector and original data source
var id = field.id;
var value = converter(entry, id);
// use UI field ID
row[field.id] = value;
});
return row;
});
}
function dictsToRows(requestedFields, rows) {
return rows.reduce((result, row) => ([...result, {'values': requestedFields.reduce((values, field) => ([...values, row[field]]), [])}]), []);
}
function getParams (request) {
var schema = this.getSchema();
var params;
if (request) {
params = {};
// ToDo: handle pagination={startRow=1.0, rowCount=100.0}
} else {
// preview only
params = {
limit: 20
}
}
return params;
}
function getData(request) {
Logger.log(request)
var credentials = getCredentials()
var schema = getSchema();
var params = getParams(request);
var requestedFields; // fields structured as I want them (see above)
var requestedSchema; // fields structured as Google expects them
if (request) {
// make sure the ordering of the requested fields is kept correct in the resulting data
requestedFields = request.fields.filter(field => !field.forFilterOnly).map(field => field.name);
requestedSchema = getFields(request).forIds(requestedFields);
} else {
// use all fields from schema
requestedFields = schema.map(field => field.id);
requestedSchema = api.getFields(request);
}
var filterPresent = request && request.dimensionsFilters;
//var filter = ...
if (filterPresent) {
// ToDo: apply request filters on API level (before the API call) to minimize data retrieval from API (number of rows) and increase speed
// see https://developers.google.com/datastudio/connector/filters
// filter = ... // initialize filter
// filter.preFilter(params); // low-level API filtering if possible
}
// get HTTP response; e.g. check for HTTT RETURN CODE on response.code if necessary
var response = httpGet(credentials.username, credentials.token, URL_DATA, params);
// get JSON data from HTTP response
var data = response.json;
// convert the full dataset including all fields (the full schema). non-requested fields will be filtered later on
var rows = entriesToDicts(schema, data, convertValue, JSON_TAG);
// match rows against filter (high-level filtering)
//if (filter)
// rows = rows.filter(row => filter.match(row) == true);
// remove non-requested fields
var result = dictsToRows(requestedFields, rows);
console.log('{0} rows received'.format(result.length));
//console.log(result);
return {
schema: requestedSchema.build(),
rows: result,
filtersApplied: filter ? true : false
};
}
A sample request that filters for all users with names starting with J.
{
configParams={},
dateRange={
endDate=2020-05-14,
startDate=2020-04-17
},
fields=[
{name=name}
],
scriptParams={
lastRefresh=1589543208040
},
dimensionsFilters=[
[
{
values=[^J.*],
operator=REGEXP_EXACT_MATCH,
type=INCLUDE,
fieldName=name
}
]
]
}
The JSON data returned by the HTTP GET contains all fields (full schema).
[ { user:
{ id: 1,
name: 'Jane Doe',
email: 'jane#doe.com' } },
{ user:
{ id: 2,
name: 'John Doe',
email: 'john#doe.com' } }
]
Once the data is filtered and converted/transformed, you'll get this result, which is perfectly displayed by Google Data Studio:
{
filtersApplied=true,
schema=[
{
isDefault=true,
semantics={
semanticType=TEXT,
conceptType=DIMENSION
},
label=Name,
name=name,
dataType=STRING
}
],
rows=[
{values=[Jane Doe]},
{values=[John Doe]}
]
}
getData should return data for only the requested fields. In request.fields should have the list of all requested fields. Limit your data for those fields only and then send the parsed data back.
I'm accessing an api and I generated using NSwagStudio a type script version of the api calls to use as services. I get the response from the server and seems to work fine, but I don't know how to access the json file with the response. I tried subscribing to the method that I'm calling but I always get null as a response. Any help or guidelines will be appreciated it.
Here is an example of the code generated by NSwagStudio and my implementation to subscribe to the response.
apiSubmissionGetResultMessageGet(...) {
protected processApiSubmissionGetResultMessageGet(response: HttpResponseBase): Observable<void> {
const status = response.status;
const responseBlob =
response instanceof HttpResponse ? response.body :
(<any>response).error instanceof Blob ? (<any>response).error : undefined;
let _headers: any = {}; if (response.headers) { for (let key of response.headers.keys()) { _headers[key] = response.headers.get(key); }};
if (status === 200) {
return blobToText(responseBlob).flatMap(_responseText => {
return Observable.of<void>(<any>null);
});
} else if (status !== 200 && status !== 204) {
return blobToText(responseBlob).flatMap(_responseText => {
return throwException("An unexpected server error occurred.", status, _responseText, _headers);
});
}
return Observable.of<void>(<any>null);
}
}
And this is where I'm trying to subscribe:
getSubmissionDetails(string): void {
this.client.apiSubmissionGetSubmissionDocumentGet('documentId')
.subscribe(
data => {
this.submissionList = this.submissionList;
console.log('data: ', data);
},
(error: any) => this.errorMessage = <any> error);
}
The return type is Observable<void> which means that it doesnt return anything...
Check that the operation has a response type in the swagger spec and regenerate.
I am using NodeJS request module to pass a JSON request to a URL and generate a JSON response from it. I tried this code and it generates a valid response. I am pasting the link for a StackOverflow question I asked for the same.
NodeJS Request returning an empty array inside a JSON response
However, when I utilize the same logic in AWS Lambda, there is no response at all from the module. Since there is no response at all, I cannot understand what the problem is.
This is the handling function for the AWS Lambda with Alexa as a trigger.
'use strict';
var request = require('request');
var accountNumberRequest = {};
var balanceResponse = {};
const url = "https://ibluatapig.indusind.com/app/uat/balinq/AccountEnquiry?client_id=6867b781-9b21-45c5-9c55-948f7cd1a33f&client_secret=hP3yB3hM2oH4pH4hM1kV3uY8vR3qV7jY8cF6bG2sF5jX8lT1vN";
var bal = {};
exports.handler = function (event,context) {
try{
console.log("Try Started");
var req = event.request;
console.log("Request Generated");
if(req.type === "LaunchRequest") {
console.log("Launch Request! Calling handleLaunchRequest");
handleLaunchRequest(context);
} else if(req.type === "IntentRequest") {
console.log("IntentRequest");
let options = {};
console.log(0);
if(req.intent.name === "BalanceIntent") {
console.log("Balance Intent");
//Got the account number from Alexa request
let accNo = req.intent.slots.AccountNumber.value;
console.log(accNo);
accountNumberRequest = {
"AERequest":{
"serviceType":"BE",
"deviceId":"Test",
"accountId":accNo
}
};
console.log(accountNumberRequest);
console.log("Calling NodeJS.Request");
request({
url: url,
method: "POST",
json: true,
header: {
"content-type": "application/json",
},
body: accountNumberRequest
},
function(error, response,body){
if(!error && response.statusCode === 200){
console.log(body.AEResponse.AcctBal[1].BalAmt);
} else {
//options.speechText = `The account <say-as interepret-as = "digits">${accNo}</say-as> does not exist`;
console.log("error: "+error);
console.log("response.statusCode"+response.statusCode);
console.log("response.statusText"+response.statusText);
}
}
);
console.log("Balance Response should be assigned by now");
console.log(bal);
/* if(accountNumbers.hasOwnProperty(accNo)) {
var balance = accountNumbers[accNo];
accountExists = true;
}
if(accountExists == true){
options.speechText = `The balance of account number <say-as interpret-as = "digits">${accNo}</say-as> is <say-as interpret-as = "cardinal">${balance}</say-as>`;
} else {
options.speechText = `The account <say-as interepret-as = "digits">${accNo}</say-as> does not exist`;
}*/
context.succeed(buildResponse(options));
}
} else if(req.type === "SessionEndedRequest") {
//Code here
} else {
throw("Unknown Intent Type");
}
} catch(e){
context.fail("Exception "+e);
}
};
function getBalance(){
//Code to parse the JSON response and extract values from the response.
}
function handleLaunchRequest(context){
//Code for handling launch requests }
function buildResponse(options){
//Code for generating response
}
This is the problem...
// You're sending an asynchronous HTTP request here.
request();
// But you sent the response here without waiting for the above request to finish.
context.succeed();
Basically, you're executing context.succeed() before request() finishes. So you're basically ending your Lambda invocation without the response from that HTTP request.
To fix your code, put the context.succeed() inside the callback that you pass to the request() call.
P.S.
You should be using callback instead of the deprecated context.succeed()/context.fail() API.
I am creating a website that reads externally hosted json files and then uses node.js to populate the sites content.
Just to demonstrate what I'm after, this is a really simplified version of what I'm trying to do in node.js
var ids = [111, 222, 333];
ids.forEach(function(id){
var json = getJSONsomehow('http://www.website.com/'+id+'.json');
buildPageContent(json);
});
Is what I want to do possible?
(Marked as a duplicate of "How do I return the response from an asynchronous call?" see my comment below for my rebuttal)
You are trying to get it synchronously. What you should aim for instead, is not a function used like this:
var json = getJSONsomehow('http://www.website.com/'+id+'.json');
but more like this:
getJSONsomehow('http://www.website.com/'+id+'.json', function (err, json) {
if (err) {
// error
} else {
// your json can be used here
}
});
or like this:
getJSONsomehow('http://www.website.com/'+id+'.json')
.then(function (json) {
// you can use your json here
})
.catch(function (err) {
// error
});
You can use the request module to get your data with something like this:
var request = require('request');
var url = 'http://www.website.com/'+id+'.json';
request.get({url: url, json: true}, (err, res, data) => {
if (err) {
// handle error
} else if (res.statusCode === 200) {
// you can use data here - already parsed as json
} else {
// response other than 200 OK
}
});
For a working example see this answer.
For more info see: https://www.npmjs.com/package/request
I think problem is in async request. Function will return result before request finished.
AJAX_req.open( "GET", url, true );
Third parameter specified async request.
You should add handler and do all you want after request finished.
For example:
function AJAX_JSON_Req( url ) {
var AJAX_req = new XMLHttpRequest.XMLHttpRequest();
AJAX_req.open( "GET", url, true );
AJAX_req.setRequestHeader("Content-type", "application/json");
AJAX_req.onreadystatechange = function() {
if (AJAX_req.readyState == 4 && AJAX_req.status == 200) {
console.log(AJAX_req.responseText);
}
};
}
How do i read HttpRequest data sent by POST method from client, on the server, in Dart?
I send a message from the client like this:
HttpRequest request = new HttpRequest();
var url = "http://127.0.0.1:8081";
request.open("POST", url, async: false);
String data = 'hello from client';
request.send(data);
On server i am catching the request like this:
HttpServer.bind('127.0.0.1', 8081).then((server) {
server.listen((HttpRequest request) {
//DATA SHOULD BE READ HERE
});
});
But i cant figure out how to actually read the data... There is not data property in HttpRequest nor anything else...
EDIT This is how i get the answer now:
HttpServer.bind('127.0.0.1', 8081).then((server) {
server.listen((HttpRequest request) {
//DATA SHOULD BE READ HERE
print("got it");
print(request.method);
if(request.method == "POST") {
print("got it 2");
List<int> dataBody = new List<int>();
request.listen(dataBody.addAll, onDone: () {
var postData = new String.fromCharCodes(dataBody);
print(postData);
});
}
});
});
But for some reason the request.method is not "POST" but "OPTIONS", and if i change to if(request.method == "OPTIONS") , then print(postData) will still return nothing...
You can use the StringDecoder to tranform from "List of Int" to "String" from the HttpRequest. Since no matter if you send json, plain text, or png, Dart always send data in form of
"List of Int" to the server.Another means is to use the Streams (http://www.dartlang.org/articles/feet-wet-streams/) tested on Heroku Steam v0.6.2 Dart Editor 0.4.3_r20602 Dat SDK 0.4.3.5_r26062
For example,
the client:
import 'dart:html';
import 'dart:json' as Json;
import 'dart:async';
import 'dart:uri';
final String data = 'Hello World!';
void _sendPNG(String pngData) {
HttpRequest request = new HttpRequest(); // create a new XHR
// add an event handler that is called when the request finishes
request.onReadyStateChange.listen((_)
{
if (request.readyState == HttpRequest.DONE &&
(request.status == 200 || request.status == 0)) {
// data saved OK.
print(request.responseText); // output the response from the server
}
}
);
// POST the data to the server Async
print('Sending Photos to the server...');
var url = "/png";
request.open("POST", url);
request.setRequestHeader("Content-Type", "text/plain");
request.send(data);
}
the server:
import 'dart:io';
import 'dart:async';
import 'dart:json' as Json;
import "package:stream/stream.dart";
import 'package:xml/xml.dart' as xml;
import 'package:unittest/unittest.dart';
import 'package:rikulo_commons/mirrors.dart';
void receivePNG(HttpConnect connect){
var request = connect.request;
var response = connect.response;
if(request.uri.path == '/png' && request.method == 'POST')
{
String png='';
response.write('The server received png request!');
//read incoming List<int> data from request and use StringDecoder to transform incoming data to string
var stream = request.transform(new StringDecoder());
stream.listen((value){
print(value);
//Hello World!
}
else
{
response.write('error');
response.statusCode = HttpStatus.NOT_FOUND;
connect.close();
}
}
configure.dart
var _mapping = {
"/": home,
"/png": receivePNG,
};
Right now, the handling of POST data is a little difficult. But essentially the HttpRequest itself has to be 'listened' to. HttpRequest is a stream itself. In particular it's a Stream<List<int>>. So basically your data may be passed to your HttpRequest as multiple List<int>'s. So we need to reconstruct the data then convert it into a string (assuming you're expecting a string, not binary data, etc). Here's more or less what I do:
HttpServer.bind('127.0.0.1', 8081).then((server) {
server.listen((HttpRequest request) {
if(request.method == "POST") {
List<int> dataBody = new List<int>();
request.listen(dataBody.addAll, onDone: () {
var postData = new String.fromCharCodes(dataBody);
// Do something with the data now.
});
}
request.response.close();
});
Note that the request.listen(dataBody.AddAll, ...) basically calls List.addAll() each time data is to the server (in cases of larger data or multi-part forms it may not come all at once). This ensures we buffer it all until the stream indicates it is 'done' In which case we can now do something with the data we received, like convert it to a string.
I have found this useful example with client/side code
GitHub json send to server Example
// XXX: Dart Editor thinks this is OK, but I haven't run it.
import 'dart:html';
String encodeMap(Map data) {
return data.keys.map((k) {
return '${Uri.encodeComponent(k)}=${Uri.encodeComponent(data[k])}';
}).join('&');
}
loadEnd(HttpRequest request) {
if (request.status != 200) {
print('Uh oh, there was an error of ${request.status}');
return;
} else {
print('Data has been posted');
}
}
main() {
var dataUrl = '/registrations/create';
var data = {'dart': 'fun', 'editor': 'productive'};
var encodedData = encodeMap(data);
var httpRequest = new HttpRequest();
httpRequest.open('POST', dataUrl);
httpRequest.setRequestHeader('Content-type',
'application/x-www-form-urlencoded');
httpRequest.onLoadEnd.listen((e) => loadEnd(httpRequest));
httpRequest.send(encodedData);
}