function getData(request){
try{
var options = {
'method' : 'post',
'contentType': 'application/json',
'payload' : JSON.stringify(request)
};
response=UrlFetchApp.fetch(getDataUrl, options);
resData = JSON.parse(response.getContentText())
return resData
}catch (e) {
e = (typeof e === 'string') ? new Error(e) : e;
Logger.log("Catch", e);
throw e;
}
}
The the above is my getData() function.
My isAdminUser() returns true.
When I try to visualize my data, I get the following error
Data Set Configuration Error
Data Studio cannot connect to your data set.
There was an error requesting data from the community connector. Please report the issue to the provider of this community connector if this issue persists.
Error ID: 3d11b88b
https://i.stack.imgur.com/x3Hki.png
The error code changes every time I refresh data and I can't find any dictionary to map the error id to an error
I tried debugging by logging the request parameter, response.getContentText() and resData variable to make sure I my data is formatted correctly.
Following are the logs printed in Stackdriver logs
request
{configParams={/Personal config data/}, fields=[{name=LASTNAME}]}
response.getContentText()
{"schema":[{"name":"LASTNAME","dataType":"STRING"}],"rows":[{"values":["test"]},{"values":["test"]},{"values":["Dummy"]},{"values":["One"]},{"values":["Nagargoje"]},{"values":[""]},{"values":[""]},{"values":[""]},{"values":[""]},{"values":[""]}],"filtersApplied":false}
resData
{rows=[{values=[test]}, {values=[test]}, {values=[Dummy]},
{values=[One]}, {values=[Nagargoje]}, {values=[]}, {values=[]},
{values=[]}, {values=[]}, {values=[]}], filtersApplied=false,
schema=[{name=LASTNAME, dataType=STRING}]}
I am not sure what is wrong with my getData() function.
The Object that I am returning seems to match the structure given here https://developers.google.com/datastudio/connector/reference#getdata
So there was no issue with my getData() function, the issue existed in the manifest file.
I was searching about passing parameter via URL and I stumbled upon a field called
dataStudio.useQueryConfig and added that to my manifest file and set its value to true.
Google Data studio was expecting me to return a query Config for getData().
But what I really wanted was this.
Anyways, I was able to debug it thanks to Matthias for suggesting me to take a look at Open-Source implementations
I implemented JSON connect which worked fine, so I Logged what it was returning in getData() and used that format/structure in my code, but my connector still didn't work.
My next assumption was maybe there is something wrong with my getSchema() return value. So I logged that as well and then copy pasted the hard coded value of both getData() and getSchema() return varaibles from JSON connect.
And even that didn't work, so my last bet was there must be something wrong with the manifest file, maybe the dummy links I added in it must be the issue. Then, after carrying out field by comparison I was finally able to get my community connector working.
This would have been easier to debug if the error messages were a bit helpful and didn't seem so generic.
First: You can always check out the Open-Source implementations that others did for custom Google Data Studio connectors. They are a great source if information. Fore more information checkout the documentation on Open Source Community Connectors.
Second: My implementation is for a time tracking system thus having confidential GDPR relevant data. That's why I can not just give you response messages. But I assembled this code. It contains authentifiction, HTTP GET data fetch and data conversions. Explanation is below the code. Again, checkout the open-source connectors if you need further assistance.
var cc = DataStudioApp.createCommunityConnector();
const URL_DATA = 'https://www.myverysecretdomain.com/api';
const URL_PING = 'https://www.myverysecretdomain.com/ping';
const AUTH_USER = 'auth.user'
const AUTH_KEY = 'auth.key';
const JSON_TAG = 'user';
String.prototype.format = function() {
// https://coderwall.com/p/flonoa/simple-string-format-in-javascript
a = this;
for (k in arguments) {
a = a.replace("{" + k + "}", arguments[k])
}
return a
}
function httpGet(user, token, url, params) {
try {
// this depends on the URL you are connecting to
var headers = {
'ApiUser': user,
'ApiToken': token,
'User-Agent': 'my super freaky Google Data Studio connector'
};
var options = {
headers: headers
};
if (params && Object.keys(params).length > 0) {
var params_ = [];
for (const [key, value] of Object.entries(params)) {
var value_ = value;
if (Array.isArray(value))
value_ = value.join(',');
params_.push('{0}={1}'.format(key, encodeURIComponent(value_)))
}
var query = params_.join('&');
url = '{0}?{1}'.format(url, query);
}
var response = UrlFetchApp.fetch(url, options);
return {
code: response.getResponseCode(),
json: JSON.parse(response.getContentText())
}
} catch (e) {
throwConnectorError(e);
}
}
function getCredentials() {
var userProperties = PropertiesService.getUserProperties();
return {
username: userProperties.getProperty(AUTH_USER),
token: userProperties.getProperty(AUTH_KEY)
}
}
function validateCredentials(user, token) {
if (!user || !token)
return false;
var response = httpGet(user, token, URL_PING);
if (response.code == 200)
console.log('API key for the user %s successfully validated', user);
else
console.error('API key for the user %s is invalid. Code: %s', user, response.code);
return response;
}
function getAuthType() {
var cc = DataStudioApp.createCommunityConnector();
return cc.newAuthTypeResponse()
.setAuthType(cc.AuthType.USER_TOKEN)
.setHelpUrl('https://www.myverysecretdomain.com/index.html#authentication')
.build();
}
function resetAuth() {
var userProperties = PropertiesService.getUserProperties();
userProperties.deleteProperty(AUTH_USER);
userProperties.deleteProperty(AUTH_KEY);
console.info('Credentials have been reset.');
}
function isAuthValid() {
var credentials = getCredentials()
if (credentials == null) {
console.info('No credentials found.');
return false;
}
var response = validateCredentials(credentials.username, credentials.token);
return (response != null && response.code == 200);
}
function setCredentials(request) {
var credentials = request.userToken;
var response = validateCredentials(credentials.username, credentials.token);
if (response == null || response.code != 200) return { errorCode: 'INVALID_CREDENTIALS' };
var userProperties = PropertiesService.getUserProperties();
userProperties.setProperty(AUTH_USER, credentials.username);
userProperties.setProperty(AUTH_KEY, credentials.token);
console.info('Credentials have been stored');
return {
errorCode: 'NONE'
};
}
function throwConnectorError(text) {
DataStudioApp.createCommunityConnector()
.newUserError()
.setDebugText(text)
.setText(text)
.throwException();
}
function getConfig(request) {
// ToDo: handle request.languageCode for different languages being displayed
console.log(request)
var params = request.configParams;
var config = cc.getConfig();
// ToDo: add your config if necessary
config.setDateRangeRequired(true);
return config.build();
}
function getDimensions() {
var types = cc.FieldType;
return [
{
id:'id',
name:'ID',
type:types.NUMBER
},
{
id:'name',
name:'Name',
isDefault:true,
type:types.TEXT
},
{
id:'email',
name:'Email',
type:types.TEXT
}
];
}
function getMetrics() {
return [];
}
function getFields(request) {
Logger.log(request)
var fields = cc.getFields();
var dimensions = this.getDimensions();
var metrics = this.getMetrics();
dimensions.forEach(dimension => fields.newDimension().setId(dimension.id).setName(dimension.name).setType(dimension.type));
metrics.forEach(metric => fields.newMetric().setId(metric.id).setName(metric.name).setType(metric.type).setAggregation(metric.aggregations));
var defaultDimension = dimensions.find(field => field.hasOwnProperty('isDefault') && field.isDefault == true);
var defaultMetric = metrics.find(field => field.hasOwnProperty('isDefault') && field.isDefault == true);
if (defaultDimension)
fields.setDefaultDimension(defaultDimension.id);
if (defaultMetric)
fields.setDefaultMetric(defaultMetric.id);
return fields;
}
function getSchema(request) {
var fields = getFields(request).build();
return { schema: fields };
}
function convertValue(value, id) {
// ToDo: add special conversion if necessary
switch(id) {
default:
// value will be converted automatically
return value[id];
}
}
function entriesToDicts(schema, data, converter, tag) {
return data.map(function(element) {
var entry = element[tag];
var row = {};
schema.forEach(function(field) {
// field has same name in connector and original data source
var id = field.id;
var value = converter(entry, id);
// use UI field ID
row[field.id] = value;
});
return row;
});
}
function dictsToRows(requestedFields, rows) {
return rows.reduce((result, row) => ([...result, {'values': requestedFields.reduce((values, field) => ([...values, row[field]]), [])}]), []);
}
function getParams (request) {
var schema = this.getSchema();
var params;
if (request) {
params = {};
// ToDo: handle pagination={startRow=1.0, rowCount=100.0}
} else {
// preview only
params = {
limit: 20
}
}
return params;
}
function getData(request) {
Logger.log(request)
var credentials = getCredentials()
var schema = getSchema();
var params = getParams(request);
var requestedFields; // fields structured as I want them (see above)
var requestedSchema; // fields structured as Google expects them
if (request) {
// make sure the ordering of the requested fields is kept correct in the resulting data
requestedFields = request.fields.filter(field => !field.forFilterOnly).map(field => field.name);
requestedSchema = getFields(request).forIds(requestedFields);
} else {
// use all fields from schema
requestedFields = schema.map(field => field.id);
requestedSchema = api.getFields(request);
}
var filterPresent = request && request.dimensionsFilters;
//var filter = ...
if (filterPresent) {
// ToDo: apply request filters on API level (before the API call) to minimize data retrieval from API (number of rows) and increase speed
// see https://developers.google.com/datastudio/connector/filters
// filter = ... // initialize filter
// filter.preFilter(params); // low-level API filtering if possible
}
// get HTTP response; e.g. check for HTTT RETURN CODE on response.code if necessary
var response = httpGet(credentials.username, credentials.token, URL_DATA, params);
// get JSON data from HTTP response
var data = response.json;
// convert the full dataset including all fields (the full schema). non-requested fields will be filtered later on
var rows = entriesToDicts(schema, data, convertValue, JSON_TAG);
// match rows against filter (high-level filtering)
//if (filter)
// rows = rows.filter(row => filter.match(row) == true);
// remove non-requested fields
var result = dictsToRows(requestedFields, rows);
console.log('{0} rows received'.format(result.length));
//console.log(result);
return {
schema: requestedSchema.build(),
rows: result,
filtersApplied: filter ? true : false
};
}
A sample request that filters for all users with names starting with J.
{
configParams={},
dateRange={
endDate=2020-05-14,
startDate=2020-04-17
},
fields=[
{name=name}
],
scriptParams={
lastRefresh=1589543208040
},
dimensionsFilters=[
[
{
values=[^J.*],
operator=REGEXP_EXACT_MATCH,
type=INCLUDE,
fieldName=name
}
]
]
}
The JSON data returned by the HTTP GET contains all fields (full schema).
[ { user:
{ id: 1,
name: 'Jane Doe',
email: 'jane#doe.com' } },
{ user:
{ id: 2,
name: 'John Doe',
email: 'john#doe.com' } }
]
Once the data is filtered and converted/transformed, you'll get this result, which is perfectly displayed by Google Data Studio:
{
filtersApplied=true,
schema=[
{
isDefault=true,
semantics={
semanticType=TEXT,
conceptType=DIMENSION
},
label=Name,
name=name,
dataType=STRING
}
],
rows=[
{values=[Jane Doe]},
{values=[John Doe]}
]
}
getData should return data for only the requested fields. In request.fields should have the list of all requested fields. Limit your data for those fields only and then send the parsed data back.
I am trying to figure out how to parse the JSON data that I am getting from the Spotify API. I am using this node module https://www.npmjs.com/package/spotify-web-api-js to get Spotify playlist tracks.
I am using this to GET my json (see what I did there)
export class HomePage {
spotifyApi = new SpotifyWebApi;
constructor() {}
}
var spotifyApi = new SpotifyWebApi();
spotifyApi.setAccessToken('Spotify OAuth Token');
spotifyApi.getPlaylistTracks('37i9dQZEVXbMDoHDwVN2tF')
.then(function(data) {
console.log('Playlist Tracks', data);
}, function(err) {
console.error(err);
var prev = null;
function onUserInput(queryTerm) {
// abort previous request, if any
if (prev !== null) {
prev.abort();
}
// store the current promise in case we need to abort it
prev = spotifyApi.searchTracks(queryTerm, {limit: 5});
prev.then(function(data) {
// clean the promise so it doesn't call abort
prev = null;
// ...render list of search results...
}, function(err) {
console.error(err);
});
}
This returns a JSON file but for some reason (probably my mistake) when I use JSON.parse(data);
console.log(data.name) it doesn't work (I know I am doing something wrong here but I don't know how to fix it). Thanks in advance :{)
If you want to get the tracks from the url you have to do this data.tracks.track[0] replace 0 with the needed tracks.
I'll preface this saying I'm a UX Designer that's new to React.
I'm using Expo/React-Native and using the "Location" feature to get the device's location information. I'm able to get the information back in a JSON object by using:
const { height, width } = Dimensions.get("window");
class Explore extends Component {
static navigationOptions = {
header: null
};
state = {
locationResult: ""
};
componentDidMount() {
this._getLocationAsync();
}
_getLocationAsync = async () => {
let { status } = await Permissions.askAsync(Permissions.LOCATION);
if (status !== "granted") {
this.setState({
locationResult: "Permission to access location was denied"
});
}
let location = await Location.getCurrentPositionAsync({});
let geocode = await Location.reverseGeocodeAsync(location.coords);
this.setState({ locationResult: JSON.stringify(geocode) });
};
And in my render I'm calling the value using:
<WelcomeText>
Discover {this.state.locationResult}
</WelcomeText>
Which returns the object:
[{"street":"Stockton St","city":"San Francisco","region":"CA","country":"United States","postalCode":"94108","isoCountryCode":"US","name":"1 Stockton St"}]
But how do I go about just displaying the value of "City" that's in the object?
Try this:
<WelcomeText>
Discover {this.state.locationResult[0].city}
</WelcomeText>
Explanation:
Your this.state.locationResult is an array of objects. With this.state.locationResult[0] we are accessing the first object. Then we can use the . operator to access the property we want. In your case .city
Edit:
You need to pass in geocode without stringifying it, otherwise you cannot access it like i described it above. reverseGeocodeAsync is already returning an array of objects, no need to transform it to a string.
Replace:
this.setState({ locationResult: JSON.stringify(geocode) });
with:
this.setState({ locationResult: geocode });
Edit2:
Here is a working example: https://snack.expo.io/B1SqSd3iN
I have a function in node.js that looks like this:
exports.getAllFlights = function(getRequest) {
// this is the package from npm called "async"
async.map(clients, getFlight, function(err, results) {
getRequest(results);
});
}
The variable clients should be a JSON that looks like this:
{'"A4Q"': 'JZA8187', "'B7P"': 'DAL2098' }.
I expect that the map function will pass the individual indices of the array of the variable clients to getFlight. However, instead it passed the values of that each(ex: 'DAL2098', 'JZA8187' and so on).
Is this the expected functionality? Is there a function in async that will do what I want?
The signature of getFlight is getFlight(identifier, callback). Identifier is what is currently messed up. It returns callback(null, rtn). Null reprsents the nonexistence of an error, rtn represents the JSON that my function produces.
Yes, that's the expected result. The documentation is not very clear but all iterating functions of async.js pass the values of the iterable, not the keys. There is the eachOf series of functions that pass both key and value. For example:
async.eachOf(clients, function (value, key, callback) {
// process each client here
});
Unfortunately there is no mapOf.
If you don't mind not doing things in parallel you can use eachOfSeries:
var results = [];
async.eachOfSeries(clients, function (value, key, callback) {
// do what getFlight needs to do and append to results array
}, function(err) {
getRequest(results);
});
Another (IMHO better) workaround is to use proper arrays:
var clients = [{'A4Q': 'JZA8187'},{'B7P': 'DAL2098'}];
Then use your original logic. However, I'd prefer to use a structure like the following:
var clients = [
{key: 'A4Q', val: 'JZA8187'},
{key: 'B7P', val: 'DAL2098'}
];
First create a custom event. Attach a listener for return data. then process it.
var EventEmitter = require('events');
var myEmitter = new EventEmitter();
myEmitter.emit('clients_data',{'"A4Q"': 'JZA8187'}); //emit your event where ever
myEmitter.on('clients_data', (obj) => {
if (typeof obj !=='undefined') {
if (obj.contructor === Object && Object.keys(obj).lenth == 0) {
console.log('empty');
} else {
for(var key in obj) {
var value = obj[key];
//do what you want here
}
}
}
});
Well, you need to format your clients object properly before you can use it with async.map(). Lodash _.map() can help you:
var client_list = _.map(clients, function(value, key) {
var item = {};
item[key] = value;
return item;
});
After that, you will have an array like:
[ { A4Q: 'JZA8187' }, { B7P: 'DAL2098' } ]
Then, you can use async.map():
exports.getAllFlights = function(getRequest) {
async.map(client_list, getFlight, function(err, results) {
getRequest(results);
});
};
I have a JSON file, which contains:
{
"/default.aspx": "headerBg",
"/about.aspx": "aboutBg",
"/contact.aspx": "contactBg",
"/registration.aspx": "regBg",
"/clients.aspx": "clientsBg",
"/onlinesessions.aspx": "bg-white-box",
"/ondemamdsessions.aspx": "bg-grey"
}
Now I am reading this json file using $http, but I want to add a filter in below fashion:
Using window.location.pathname, I am reading path of the current page, suppose the current page is /about.aspx
Then I want to add a filter in $http response by which I want to read only aboutBg.
The code I wrote can retrieve all the values, but unable to filter that. Please help.
User this function where you receive the response.
function getPageBgClass(currentPage, responseData) {
if (responseData.hasOwnProperty(currentPage))
return responseData[currentPage]
else
return "none"
}
Here is how it should be used in your promise then function
function(response) {
var bg = getPageBgClass(window.location.pathname, response.data);
//Your code here ...
}
there is no direct method to get key using value from json.
you should make sure that there are no 2 keys having same value for below code to work
function swapJsonKeyValues(input) {
var one, output = {};
for (one in input) {
if (input.hasOwnProperty(one)) {
output[input[one]] = one;
}
}
return output;
}
var originaJSON = {
"/default.aspx": "headerBg",
"/about.aspx": "aboutBg",
"/contact.aspx": "contactBg",
"/registration.aspx": "regBg",
"/clients.aspx": "clientsBg",
"/onlinesessions.aspx": "bg-white-box",
"/ondemamdsessions.aspx": "bg-grey"
}
var invertedJSON = swapJsonKeyValues(originaJSON);
var samplepathname = "aboutBg";
var page = invertedJSON[samplepathname];
[function swapJsonKeyValues from https://stackoverflow.com/a/1970193/1006780 ]