I am using confluent 0.9 rest proxy and schema registry to produce avro messages in a topic.
I am using "kafka-rest" node module. After 1000 messages it starts giving 500 internal server error. After reading though some posts I understand that this error is related to a setting for max number for subjects that can be registered for a topic and reusing the schema is the solution for it.
kafka-rest documentation says the schema reuse is done transparently. Below is the sample code I used to test. I am still getting the same error.
Could someone help me understand how I can reuse the schema?
Thanks,
Rajesh
"use strict";
var async = require('async');
var kafkaRest = require('kafka-rest');
var topicName = "avro-test";
var valueSchema = new kafkaRest.AvroSchema({
"name": "UserInfo",
"type": "record",
"fields": [
{ "name": "id", "type": "int" },
{ "name": "name", "type": "string" }]
});
var kafka = new kafkaRest({"url": "http://localhost:8082"});
var target = kafka.topic(topicName);
// Avro key and value schema.
for(var i = 0; i <= 1005; i++) {
var line = {'id': 1, 'name': 'Bob '};
target.produce(valueSchema, line, handleResponse);
}
function handleResponse( err, res) {
if (err) {
console.log("Error producing messages " + err);
}
}
Looks like a fix is coming in soon https://github.com/confluentinc/kafka-rest-node/issues/21
Meanwhile, we are using the schema Id to produce messages
Related
Basically, I am setting up a web server via Node.js and Express (I am a beginner at this) to retrieve data by reading a JSON file.
For example, this is my data.json file:
[{
"color": "black",
"category": "hue",
"type": "primary"
},
{
"color": "red",
"category": "hue",
"type": "primary"
}
]
I am trying to retrieve all of the colors by implementing this code for it to display on localhost:
router.get('/colors', function (req, res) {
fs.readFile(__dirname + '/data.json', 'utf8', function (err, data) {
data = JSON.parse(data);
res.json(data); //this displays all of the contents of data.json
})
});
router.get('/colors:name', function (req, res) {
fs.readFile(__dirname + '/data.json', 'utf8', function (err, data) {
data = JSON.parse(data);
for (var i = 0; i < data.length; i++) {
res.json(data[i][1]); //trying to display the values of color
}
})
});
How do I go about doing this?
What you are trying to do is actually pretty simple once you break it into smaller problems. Here is one way to break it down:
Load your JSON data into memory for use by your API.
Define an API route which extracts only the colours from your JSON data and sends them to the client as a JSON.
var data = [];
try {
data = JSON.parse(fs.readFileSync('/path/to/json'));
} catch (e) {
// Handle JSON parse error or file not exists error etc
data = [{
"color": "black",
"category": "hue",
"type": "primary"
},
{
"color": "red",
"category": "hue",
"type": "primary"
}
]
}
router.get('/colors', function (req, res, next) {
var colors = data.map(function (item) {
return item.color
}); // This will look look like: ["black","red"]
res.json(colors); // Send your array as a JSON array to the client calling this API
})
Some improvements in this method:
The file is read only once synchronously when the application is started and the data is cached in memory for future use.
Using Array.prototype.map Docs to extract an array of colors from the object.
Note:
You can structure the array of colors however you like and send it down as a JSON in that structure.
Examples:
var colors = data.map(function(item){return {color:item.color};}); // [{"color":"black"},{"color":"red"}]
var colors = {colors: data.map(function(item){return item.color;})} // { "colors" : ["black" ,"red"] }
Some gotchas in your code:
You are using res.json in a for loop which is incorrect as the response should only be sent once. Ideally, you would build the JS object in the structure you need by iterating over your data and send the completed object once with res.json (which I'm guessing internally JSON.stringifys the object and sends it as a response after setting the correct headers)
Reading files is an expensive operation. If you can afford to read it once and cache that data in memory, it would be efficient (Provided your data is not prohibitively large - in which case using files to store info might be inefficient to begin with)
in express, you can do in this way
router.get('/colors/:name', (req, res) => {
const key = req.params.name
const content = fs.readFileSync(__dirname + '/data.json', 'utf8')
const data = JSON.parse(content)
const values = data.reduce((values, value) => {
values.push(value[key])
return values
}, [])
// values => ['black', 'red']
res.send(values)
});
and then curl http://localhost/colors/color,
you can get ['black', 'red']
What you're looking to do is:
res.json(data[i]['color']);
If you don't really want to use the keys in the json you may want to use the Object.values function.
...
data = JSON.parse(data)
var values = []
for (var i = 0; i < data.length; i++) {
values.push(Object.values(data[i])[0]) // 0 - color, 1 - category, 2 - type
}
res.json(values) // ["black","red"]
...
You should never use fs.readFileSync in production. Any sync function will block the event loop until the execution is complete hence delaying everything afterwords (use with caution if deemed necessary). A few days back I had the worst experience myself and learnt that in a hard way.
In express you can define a route with param or query and use that to map the contents inside fs.readFile callback function.
/**
* get color by name
*
* #param {String} name name of the color
* #return {Array} array of the color data matching param
*/
router.get('/colors/:name', (req, res) => {
const color = req.params.name
const filename = __dirname + '/data.json';
fs.readFile('/etc/passwd', 'utf8', (err, data) => {
if(err){
return res.send([]); // handle any error returned by readFile function here
}
try{
data = JSON.parse(data); // parse the JSON string to array
let filtered = []; // initialise empty array
if(data.length > 0){ // we got an ARRAY of objects, right? make your check here for the array or else any map, filter, reduce, forEach function will break the app
filtered = data.filter((obj) => {
return obj.color === color; // return the object if the condition is true
});
}
return res.send(filtered); // send the response
}
catch(e){
return res.send([]); // handle any error returned from JSON.parse function here
}
});
});
To summarise, use fs.readFile asynchronous function so that the event loop is not clogged up. Inside the callback parse through the content and then return the response. return is really important or else you might end up getting Error: Can't set headers after they are sent
DISCLAIMER This code above is untested but should work. This is just to demonstrate the idea.
I think you can’t access JSON without key. You can use Foreach loop for(var name : object){} check about foreach it may help you
I am trying to use Map datatype in dynamodb to insert my JSON object. The JSON I am getting from external API is bit long and got nested Array of objects in it. (I am using nodejs.)
{
"a": "foo",
"b": "foo1",
"c": [{
"boo": 10,
"boo1": 15
}, {
"boo": 19,
"boo1": 45
}, {
"boo": 11,
"boo1": 25
}]
}
From the research i made so far it looks like i have to specify types for every single element in the json i am trying to insert/update. It make it harder since in my case the json could have anything.
If anyone experienced the same issue and know any solution for it please let me know.
You need to specify the exact types for every value only if you use the low-level AmazonDB API.
But you can use an AWS SDK which makes things much easier and you can use the json notation directly.
I didn't use node.js SDK (have experience with python SDK), but looking at the examples, this is true for node.js too.
Check this one:
var AWS = require("aws-sdk");
AWS.config.update({
region: "us-west-2",
endpoint: "http://localhost:8000"
});
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "Movies";
var year = 2015;
var title = "The Big New Movie";
var params = {
TableName:table,
Item:{
"year": year,
"title": title,
"info":{
"plot":"Something happens."
}
}
};
console.log("Adding a new item...");
docClient.put(params, function(err, data) {
if (err) {
console.error("Unable to add item. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Added item:", JSON.stringify(data, null, 2));
}
});
Here the Item inside the params passed into the docClient.put call is just a plain javascript object.
It's necessary for dynamodb to know what type of data your are inserting/updating. So dynamodb or any db allows only
datatype that is specific to that column. If you looking to parse json(with Jackson api) to specific type and insert. Below is the code
JsonParser parser = new JsonFactory().createParser(new File(
"target/resources/json_file.json"));
JsonNode rootNode = new ObjectMapper().readTree(parser);
Iterator<JsonNode> iter = rootNode.iterator();
ObjectNode currentNode;
while (iter.hasNext()) {
currentNode = (ObjectNode) iter.next();
table.putItem(new Item().withPrimaryKey("a", currentNode.path("a").asText(), "b",
currentNode.path("b").asText()).withJSON("c",currentNode.path("c").toString()));
}
I'm able to successfully connect and query data from a mySql db via a Breeze/Angular client, following the todo-angular example. I switched out the db table and the GUI and was still ok. The problem starts when I try to use a naming convention. (I don't have control over the db that I have to connect to & I really don't want to use Uppercase_Underscored_Words in my client!)
I'm getting the following exception:
/Users/Sherri/Sites/awdb-web/node_modules/breeze-sequelize/node_modules/breeze-client/breeze.debug.js:1852
throw new Error("Unable to locate a registered object by the name: " + k
^
Error: Unable to locate a registered object by the name: NamingConvention.underscoreCamelCase
at Object.__config._fetchObject (/Users/Sherri/Sites/awdb-web/node_modules/breeze-sequelize/node_modules/breeze-client/breeze.debug.js:1852:13)
at MetadataStore.proto.importMetadata (/Users/Sherri/Sites/awdb-web/node_modules/breeze-sequelize/node_modules/breeze-client/breeze.debug.js:6517:40)
at new module.exports.MetadataMapper (/Users/Sherri/Sites/awdb-web/node_modules/breeze-sequelize/MetadataMapper.js:19:8)
at SequelizeManager.importMetadata (/Users/Sherri/Sites/awdb-web/node_modules/breeze-sequelize/SequelizeManager.js:46:24)
at createSequelizeManager (/Users/Sherri/Sites/awdb-web/server/routes.js:114:8)
at /Users/Sherri/Sites/awdb-web/server/routes.js:23:27
When I take the "namingConvention": "camelCase" line out of the metadata.json file, the error goes away, but of course, the database property is not able to be correctly converted.
Here is the relevant code I use to set up the Entity Manager: (EDIT: I'm pretty sure my problem is server side and has nothing to do with this code, though)
var namingConvention = new UnderscoreCamelCaseConvention();
namingConvention.setAsDefault();
breeze.core.config.initializeAdapterInstance("uriBuilder", "json");
var serviceName = 'breeze/awdb';
var manager = new breeze.EntityManager(serviceName);
// Take any server property name and make it camelCase for the client to use.
// also, save it so that we can convert from the client back to the server's name
function UnderscoreCamelCaseConvention() {
var serverNames = {
netPoints: 'netPoints',
netPointsSpent: 'netPointsSpent'
}; // every translated server name
return new breeze.NamingConvention({
name: 'underscoreCamelCase',
clientPropertyNameToServer: clientPropertyNameToServer,
serverPropertyNameToClient: serverPropertyNameToClient
});
function clientPropertyNameToServer(clientPropertyName) {
return serverNames[clientPropertyName];
}
function serverPropertyNameToClient(serverPropertyName) {
var clientName = _.camelCase(serverPropertyName);
serverNames[clientName] = serverPropertyName;
return clientName;
}
}
And here is a snippet of my metadata.json file:
{
"metadataVersion": "1.0.5",
"namingConvention": "underscoreCamelCase",
"localQueryComparisonOptions": "caseInsensitiveSQL",
"dataServices": [
{
"serviceName": "breeze/awdb/",
"hasServerMetadata": true,
"jsonResultsAdapter": "webApi_default",
"useJsonp": false
}
],
"structuralTypes": [
{
"shortName": "person",
"namespace": "AWdb.Models",
"autoGeneratedKeyType": "Identity",
"defaultResourceName": "people",
"dataProperties": [
{
"name": "Person_ID",
"dataType": "Int32",
"isNullable": false,
"defaultValue": 0,
"isPartOfKey": true,
"validators": [
{
"name": "required"
},
{
"min": -2147483648,
"max": 2147483647,
"name": "int32"
}
]
},
{
"name": "Household_ID",
"dataType": "Int32",
"validators": [
{
"min": -2147483648,
"max": 2147483647,
"name": "int32"
}
]
},
....
]
}
],
"resourceEntityTypeMap": {"people": "person:#AWdb.Models"}
}
EDIT:
Here is code from my routes.js file that gets the metadata.
var fs = require('fs');
var breezeSequelize = require('breeze-sequelize');
var SequelizeManager = breezeSequelize.SequelizeManager;
var SequelizeQuery = breezeSequelize.SequelizeQuery;
var SequelizeSaveHandler = breezeSequelize.SequelizeSaveHandler;
var breeze = breezeSequelize.breeze;
var EntityQuery = breeze.EntityQuery;
var dbConfig = {
host: 'localhost',
user: 'xx',
password: 'xx',
dbName: 'xx'
};
var _sequelizeManager = createSequelizeManager();
// _sequelizeManager.sync(true).then(seed).then(function(){
// console.log('db init successful');
// });
exports.init = init;
function init(app) {
app.get('/breeze/awdb/Metadata', function (req, res, next) {
try {
var metadata = readMetadata();
res.send(metadata);
} catch(e){
next(e);
}
});
function createSequelizeManager() {
var metadata = readMetadata();
var sm = new SequelizeManager(dbConfig);
sm.importMetadata(metadata);
return sm;
}
function readMetadata() {
var filename = "server/AWdbMetadata.json";
if (!fs.existsSync(filename)) {
filename = "AWdbMetadata.json";
if (!fs.existsSync(filename)) {
throw new Error("Unable to locate file: " + filename);
}
}
var metadata = fs.readFileSync(filename, 'utf8');
return JSON.parse(metadata);
}
Any ideas? Should I be able to use a custom naming convention when I'm on a node.js server, using a metadata.json file instead of a .net entity framework?
If I'm looking at this correctly, then I think your issue is the metadata on the server. If I understand correctly, your table and column names follow the Uppercase_Underscored_Word pattern. The Breeze/Sequelize stack on the server currently doesn't have the ability to convert names, so you must use the names of entities and properties exactly as they are in the DB schema. Otherwise, the Breeze to Sequelize translation will fail. You can still use a naming convention on the client to turn the underscored server names into whatever you want them to be on the client.
So, you need two metadata files. One for the server that is used by the Breeze/Sequelize stack and that uses names exactly as they are in the DB and then a separate metadata file for the client, where you can do the translation.
Below mentioned sample json documents.It contains two fields.
{
"_id": "daef4a0e39c0c7a00feb721f6c4ce8b9",
"_rev": "2-8c7ef28df59ecbdaa23b536e58691416",
"name": "sukil",
"skills": "java"
}
In server.js
var express = require('express');
var app = express();
var cloudant = require('cloudant');
cloudant({account:"test", password:"test"}, function(err, cloudant) {
var alice = cloudant.use('opti-update')
alice.atomic("_design/sample", "inplace", "daef4a0e39c0c7a00feb721f6c4ce8b9", {field: "name", value: "bar"}, function (error, response) {
console.log(error+""+response);
})
})
Here _design/sample is a design document name and inplace is update function name then next is document id.It returns error is document update conflict and response is undefined.
In design document mentioned below
{
"_id": "_design/sample",
"_rev": "9-94393ee4665bdfd6fb283e3419a53f24",
"updates": {
"inplace": "function(doc,req){var field = req.body.field;var value = req.body.value;doc[field] = value;return [doc,''];}"
}
}
I want to update the data in cloudant using node cloudant module. I want to update the name field in json document.Above method i tried but it shows document update conflict error.How to resolve this?
The atomic method assumes the first parameter as the design document only. So need to explicitly specify "_design".
alice.atomic("sample", "inplace", "daef4a0e39c0c7a00feb721f6c4ce8b9", {field: "name", value: "bar"}, function (error, response) {
console.log(error+""+response);
})
This may be causing the problem.
I am building a website with NodeJS which asks for a data-file to be uploaded, then I have to check and (if needed) transform the content of this file.
The source file is a JSON or XML configuration file, I just need to ensure its content is well-formatted for the rest of the application.
I am wondering what would be the best way to check the global file's content.
I usually manipulate files with Streams, but I am not sure if it allows me to do what I want...
The source file has a similar format :
{
"parameters": [{
"name": "name",
"settings": {
"key": "value"
}
}],
"data": [{
"id": "1",
"label": "label 1",
}, {
"id": "2",
"label": "label 2"
}]
}
What I need to do is to parse the file's content, and check if the file-format is good ;
Otherwise transform the file to a well-formatted one :
// Read the file content
var parameters = [],
data = [],
p = parameters.length,
d = data.length;
// Loop on the parameters, and check the format
while (p--) {
var parameter = parameters[p];
if (name in parameter && typeof parameter.name == "string") {
// Add several rules
parameters.push(parameter);
}
}
// Do a similar control for "data".
// Then save the well-formatted parameters and data into a file
The thing is that the uploaded file might be very large...
Can I perform it with transform Streams ? Because I need to check the full-content of the file as a object !
How can I be sure a Stream transformer won't give a chunk with just a part of data, for instance ?
I'd first try something like this:
var fs = require('fs');
try {
var inputFile = require('./config.json');
} catch (e) {
console.log(e.message); // Do proper error handling.
}
// Loop on the parameters, and check the format
if (!'parameters' in inputFile) {
console.log("Got a problem here!");
}
var parameters = [];
var p = inputFile['parameters'].length;
while(p--) {
var parameter = inputFile['parameters'][p];
if ('name' in parameter && typeof parameter.name == 'string') {
// Add several rules
parameters.push(parameter);
}
}
// Do a similar control for "data".
var data = inputFile['data'];
// More code needed here...
// Then save the well-formatted parameters and data into a file
fs.writeFileSync('./data.json', JSON.stringify({parameters: parameters, data: data}, null, 4) , 'utf-8');
If you are dealing with mammoth files that cannot fit into memory, you've got a HUGELY more difficult task ahead of you. In general, you cannot guarantee that a partial read will give you enough of the JSON to parse anything out of (e.g. {"data": ["<FOUR PETABYTE STRING>"]}).