Impossible to read Kafka message value - json

I'm using a simple producer in order to produce a message in kafka.
This message is a json object.
This is the code of my producer :
const kafka = require('kafka-node');
const Producer = kafka.Producer;
const KeyedMessage = kafka.KeyedMessage;
const Offset = kafka.Offset;
const Client = kafka.Client;
const p = 0;
const a = 0;
const topic = 'claims';
const client = new Client('192.168.99.100:2181');
const producer = new Producer(client, { requireAcks: 1 });
producer.on('error', function (err) {
done.fail(err);
});
producer.on('ready', function () {
const message = {status: 'Created', id_claims: '12345'};
producer.send([
{ topic: topic, partition: p, messages: message, attributes: a }
], function (err, result) {
expect(result).not.toBeNull();
expect(err).toBeNull();
done();
});
});
But then when i try to read the content of this message I always get
[Object object]
This is the code of my consumer :
'use strict';
const util = require('util')
const kafka = require('kafka-node');
const Consumer = kafka.Consumer;
const Offset = kafka.Offset;
const Client = kafka.Client;
const topic = 'claims';
const client = new Client('192.168.99.100:2181');
const topics = [
{topic: topic, partition: 0}
];
const options = { autoCommit: false, fetchMaxWaitMs: 1000, fetchMaxBytes: 1024 * 1024 };
const consumer = new Consumer(client, topics, options);
const offset = new Offset(client);
consumer.on('message', function (message) {
console.log(JSON.stringify(message.value));
console.log("Value: %j", message.value);
console.log("Value: %o", message.value);
console.log('Value: ' + util.inspect(message.value, 10));
console.log(util.inspect(message.value, {showHidden: false, depth: null}));
});
consumer.on('error', function (err) {
console.log(err);
});
Every method I try in the handler return the same result : [Object object]

Problem lies in below shown code:
producer.send([
{ topic: topic, partition: p, messages: message, attributes: a }
], function (err, result) {
expect(result).not.toBeNull();
expect(err).toBeNull();
done();
});
From the documentation as shown below, you cannot supply object to messages. Supplying JSON.stringfy(message) to messages will work.
{
topic: 'topicName',
messages: ['message body'], // multi messages should be a array, single message can be just a string or a KeyedMessage instance
key: 'theKey', // string or buffer, only needed when using keyed partitioner
partition: 0, // default 0
attributes: 2, // default: 0
timestamp: Date.now() // <-- defaults to Date.now() (only available with kafka v0.10 and KafkaClient only)
}

Your producer message likely needs to have a value of:
JSON.stringify(message)

Related

× Unhandled Rejection (Error): invalid BigNumber value (argument="value", value=[11,15,17,18,19,21], code=INVALID_ARGUMENT, version=bignumber/5.6.2)

I'm attempting to rework this function which currently cycles through the call of the walletOfOwner to return all owned tokenIDs, then stages each for a separate transaction.
I cannot seem to get the array pulled in a way that is accepted as it can be on etherscan and looking for some assistance.
Original Code
async function stakeall() {
var rawnfts = await vaultcontract.methods.tokensOfOwner(account).call();
const arraynft = Array.from(rawnfts.map(Number));
const tokenid = arraynft.filter(Number);
await Web3Alc.eth.getMaxPriorityFeePerGas().then((tip) => {
Web3Alc.eth.getBlock('pending').then((block) => {
var baseFee = Number(block.baseFeePerGas);
var maxPriority = Number(tip);
var maxFee = maxPriority + baseFee;
tokenid.forEach(async (id) => {
await vaultcontract.methods.stake([id])
.send({
from: account,
maxFeePerGas: maxFee,
maxPriorityFeePerGas: maxPriority
})
})
});
})
}
Reworked
async function stakeall() {
var rawnfts = await contract.methods.walletOfOwner(account).call();
const arraynft = Array.from(rawnfts.map(Number));
const tokenids = arraynft
await Web3Alc.eth.getMaxPriorityFeePerGas().then((tip) => {
Web3Alc.eth.getBlock('pending').then((block) => {
var baseFee = Number(block.baseFeePerGas);
var maxPriority = Number(tip);
var maxFee = maxPriority + baseFee;
vaultcontract.methods.stake([tokenids])
.send({
from: account,
maxFeePerGas: maxFee,
maxPriorityFeePerGas: maxPriority
})
});
})
}
I can perform this function successfully with the same array on etherscan, but am not too sure where to go from here, everything I've found states it's only possible to send one tokenId at a time, however, I can call the same function on etherscan successfully using their ui

How can you sign hyperledger-sawtooth transactions using metamask keys?

Hyperledger sawtooth uses secp256k1 ECDSA to sign transactions:
https://sawtooth.hyperledger.org/docs/core/releases/1.2.5/_autogen/txn_submit_tutorial.html?highlight=transaction%20sign
And aparently ethereum uses the same type of signature:
https://hackernoon.com/a-closer-look-at-ethereum-signatures-5784c14abecc
Thus, it would seem that because Metamask is used with Ethereum it would also work with sawtooth. However, I haven't found examples of this, and although I've tried signing transactions with web3.js and ethers.js with Metamask those signatures get rejected by Sawtooth.
It's possible, this is an example I made using web3:0.20.7:
https://github.com/le99/sawtooth-with-metamask-signatures/blob/master/src/App.js
The important function is onClick()
import './App.css';
import React, { useState } from 'react';
var ethUtil = require('ethereumjs-util')
const secp256k1 = require('secp256k1')
const CryptoJS = require('crypto-js');
const axios = require('axios').default;
const cbor = require('cbor')
const Web3 = require('web3');
//https://github.com/ethereum/web3.js/blob/0.20.7/DOCUMENTATION.md
// let web3 = new Web3(Web3.givenProvider || "ws://localhost:8545");
let web3;
if (typeof window.web3 !== 'undefined') {
web3 = new Web3(window.web3.currentProvider);
} else {
// set the provider you want from Web3.providers
web3 = new Web3(new Web3.providers.HttpProvider("http://localhost:8545"));
}
const hash = (x) =>
CryptoJS.SHA512(x).toString(CryptoJS.enc.Hex)
// https://stackoverflow.com/questions/33914764/how-to-read-a-binary-file-with-filereader-in-order-to-hash-it-with-sha-256-in-cr
function arrayBufferToWordArray(ab) {
var i8a = new Uint8Array(ab);
var a = [];
for (var i = 0; i < i8a.length; i += 4) {
a.push(i8a[i] << 24 | i8a[i + 1] << 16 | i8a[i + 2] << 8 | i8a[i + 3]);
}
return CryptoJS.lib.WordArray.create(a, i8a.length);
}
async function onClick(){
const ethereum = window.ethereum;
var from = web3.eth.accounts[0]
// var msgHash = ethUtil.keccak256(Buffer.from('An amazing message, for use with MetaMask!'))
var msgHash = Buffer.from('8144a6fa26be252b86456491fbcd43c1de7e022241845ffea1c3df066f7cfede', 'hex');
console.log(from);
let signature1 = await new Promise((resolve, reject)=>{
web3.eth.sign(from, msgHash, function (err, result) {
if (err) return reject(err)
return resolve(result)
})
});
const rpk3 = secp256k1.ecdsaRecover(Uint8Array.from(Buffer.from(signature1.slice(2, -2), 'hex')), parseInt(signature1.slice(-2), 16) - 27, Uint8Array.from(msgHash));
let publicKey = Buffer.from(rpk3, 'hex').toString('hex')
console.log(msgHash.toString('hex'));
console.log(signature1);
console.log(publicKey);
console.log();
const INT_KEY_FAMILY = 'intkey'
const INT_KEY_NAMESPACE = hash(INT_KEY_FAMILY).substring(0, 6)
const address = INT_KEY_NAMESPACE + hash('foo').slice(-64)
console.log('address:',address);
const payload = {
Verb: 'set',
Name: 'foo',
Value: 41
}
console.log('public:', publicKey);
const payloadBytes = cbor.encode(payload)
const protobuf = require('sawtooth-sdk/protobuf')
const transactionHeaderBytes = protobuf.TransactionHeader.encode({
familyName: 'intkey',
familyVersion: '1.0',
inputs: [address],
outputs: [address],
signerPublicKey: publicKey,
// In this example, we're signing the batch with the same private key,
// but the batch can be signed by another party, in which case, the
// public key will need to be associated with that key.
batcherPublicKey: publicKey,
// In this example, there are no dependencies. This list should include
// an previous transaction header signatures that must be applied for
// this transaction to successfully commit.
// For example,
// dependencies: ['540a6803971d1880ec73a96cb97815a95d374cbad5d865925e5aa0432fcf1931539afe10310c122c5eaae15df61236079abbf4f258889359c4d175516934484a'],
dependencies: [],
payloadSha512: CryptoJS.SHA512(arrayBufferToWordArray(payloadBytes)).toString(CryptoJS.enc.Hex),
nonce:"hey4"
}).finish()
let sss=CryptoJS.SHA256(arrayBufferToWordArray(transactionHeaderBytes)).toString(CryptoJS.enc.Hex);
let dataHash=Uint8Array.from(Buffer.from(sss, 'hex'));
let signature = await new Promise((resolve, reject)=>{
web3.eth.sign(from, dataHash, function (err, result) {
if (err) return reject(err)
return resolve(result)
})
});
signature = signature.slice(2, -2)
console.log('sha1:', CryptoJS.SHA512(arrayBufferToWordArray(transactionHeaderBytes)).toString(CryptoJS.enc.Hex))
console.log('signature1:', signature)
const transaction = protobuf.Transaction.create({
header: transactionHeaderBytes,
headerSignature: signature,
payload: payloadBytes
})
//--------------------------------------
//Optional
//If sending to sign outside
const txnListBytes = protobuf.TransactionList.encode({transactions:[
transaction
]}).finish()
//const txnBytes2 = transaction.finish()
let transactions = protobuf.TransactionList.decode(txnListBytes).transactions;
//----------------------------------------
//transactions = [transaction]
const batchHeaderBytes = protobuf.BatchHeader.encode({
signerPublicKey: publicKey,
transactionIds: transactions.map((txn) => txn.headerSignature),
}).finish()
//
sss=CryptoJS.SHA256(arrayBufferToWordArray(batchHeaderBytes)).toString(CryptoJS.enc.Hex);
dataHash=Uint8Array.from(Buffer.from(sss, 'hex'));
signature = await new Promise((resolve, reject)=>{
web3.eth.sign(from, dataHash, function (err, result) {
if (err) return reject(err)
return resolve(result)
})
});
signature = signature.slice(2, -2)
const batch = protobuf.Batch.create({
header: batchHeaderBytes,
headerSignature: signature,
transactions: transactions
})
const batchListBytes = protobuf.BatchList.encode({
batches: [batch]
}).finish()
console.log(Buffer.from(batchListBytes).toString('hex'));
console.log('batchListBytes has the batch bytes that ca be sent to sawtooth')
// axios.post(`${HOST}/batches`, batchListBytes, {
// headers: {'Content-Type': 'application/octet-stream'}
// })
// .then((response) => {
// console.log(response.data);
// })
// .catch((err)=>{
// console.log(err);
// });
}
The example is based on:
https://sawtooth.hyperledger.org/docs/core/releases/1.2.6/_autogen/sdk_submit_tutorial_js.html
There is a lot of low level stuff, hyperledger and Metamask represent signatures slightly differently. Also most libraries for Metamask automatically wrap the data (https://web3js.readthedocs.io/en/v1.2.11/web3-eth-accounts.html#sign), they then hash it using keccak256, and that hash is what is finnally signed with secp256k1, which is not what you need for Sawtooth.
An example where no wraping or intermediaries are used to sign is: https://github.com/danfinlay/js-eth-personal-sign-examples/blob/master/index.js

Why am I getting an internal server error when trying to post a new Activity using Design Automation API?

I am creating a simple Node application that posts a new Activity using the Forge Design Automation API. The activity is not supposed to do anything. This is just a test and I plan to delete the Activity right away. When I run this, I get an internal server error. What am I doing wrong?
const config = require(`./utils/Config`);
const CLIENT_ID = config.forge.credentials.client_id;
const CLIENT_SECRET = config.forge.credentials.client_secret;
const autoRefresh = true;
const ForgeSDK = require(`forge-apis`);
const oAuth2TwoLegged = new ForgeSDK.AuthClientTwoLegged(CLIENT_ID, CLIENT_SECRET, [`code:all`], autoRefresh);
const ActivitiesApi = new ForgeSDK.ActivitiesApi();
const activityObject = {
id: `TestActivity`,
instruction: {
CommandLineParameters: null,
Script: ``
},
appPackages: [``],
requiredEngineVersion: `20.1`,
parameters: { InputParameters: [], OutputParameters: [] },
allowedChildProcesses: [],
version: 1,
isPublic: true,
theData: null,
obj: null
};
const activity = new ForgeSDK.Activity(
activityObject.id,
activityObject.instruction,
activityObject.appPackages,
activityObject.requiredEngineVersion,
activityObject.parameters,
activityObject.allowedChildProcesses,
activityObject.version,
activityObject.isPublic,
activityObject.theData,
activityObject.obj
);
const main = async () => {
try {
await oAuth2TwoLegged.authenticate();
createActivity();
} catch (error) {
console.log(error);
}
}
const createActivity = async () => {
try {
await ActivitiesApi.createActivity(activity, oAuth2TwoLegged, oAuth2TwoLegged.getCredentials());
} catch (error) {
console.log(`Creating the activity did not work!`);
console.log(error);
}
};
main();
And here's what I get from logging the error...
Not much there, so I'm at a loss.

Node loop insert with mySQL and Mongodb

I have a form with one field that allows user to enter multiple developer id via comma delimited (ab1234,bc5678).
Once the form is submitted I perform the following tasks:
Get the the project
Loop through array of developer IDs to get their full name using mySQL
update the project using MongoDB
I'm new and sure this this is possible, The codes I have below is not working for me. Can someone please let me know if the codes below is even close.
const mongoose = require('mongoose'
const mysql = require('mysql');
// Create mySQL connection
const mySQLdb = mysql.createConnection({
host : 'localhost',
user : 'root',
password : 'root',
database : 'projects'
});
const Project = mongoose.model('project');
router.post('/developerSave', async (req, res) => {
let devList = req.body.dev_ids,
devIdArr = devList.split(','),
rData = {};
// get project
const project = await Project.findById(req.body.projectID);
mySQLdb.connect();
for(var i=0, len=devIdArr.length; i < len; i++) {
let sql = `SELECT CONCAT(first_name, ' ', last_name) as full_name FROM users WHERE id= '${devIdArr[i]}'`;
mySQLdb.query(sql, function (err, results) {
if (err) throw err;
let newDev = {
userId: devIdArr[i],
fullName: results[0].full_name
}
project.developers.unshift(newDev);
await project.save();
});
}
mySQLdb.end();
rData.success = true;
rData.msg = 'Developer was added successfully.';
res.status(200).json(rData);
});
The reason you are seeing this is because your await project.save(); is inside the callback function. Your main function will not wait for all the callbacks to complete and close the db connection. Lets look at the example below
const myCallback = (param, callback) => {
setTimeout(() => {
console.log('callback function', param);
callback();
}, 1000)
}
const myAsync = async () => {
console.log('inside async');
const result = await axios.get('http://google.com/');
for (let i = 0; i < 10; i++) {
myCallback(i, () => {
console.log('this is the actual callback function');
});
}
const result2 = await axios.get('http://bing.com/');
console.log('after second call');
}
myAsync();
The output of this is
inside async
after second call
callback function 0
this is the actual callback function
...
As you can see, the after second call is printed before the callback functions.
To solve this problem, you can wrap your callback function in a promise and resolve that once save is complete.

Reading a JSON file in NodeJs

I have a small JSON file with this content
{
"users": [
{
"id": 1593,
"name": "Foo Bar"
}
]
}
and I want to read this content by using the filesystem module. So my application looks this
const fs = require('fs');
const express = require('express');
const app = express();
app.get('/users/:id', function (req, res) {
fs.readFile('./userDb.json', 'utf8', function (err, data) {
var json = JSON.parse(data);
var users = json.users;
console.log(users[0].id); // returns 1593
console.log(req.params.id); // returns 1593
var userObj = null;
for(var i = 0; i < users.length; i++){
var currentUser = users[i];
console.log(currentUser.id); // returns 1593
if (currentUser.id === req.params.id) { // this should be fine 1593 === 1593
userObj = currentUser;
break;
}
}
console.log(userObj); // returns undefined
res.render('users', {
user: userObj
});
});
});
app.listen(3000, function () {
console.log('Server running on port 3000');
});
The log will always return 1593, when I pass it as a parameter but when I want to render my handlebars template, the assigned object is null.
userObj is null, even when trying this code
var userObj = users.find(u => u.id === req.params.id);
but I think the database is not wrong. Where did I made a mistake =?
Pretty sure that req.params.id is a String. Try :
const userObj = users.find(u => u.id === Number(req.params.id));
When you have a doubt about it :
console.log(
`First value : ${v1} - ${typeof v1}`,
`Second value : ${v2} - ${typeof v2}`,
);
Please replace
if (currentUser.id === req.params.id)
with
if (currentUser.id == req.params.id)
Since req.params.id is a string and and currentUser.id is a number, they cannot be compared strictly .
However a non-strict comparison should work fine.
console.log(userObj); // returns undefined
is called after the readFile callback, this is an asynchronous non-blocking event!
You have to move your:
res.render('users', {
user: userObj
});
at the end of readFile function