I'm trying to use node stream to insert 10 million records into mysql. Is there a way to do this with node stream? I'm not finding very useful or 'friendly' answers or documentation about this issue anywhere. So far I'm able to insert 45K records, but I'm getting some errors trying with a record set any bigger than that.
Also, what's the callback in the code below supposed to do here? I'm not sure where I got this code from and I'm not actually passing a call back, so, maybe that's the problem!! :D Any ideas? What would the callback actually be? Maybe the callback is supposed to take chunk and pass a chunk at a time? How could I rework this to get it to work consistently? I just don't think this code below is actually splitting the data up into chunks at all. How do I split it up into manageable chunks?
Depending on the amount of records I try this with I get different errors. The errors I am getting are:
For 50K - 80K sometimes I get this error:
Error: connect ETIMEDOUT
at Connection._handleConnectTimeout
I get this error for 100K records or above:
Error: ER_NET_PACKET_TOO_LARGE: Got a packet bigger than 'max_allowed_packet' bytes
at Query.Sequence._packetToError
This error for around 55K records:
Error: write EPIPE
at WriteWrap.afterWrite [as oncomplete] (net.js:788:14)
It's kind of wild to get 3 different errors depending on the amount of records I'm trying to insert.
Here's the code (It's working fine for 45000 records, but not for anything bigger):
var db = require('./index.js');
var faker = require('faker');
var stream = require('stream');
var songs = [];
var size = 10000000;
var songList = function(){
for (var i = 0; i < size; i++) {
var song = [i, faker.random.words(1,2), faker.name.findName(), faker.internet.url(1,50), faker.random.words(1,2), faker.random.words(1,20)];
songs.push(song);
}
console.log('songs', songs);
return songs;
}
var songSql = "INSERT INTO songs (song_id, song_name, artist, song_url, song_album, song_playlist) VALUES ?";
var songValues = songList();
var songSeed = function() {
console.log('x: ', x);
var query = db.connection.query(songSql, [songValues]).stream({highWaterMark: 5});
var testStream = new stream.Transform({highWaterMark: 5, objectMode: true});
testStream._write = function(chunk,encoding,callback) {
setTimeout(function() {
console.log('my chunk: ', chunk);
callback();
},1000);
}
// Pipe the query stream into the testStream
query.pipe(testStream)
// Monitor data events on the side to see when we pause
query.on("result",function(d,i) {
console.log("Data Sent")
})
}
songSeed();
On the MySQL server increase max_allowed_packet to 1G. There's no real downside to this.
Related
I carried out join operation to help in smoothing out my images in a Landsat ImageCollection by getting at least 3 Images in a specified time window, obtain the median and then adding those images to the ImageCollection. I expected to obtain an ImageCollection that I could still carry out some filter functions to create a time series with the median images, but it didn't work.
// 1. selecting the time window
var days = 32;
var milli = ee.Number(days).multiply(1000*24*60*60)
console.log(milli)
var join = ee.Join.saveAll({
matchesKey:"images"
})
// 2. Apply the filter
var dif_filter = ee.Filter.maxDifference({
difference:milli,
leftField:"system:time_start",
rightField:"system:time_start"
})
// 3. Apply the join
var joined_collection = join.apply({
primary: original_collection,
secondary:original_collection,
condition:dif_filter
})
// Adding the median images to our collection
function medianCalculator(image){
var imageGetter = ee.ImageCollection.fromImages(image.get("images"))
var medianImage = imageGetter.reduce(ee.Reducer.median())
return ee.Image(image).addBands(medianImage).select("NDVI_median", "EVI_median", "NDVI", "EVI")
}
// collection with smoothed Images
var collection_2 = joined_collection.map(medianCalculator)
print(collection_2)
I wanted to carry out some filters on "collection_2" but there was an error, "Line 110: collection_2.filter(...).median is not a function". What am I missing.
I suspect that after performing the Join, the ImageCollection got turned into a FeatureCollection so I tried looking for ways of converting a FeatureCollection to an ImageCollection...to no avail.
I am trying to get a list of titles that was stored in a mysql database to be put into an array, so I can put it into the various commands found in card-deck and then save it back to the database.
sql = `SELECT * FROM ${brawlname} WHERE brawlid = ("${brawlID}")`;
con.query(sql, function(err, rows, fields) {
if (err) throw err;
var playerAid = rows[0].playerAid
var playerBid = rows[0].playerBid
var playerAname = rows[0].playerAname
var playerBname = rows[0].playerBname
var deckanme = rows[0].deckname
var brawlID = rows[0].brawlid
console.log(`${deckname} found in database!`)
if (message.author.id == playerAid) {
var playerAdeck = JSON.stringify(list) //I've omitted this in various attempts and went right to .split.
console.log(playerAdeck)
var playerAdeckarray = playerAdeck.split(", ")
var maindeckA = new Deck()
maindeckA.cards(playerAdeckarray)
console.log(maindeckA)
newdeckA = maindeckA.join(", ") //error happens here.
console.log(newdeckA)
The error I get from the indicated line is
throw err; // Rethrow non-MySQL errors
TypeError: maindeckA.join is not a function
The first line above is when the list goes through Stringify, the last part is after it goes through the .card method.
So in the example text above, I then take the list of names that is obtained from a mysql query and turn it into a string that then creates the deck object. In another file, I take this list of card names, pull it from the database, shuffle it, then try and transform it into a new string and save it back all to have the same error on the .join(", ") line. The functions work from card-deck, but the way to transform it into a string to put into the database doesn't. Anytime I put something into one of the functions related to card-deck1 I can't get it to a form that works in the database. I've tried .join, .toString(), etc... and most of them either give me [object Object] or the "is not a function" error.
Thanks for any and all help! :D
I'm using a node.js + websocket + https://github.com/felixge/node-mysql.
I have a small script here:
sql.query('SELECT hash FROM dateinfo ORDER BY date LIMIT 1',function(error,erows){
if(error) throw err;
var oldhash,oldrolls;
for (var i = 0; i < rows.length; i++) {
oldhash = rows[i].hash;
oldrolls = rows[i].rolls;
};
console.log(oldhash);
});
After successful query, console prints "undefined". But if I print that variable after some time (let's say 1s), it will print "test" (as the mysql field contains "test").
I heard that this is the problem with node.js async...
I'm stuck here, what should I do?
Thanks for the help.
rows is actually undefined since your second parameter is called erows
So with TCP in AS3, I'm trying to write strings over to the server and then to the clients, but it appears to only be able to read one at a time. This is not good because I'm sending messages based on keys being pressed, and the client has an action that needs to be taken place based on what key is pressed. Since multiple keys can be pressed at a time, obviously it does not work correctly.
Client example:
if (keys[p1_jump_key])
{
p1_up = "down";
sock.writeUTF("p1_up_down");
sock.flush();
}
else
{
p1_up = "up";
sock.writeUTF("p1_up_up");
sock.flush();
}
if (keys[p1_crouch_key])
{
p1_down = "down";
sock.writeUTF("p1_down_down");
sock.flush();
}
else
{
p1_down = "up";
sock.writeUTF("p1_down_up");
sock.flush();
}
And then here is the server:
function socketDataHandler(event:ProgressEvent):void{
var socket:Socket = event.target as Socket;
var message:String = socket.readUTF();
for each (var socket:Socket in clientSockets)
{
socket.writeUTF(message);
socket.flush();
}}
And finally, here is the recieving client (I have a method that allows the server to differentiate between the two):
if(msg=="p1_down_down"){
p1_down="down";
}
if(msg=="p1_down_up"){
p1_down="up";
}
if(msg=="p1_up_down"){
p1_down="down";
}
if(msg=="p1_up_up"){
p1_down="up";
}
Now many of you already see the issue, as when the down key is up, it sends the message "p1_down_up". When the up key is up, it sends the message "p1_up_up". Both messages are sending at once when neither of them are being pressed. The receiving client is, I suppose, just getting one of the signals, or perhaps neither of them. How do I make MULTIPLE signals get wrote and read over the server? I tried using an array but you can't write those apparently. Thank you.
For anyone else who comes across this like I did trying to figure out how to accomplish this, I have to say that the other solution supplied is highly inefficient, overly complicated and too hard to read. Instead of adding a null byte to the end of your message and pushing packets into arrays, you can simply have the server send the length of the entire message to determine whether the message is in multiple packets, and to make sure you read multiple writes in the correct order.
Here is a working client receive function. I send data as compressed Objects converted to ByteArrays. Objects can store any type of data which make them a good candidate for transferring complex information from server to client.
var messageBytes: ByteArray = new ByteArray;
var messageLength: int = 0;
var readBuffer = false;
function receiveTCP(e: ProgressEvent): void {
while (e.target.bytesAvailable >= 4) {//Loop to see if there is data in the socket that is available to read.
if (messageLength == 0) {
messageLength = e.target.readUnsignedInt();//read the first 4 bytes of data to get the size of the message and store it as our message length.
readBuffer = true;//set the readBuffer to true. While this is true, the function wont be reading any more data from the server until the current message is finished being read.
}
if (messageLength <= e.target.bytesAvailable) {//check to see if our message is finished being read.
if (readBuffer == true) {//Make sure another message isn't being read. Without this check, another incoming message from the server will overwrite the current one from being executed.
e.target.readBytes(messageBytes, 0, messageLength);//reads the byte array into our messageBytes ByteArray declared at the top.
messageBytes.uncompress();//decompresses our message
var message: Object = new Object();
message = messageBytes.readObject();//Writes our ByteArray into an object again.
receiveMessage(message);//send our message to another function for the data to be executed.
messageLength = 0;//reset our messageLength for the next message.
readBuffer = false;//reset our buffer so that the client can read the next message being received, or the next message in line.
} else {
receiveTCP(e);//if there is another message in wait and the previous one isn't finished reading, execute the same function with the same event data again. Eventually the buffer will open and the message will be read.
}
} else {
break;//If the message is split up into multiple packets, this will reset the loop to continue reading the message until all the packets are accounted for.
}
}
}
This will prevent Clogging, allow you to receive multiple messages simultaneously and put together messages that were delivered in multiple packets.
Admittedly, this took a while to figure out. If you need the serverside code for sending the message, let me know.
The TCP protocol consists of a bi-directional stream of bytes, and as such message boundaries are not preserved. Due to this, if you are sending messages too often, you can end up reading from the buffer two messages instead of one.
Thus, some of the messages you are sending might be being processed together, which fails the comparison test on the receiving end.
To fix this, you can append a delimiter at the end of each message. This way, you'll know where every message you sent begins and ends, allowing you to split and process them separately. This also means you don't need to flush after every write to the buffer.
For instance, you could use a null byte (\0) as delimiter.
if(keys[p1_jump_key])
{
p1_up = "down";
sock.writeUTF("p1_up_down\0");
}
...
sock.flush();
And on the receiving end, we trim any trailing delimiters and then split the message into the smaller packets.
EDIT: Fixing the issue you described gives rise to another one: your messages might be being split over two buffer reads. I've edited my code to reflect a solution to this, by creating a temporary variable where partial messages are stored, to be prepended to the other part(s) of the message in the following reads.
// First we check to see if there are any 'parts' of a message that we've stored temporarily from the last read. If so, prepend them to the message string, which must be its continuation
if(temp != "")
{
msg = temp + msg;
temp = "";
}
// Now we begin processing our packet
var splitPackets:Array = msg.split("\0");
var packets:Array = new Array();
// Deal with incomplete message reads
if(splitPackets[splitPackets.length - 1] != "")
{
// If the last item in our array isn't empty, it means out message stream ended in a partial message. Hence, we have to store this value temporarily, and append it on the next read of the stream
temp = splitPackets[splitPackets.length - 1];
}
// If the last item in our array is empty, it means our message stream ends in a full message
for(var i = 0; i < splitPackets.length - 1; i++)
{
packets[i] = splitPackets[i];
}
for(var i = 0; i < packets.length; i++)
{
if(packets[i] == "p1_down_down")
{
// Process each packet individually here
...
// If you have already processed the packet, move on to the next
continue;
}
// Processing code for other packets
...
}
Hope this helps!
If you write a simple object to a socket:
var o:Object = new Object();
o.type = e.type;
o.params = e.params;
_socket.writeObject(o);
_socket.flush();
Then on the client do you simply use:
private function onData(e:ProgressEvent):void
{
var o:Object = _clientSocket.readObject();
}
Or do you have to implement some way of checking all of the data has been received recieved before calling .readObject()
There's 2 approaches:
If you're confident that your object will fit into one packet, you can do something like:
var fromServer:ByteArray = new ByteArray;
while( socket.bytesAvailable )
socket.readBytes( fromServer );
fromServer.position = 0;
var myObj:* = fromServer.readObject();
If you have the possibility of having multiple packet messages, then a common usage is to prepend the message with the length of the message. Something like (pseudo code):
var fromServer:ByteArray = new ByteArray();
var msgLen:int = 0;
while ( socket.bytesAvailable > 0 )
{
// if we don't have a message length, read it from the stream
if ( msgLen == 0 )
msgLen = socket.readInt();
// if our message is too big for one push
var toRead:int = ( msgLen > socket.bytesAvailable ) ? socket.bytesAvailable : msgLen;
msgLen -= toRead; // msgLen will now be 0 if it's a full message
// read the number of bytes that we want.
// fromServer.length will be 0 if it's a new message, or if we're adding more
// to a previous message, it'll be appended to the end
socket.readBytes( fromServer, fromServer.length, toRead );
// if we still have some message to come, just break
if ( msgLen != 0 )
break;
// it's a full message, create your object, then clear fromServer
}
Having your socket able to read like this will mean that multiple packet messages will be read properly as well as the fact that you won't miss any messages where 2 small messages are sent almost simultaneously (as the first message will treat it all as one message, thereby missing the second one)
Rule # 1 when dealing with TCP: it is an octet stream transfer protocol. You may never ever assume anything about how many octets (8 bit long values, commonly called bytes) you get in one go, always write code that can deal with any amount, both too few and too many. There is no gurantee that the write will not be split into multiple reads. There is also no gurantee that a single read will be from a single write.
The way I handled it was to make a call back that the server tell the client that the null bit was received.
The null bit is appended to the end of the data string you are sending to the server.
String.fromCharCode(0)
Also in your case you are doing
_socket.writeObject(o);
You should be sending a string not an object.
So Like this.
_socket.writeUTFBytes( 'Hellow World" + String.fromCharCode(0) );
NOTE *************
And one thing that most first time socket creators over look is the
fact that the first request to from the client to the server over the
port that the socket is connected on is a request for the
crossdomainpolicy.xml
If you only wish to send Objects, the simplest solution is if you send an int(size) before every object. Its not important to send the exact size, you can send a bit less. In my case, I've sent a bitmapdata, and the width and height of the object. obviously the bitmapdata's size is so big, its okay if you send only that, and ignore the rest.
var toRead=0;
protected function onSocketData(event:ProgressEvent):void{
if(toRead==0) {
toRead=socket.readInt()
}
if(socket.bytesAvailable>toRead){
var recieved:Object=socket.readObject()
/*do stuff with recieved object*/
toRead=0
}