Trying to translate the following sql query to map reduce mongodb:
select
100.00 * sum(case
when p_type like 'PROMO%'
then l_extendedprice*(1-l_discount)
else 0
end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue
from
lineitem,
part
where
l_partkey = p_partkey
and l_shipdate >= date '1995-09-01'
and l_shipdate < date '1995-09-01' + interval '1' month;
I used in the collection above:
{
"_id" : ObjectId("511b7d1b3daee1b1446ecdfe"),
"l_linenumber" : 1,
"l_quantity" : 17,
"l_extendedprice" : 21168.23,
"l_discount" : 0.04,
"l_tax" : 0.02,
"l_shipdate" : ISODate("1996-03-13T03:00:00Z"),
"l_commitdate" : ISODate("1996-02-12T03:00:00Z"),
"l_receiptdate" : ISODate("1996-03-22T03:00:00Z"),
"l_comment" : "blithely regular ideas caj",
"partsupp" : {
"ps_availqty" : 6157,
"ps_supplycost" : 719.17,
"ps_comment" : "blithely ironic packages haggle quickly silent platelets",
"ps_partkey" : {
"p_partkey" : NumberLong(155190),
"p_name" : "slate lavender tan lime lawn",
"p_mfgr" : "Manufacturer#4",
"p_brand" : "Brand#44",
"p_type" : "PROMO BRUSHED NICKEL"
}
}
}
This mapreduce:
db.runCommand({
mapreduce: "lineitem",
query: {
l_shipdate: {'$gte': new Date("Sept 01, 1995"), '$lt': new Date("Oct 01, 1995")}
},
map: function() {
var data = {promo_revenue:0, divisor:0, dividendo: 0 };
var pattern = /PROMO$/;
var discount = 1 - this.l_discount;
var revenue = this.l_extendedprice * discount;
if(this.partsupp.ps_partkey.p_type.match(pattern)!= null){
data.divisor = revenue;
}else{
data.divisor = 0;
}
data.dividendo = revenue;
emit("PROMO_REVENUE", data);
},
reduce: function(key, values) {
var data = {promo_revenue:0, divisor:0, dividendo: 0 };
for (var i = 0; i < values.length; i++) {
data.divisor += values[i].divisor;
data.dividendo += values[i].dividendo;
}
var resultado = data.divisor / data.dividendo;
data.promo_revenue = 100.00 * resultado;
return data;
},
out: 'query014'
});
I retrieve this answer:
{ "_id" : "PROMO_REVENUE", "value" : { "promo_revenue" : 0, "divisor" : 0, "dividendo" : 2761949328.227077 } }
The problem:
The RegEx that i was applied, is it correct? It's strange the divisor be ever zero. And only if the divisor be ever zero the answer is correct or not?
Thanks
LIKE 'PROMO%' is not a regex, but % is a wildcard for "zero or more characters of anything." The regex equivalent would probably be /PROMO.*/. /PROMO$/ means "ends with PROMO."
Related
I'm trying to create a choroplet map with diverging data (going from -0.7 to 0.7) and can't find a way so that my legend show the correct colors. The problem is because when colors are all positive signs is always > and is sequential. Now because I have negative numbers, it doesn't work anymore. How can I correct this?
// get color depending on population differentiel value
function getColor(d) {
return d > 0.7 ? '#b2182b' :
d > 0.5 ? '#d6604d' :
d > 0.3 ? '#f4a582' :
d > 0.1 ? '#fddbc7' :
d < -0.7 ? '#2166ac' :
d < -0.5 ? '#4393c3' :
d < -0.3 ? '#92c5de' :
d < -0.1 ? '#d1e5f0' :
'#f7f7f7';
}
var legend = L.control({position: 'bottomright'});
legend.onAdd = function (map) {
var div = L.DomUtil.create('div', 'info legend'),
grades = [-0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7],
labels = [],
from, to;
for (var i = 0; i < grades.length; i++) {
from = grades[i];
to = grades[i + 1];
labels.push(
'<i style="background:' + getColor(from + 1) + '"></i> ' +
from + (to ? ' à ' + to : '+'));
}
div.innerHTML = labels.join('<br>');
return div;
};
The problem is because when colors are all positive signs is always > and is sequential. Now because I have negative numbers, it doesn't work anymore.
No, it's because you're mixing > and < without any good reason, instead of following a regular pattern of decreasing numbers, e.g.:
function getColor(d) {
return d > 0.7 ? '#b2182b' :
d > 0.5 ? '#d6604d' :
d > 0.3 ? '#f4a582' :
d > 0.1 ? '#fddbc7' :
d > -0.1 ? '#d1e5f0' :
d > -0.3 ? '#92c5de' :
d > -0.5 ? '#4393c3' :
d > -0.7 ? '#2166ac' :
'#f7f7f7';
}
Now the stop points are aligned, so every stop point is the smallest value for the color range.
I'd go even further and store the stops and range colors in a data structure, correlating the smallest end of a range with the corresponding colour:
var stops = [
{ stop: 0.7, color: '#b2182b' },
{ stop: 0.5, color: '#d6604d' },
{ stop: 0.3, color: '#f4a582' },
{ stop: 0.1, color: '#fddbc7' },
{ stop: -0.1, color: '#d1e5f0' },
{ stop: -0.3, color: '#92c5de' },
{ stop: -0.5, color: '#4393c3' },
{ stop: -0.7, color: '#2166ac' },
{ stop: -Infinity, color: '#f7f7f7' },
];
And recreate the getColor() function by looping through that data structure:
function getColor(d) {
for (var i in stops) {
if (d > stops[i].stop) { return stops[i].color; }
}
}
Note that all numbers are greater than -Infinity, so the last entry in that data structure shall work as the default case.
And create a legend by iterating through the same data structure, carrying over the range's upper stop from the previous step (and initializing that at Infinity, as that's the implicit upper stop for the first range):
var rangeMax = 'Infinity';
for (var i in stops) {
var rangeMin = stops[i].stop.toString();
var rangeColour = stops[i].color;
labels.push(
'<i style="background:' + rangeColour + '"></i> ' +
rangeMin + ' à ' + rangeMax
);
rangeMax = stops[i].stop;
}
getColor(from + 1) is the culprit. Because of the "+ 1", you will always be off your scale.
Note that even after removing it, you will be missing the lowest legend (< 0.7). You can handle it in a similar way as your highest legend.
I am testing push notification using the node-apn module in node js. All works fine except that special characters such as a single quote does not properly escape. The push notification with a single quote (i.e. we'll becomes we\'ll in the actual notification). I tried regex, mongoose.toObject, message.message.toString() to no avail.
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var mesg = message.toObject();
var msg = JSON.stringify(mesg);
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : message.message
}
var clean = message.message.toString().replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = message.message;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
I solved this issue by escaping the json formmatted single quote (which was //') in the message.message value using the replace function:
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
I have a database look like this
I'm trying to get the NamaHalaman by doing some sql Query
function getPageByMdl(modul) {
var sql = "SELECT * FROM Halaman WHERE ModulLayanan = '"+modul+"' ORDER BY Urutan ASC";
var results = [];
var sqlResultSet = db.execute(sql);
while (sqlResultSet.isValidRow()) {
results.push({
namahalaman : sqlResultSet.fieldByName('NamaHalaman')
});
sqlResultSet.next();
}
sqlResultSet.close();
return results;
}
However the result length always give me 0 although as you can see it has 2 value in it. Another problem also happen when i try to run
var sql = "SELECT Distinct ModulLayanan FROM Halaman Order ASC";
it only returned the first value from the database which is Whistleblowing system.
Oh here's my function on showing the result
function loadDetailModule() {
data = getPageByMdl(modulTemp);
if (data.length > 0) {
var tableData = [];
//get through each item
for (var i = 0; i < data.length; i++) {
var detailmodul = data[i];
//create table row
var row = Titanium.UI.createTableViewRow({
dataIndex : i,
_title : detailmodul.namahalaman,
hasChild : true,
className : 'detailmodul_halaman',
filter : detailmodul.namahalaman,
height : 70,
backgroundColor : '#fff',
});
//title label for row at index i
var titleLabel = Titanium.UI.createLabel({
text : detailmodul.namahalaman,
font : {
fontSize : 14,
fontWeight : ' bold'
},
left : 70,
top : 10,
height : 50,
width : 210,
color : '#232',
dataIndex : i
});
row.add(titleLabel);
//add our little icon to the left of the row
var iconImage = Titanium.UI.createImageView({
image : 'img/eggcooking.png',
width : 50,
height : 50,
left : 10,
top : 10
});
row.add(iconImage);
//add the row to data array
tableData.push(row);
}
// set the data to tableview's data
detailModuleTable.data = tableData;
detailModuleTable.show();
} else {
detailModuleTable.hide();
}
}
Thank you in advance
the problem is because the Titanium does not refresh your Database when you run your application.
What you need is:
1. uninstall your application in emulator
or
2. you can change the database name
Titanium.Database.install('SLTemp.sqlite', 'SLTemp1');
var db = Ti.Database.open('SLTemp1');
to
Titanium.Database.install('SLTemp.sqlite', 'SLTemp');
var db = Ti.Database.open('SLTemp');
I am trying to evaluate which Databasesystem to use for a new Project.
At the moment I compare MySQL and MongoDB for the task at hand.
I have abotu 5 Million Recoreds with 350 Numeric fields, and I have to use this data to provide different granularity levels for some graph plotting.
I pumped the data into a MongoDB and into a Mysql, and on Mysql I generated some interim tables with 10/th, 100/th and 1000/th of the granularity. The application then chooses the correct table that matches best for the current task and then queries the data there.
With this technique I can get the data fast enough ( < 100 ms).
The SQL query I use is:
SELECT from_unixtime(CAST(FLOOR(MIN(STAMP/1000)) AS SIGNED INTEGER)),
MIN(RING),MIN(STATE),CAST(FLOOR(MIN(STAMP)) as SIGNED INTEGER),AVG(w21030401)
FROM project1 GROUP BY FLOOR((stamp - 1181589892000)/60000);
I use the identical query for creating the interim tables. The only difference is, tat there are 350 wXXXXXX fields.
INSERT INTO project1_10 (TTIME,RING,STATE,STAMP,w21030401,.........)
SELECT from_unixtime(CAST(FLOOR(MIN(STAMP/1000)) AS SIGNED INTEGER)),
MIN(RING),MIN(STATE),CAST(FLOOR(MIN(STAMP)) as SIGNED INTEGER),AVG(w21030401),.......
FROM project1 GROUP BY FLOOR((stamp - 1181589892000)/60000);
Then I tried to do the same thing with MongoDB.
I pumed all the data into MongoDB and got 4,8 Million documents in the Form:
{ "_id" : ObjectId("50040b3f0cf2872a8d3af90d"), "TTIME" :
ISODate("2008-11-30T06:40:07Z"), "STAMP" : NumberLong("1228027207000"),
"STATE" : 2531, "RING" : 1, "w13010096" : 34.991, "w13010097" : 1.432,
"w23010001" : 292, "w18030180" : 84, "w18030380" : 95, "w21030002" : 51.113,
"w21030005" : 60.321, "w21030004" : 274.662, "w21030008" : 149.629,
"w21030009" : 126.565, "w21030010" : 576.296, ........... }
Then I tried to generate the Interim Documents with the following mapReduce:
keylist = [ 'w21030401', 'w13011114', .... ];
m = function (){
var result = {};
result['STAMP'] = this['STAMP'];
result['RING'] = this['RING'];
result['TTIME'] = this['TTIME'];
result['STATE'] = this['STATE'];
for(var key in keylist){
if(key in this) {
result[key] = this[key];
result['cnt_' + key] = 1;
}
}
var zone = Math.floor((this['STAMP'] - 1171004118000) / 1000000);
emit( zone , result );
};
r = function (name, values){
var result = {};
result['STAMP'] = values[0]['STAMP'];
result['RING'] = values[0]['RING'];
result['TTIME'] = values[0]['TTIME'];
result['STATE'] = values[0]['STATE'];
for(var key in keylist) {
result[key] = 0;
result['cnt_' + key] = 0;
}
for ( var i=0; i<values.length; i++ ) {
if(values[i]['STAMP'] < result['STAMP']) {
result['STAMP'] = values[i]['STAMP'];
result['TTIME'] = values[i]['TTIME'];
}
if(values[i]['RING'] < result['RING']) {
result['RING'] = values[i]['RING'];
}
if(values[i]['STATE'] < result['STATE']) {
result['STATE'] = values[i]['STATE'];
}
for(var key in keylist) {
if(key in values[i]) {
result[key] += values[i][key];
result['cnt_' + key] += values[i]['cnt_' + key];
}
}
}
return result;
};
f = function(who, val){
var result = {};
result['STAMP'] = val['STAMP'];
result['RING'] = val['RING'];
result['TTIME'] = val['TTIME'];
result['STATE'] = val['STATE'];
for(var key in keylist) {
if(key in val) {
result[key] = val[key]/val['cnt_'+key];
}
}
return result;
};
db.project1.mapReduce( m, r, { finalize : f, scope: { keylist: keylist }, out : {replace : 'project1_100'} , jsMode : false });
MySQL used 210 Seconds for the creation fo the Interim Table, MongoDB used about 4 Hours.
My Question is:
Is MongoDB not suitable for my Problem, do I just need bigger Hardware for MongoDB than for MySQL, or did I do something wrong wih my MapReduce
Thanks
Peter
[
{
"tree_id": 6,
"fields" : ["id","lft", "rgt"], // tree_id is stripped if requested via fields because redundant
"values" :
[1,1,4,[
[2,2,3,[]]
]]
}
// more could follow ...
]
above is the json code that Bobab uses to export/import nested sets.
Baobab nested set json export/import format
How can i parse a nested html list to yield json like above?
I am trying to manipulate nested lists using drag and drop
Nestable list
It has 2 functions that kind of do what i want to achieve, but my head keeps twisting around it.
toHierarchy: function(options) {
var o = $.extend({}, this.options, options),
sDepth = o.startDepthCount || 0,
ret = [];
$(this.element).children(o.items).each(function () {
var level = _recursiveItems(this);
ret.push(level);
});
//console.log(JSON.stringify(ret));
return ret;
function _recursiveItems(item) {
var id = ($(item).attr(o.attribute || 'id') || '').match(o.expression || (/(.+)[-=_](.+)/));
if (id) {
var currentItem = {"id" : id[2]};
if ($(item).children(o.listType).children(o.items).length > 0) {
currentItem.children = [];
$(item).children(o.listType).children(o.items).each(function() {
var level = _recursiveItems(this);
currentItem.children.push(level);
});
}
return currentItem;
}
}
},
toArray: function(options) {
var o = $.extend({}, this.options, options),
sDepth = o.startDepthCount || 0,
ret = [],
left = 2;
ret.push({
"item_id": o.rootID,
"parent_id": 'none',
"depth": sDepth,
"left": '1',
"right": ($(o.items, this.element).length + 1) * 2
});
$(this.element).children(o.items).each(function () {
left = _recursiveArray(this, sDepth + 1, left);
});
ret = ret.sort(function(a,b){ return (a.left - b.left); });
//console.log(JSON.stringify(ret));
return ret;
function _recursiveArray(item, depth, left) {
var right = left + 1,
id,
pid;
if ($(item).children(o.listType).children(o.items).length > 0) {
depth ++;
$(item).children(o.listType).children(o.items).each(function () {
right = _recursiveArray($(this), depth, right);
});
depth --;
}
id = ($(item).attr(o.attribute || 'id')).match(o.expression || (/(.+)[-=_](.+)/));
if (depth === sDepth + 1) {
pid = o.rootID;
} else {
var parentItem = ($(item).parent(o.listType)
.parent(o.items)
.attr(o.attribute || 'id'))
.match(o.expression || (/(.+)[-=_](.+)/));
pid = parentItem[2];
}
if (id) {
ret.push({"item_id": id[2], "parent_id": pid, "depth": depth, "left": left, "right": right});
}
left = right + 1;
return left;
}
},
If your goal is to insert that data on the database using the Baobab library then you do not need to create the JSON code with the left/right indexes yourself, which can be fairly complicated to do.
Just send to the server tree structured data and on server side iterate over it adding the objects to the database.
You could create a generic tree structure with something like this (using jQuery to have a shorter example):
function genTree(domNode){
var parentObj = {
data : { /* filled with data found in domNode, e.g. the baobab node id */ },
children: []
};
$(domNode).find('> li, > ul > li').each(function(){
parentObj.children.push(genTree(this));
});
return parentObj;
}
Then when you'll travel the structure you will use the Baobab API to add the nodes to your database (at that point you can export it to JSON if you really need it)