How do I transform my SQL query to a MongoDB query? - mysql

Question

I migrated timeseries data from SQL to MongoDB. I'll give you an example:
Let's say we have a measurement device with an ID, where once per minute a value gets read. So per day, we have 24 hours * 60 minutes = 1440 values for that device.
In SQL, we have 1440 single rows for this device per day:
ID Timestamp Value
400001 01.01.2017 00:00:00 ...
"" 01.01.2017 00:01:00 ...
"" ... ...
"" 01.01.2017 23:59:00 ...
I migrated the data to MongoDB where I now have one document per day, with the values distributed to 24 hour array that respectively contain 60 minute fields containing the values (and only one Timestamp with the date XX-XX-XXXX 00:00:00):
{ ID: 400001,
Timestamp: 01.01.2017 00:00:00,
Hours:
[ 0: [0: ..., 1: ..., 2: ..., ....... 59: ... ],
1: [0: ..., 1: ..., 2: ..., ....... 59: ... ],
.
.
23: [0: ..., 1: ..., 2: ..., ....... 59: ... ]
]
}
My Problem is:
I want to transform the following SQL statement to mongoDB:
SELECT (Val) AS Val, (UNIX_TIMESTAMP(DATE_FORMAT(ArrivalTime, '%Y-%m-%d %H:%i:00'))) * 1000 AS timestmp FROM database WHERE ID = 400001 AND ArrivalTime BETWEEN FROM_UNIXTIME(1470002400) AND FROM_UNIXTIME(1475272800) ORDER BY ArrivalTime ASC
Output
Since in MongoDB I only save the day Timestamp and then split the values in arrays, I don't have a Timestamp for each Value like in SQL. So if I want to for example, get the values between 01.01.2017 02:14:00 and 01.01.2017 18:38:00, how would I do that?
I made a MongoDB query that can give me the Values between two whole days:
db.getCollection('test').aggregate([{$match: {ID: '400001', $and: [ {Timestamp_day: {$gte: new ISODate("2016-08-01 00:00:00.000Z")}}, {Timestamp_day: {$lte: new ISODate("2016-10-01 00:00:00.000Z")}}]}},{$unwind:"$Hours"}, {$unwind:"$Hours"}, {$group: {_id: '$Timestamp_day', Value: {$push: "$Hours"}}}, {$sort: {_id: 1}}]);
Output
But I need it like in SQL that I can also just give out the Values for a few hours, and with the correct Timestamp given per each Values.
I hope you can help me.

Answer 1

This should get you going:
db.collection.aggregate([{
$match: {
"ID": '400001',
"Timestamp_day": {
$gte: new ISODate("2017-01-01T00:00:00.000Z"),
$lte: new ISODate("2017-01-01T00:00:00.000Z")
}
}
}, {
$unwind: {
path: "$Hours",
includeArrayIndex: "Hour"
}
}, {
$unwind: {
path: "$Hours",
includeArrayIndex: "Minute"
}
}, {
$project: {
"_id": 0, // remove the "_id" field
"Val": "$Hours", // rename "Hours" to "Val"
"Timestamp": { // "resolve" our timestamp...
$add: // ...by adding
[
{ $multiply: [ "$Hour", 60 * 60 * 1000 ] }, // ...the number of hours in milliseconds
{ $multiply: [ "$Minute", 60 * 1000 ] }, // ...plus the number of minutes in milliseconds
"$Timestamp_day", // to the "Timestamp_day" value
]
}
}
}, {
$sort: {
"Timestamp": 1 // oh well, sort by timestamp ascending
}
}]);
With an input document of
{
"_id" : ObjectId("5a0e7d096216d24dd605cdec"),
"ID" : "400001",
"Timestamp_day" : ISODate("2017-01-01T00:00:00.000Z"),
"Hours" : [
[
0.0,
0.1,
2.0
],
[
1.0,
1.1,
2.1
],
[
2.0,
2.1,
2.2
]
]
}
the results look like this:
/* 1 */
{
"Val" : 0.0,
"Timestamp" : ISODate("2017-01-01T00:00:00.000Z")
}
/* 2 */
{
"Val" : 0.1,
"Timestamp" : ISODate("2017-01-01T00:01:00.000Z")
}
/* 3 */
{
"Val" : 2.0,
"Timestamp" : ISODate("2017-01-01T00:02:00.000Z")
}
/* 4 */
{
"Val" : 1.0,
"Timestamp" : ISODate("2017-01-01T01:00:00.000Z")
}
/* 5 */
{
"Val" : 1.1,
"Timestamp" : ISODate("2017-01-01T01:01:00.000Z")
}
/* 6 */
{
"Val" : 2.1,
"Timestamp" : ISODate("2017-01-01T01:02:00.000Z")
}
/* 7 */
{
"Val" : 2.0,
"Timestamp" : ISODate("2017-01-01T02:00:00.000Z")
}
/* 8 */
{
"Val" : 2.1,
"Timestamp" : ISODate("2017-01-01T02:01:00.000Z")
}
/* 9 */
{
"Val" : 2.2,
"Timestamp" : ISODate("2017-01-01T02:02:00.000Z")
}
UPDATE:
Based on your comment you need to calculate the difference between any value and its respective preceding value. This can be done the following way - there might be nicer ways of achieving the same thing, though... The first part is almost identical to the solution above except it has an added $match stage to remove null values as per your specification.
db.collection.aggregate([{
$match: {
"ID": '400001',
"Timestamp_day": {
$gte: new ISODate("2017-01-01T00:00:00.000Z"),
$lte: new ISODate("2017-01-01T00:00:00.000Z")
}
}
}, {
$unwind: {
path: "$Hours",
includeArrayIndex: "Hour"
}
}, {
$unwind: {
path: "$Hours",
includeArrayIndex: "Minute"
}
}, {
$match: {
"Hours": { $ne: null } // get rid of all null values
}
}, {
$project: {
"_id": 0, // remove the "_id" field
"Val": "$Hours", // rename "Hours" to "Val"
"Timestamp": { // "resolve" our timestamp...
$add: // ...by adding
[
{ $multiply: [ "$Hour", 60 * 60 * 1000 ] }, // ...the number of hours in milliseconds
{ $multiply: [ "$Minute", 60 * 1000 ] }, // ...plus the number of minutes in milliseconds
"$Timestamp_day", // to the "Timestamp_day" value
]
}
}
}, {
$sort: {
"Timestamp": 1 // oh well, sort by timestamp ascending
}
}, {
$group: {
"_id": null, // throw all documents in the same aggregated document
"Docs": {
$push: "$$ROOT" // and store our documents in an array
}
}
}, {
$unwind: {
path: "$Docs", // we flatten the "values" array
includeArrayIndex: "Docs.Index", // this will give us the index of every element - there might be more elegant solutions using $map and $let...
}
}, {
$group: { // YES, unfortunately a second time... but this time we have the array index for each element
"_id": null, // throw all documents in the same aggregated document
"Docs": {
$push: "$Docs" // and store our documents in an array
}
}
}, {
$addFields: {
"Docs": {
$let: {
vars: { "shiftedArray": { $concatArrays: [ [ null ], "$Docs.Val" ] } }, // shift value array by one to the right and put a null object at the start
in: {
$map: {
input: "$Docs",
as: "d",
in: {
"Timestamp" : "$$d.Timestamp",
"Val": { $ifNull: [ { $abs: { $subtract: [ "$$d.Val", { $arrayElemAt: [ "$$shiftedArray", "$$d.Index" ] } ] } }, 0 ] }
}
}
}
}
}
}
}, {
$unwind: "$Docs"
}, {
$replaceRoot: {
newRoot: "$Docs"
}
}]);
The results using your sample data set look like this:
/* 1 */
{
"Timestamp" : ISODate("2017-01-01T00:00:00.000Z"),
"Val" : 0.0
}
/* 2 */
{
"Timestamp" : ISODate("2017-01-01T00:01:00.000Z"),
"Val" : 0.0
}
/* 3 */
{
"Timestamp" : ISODate("2017-01-01T00:02:00.000Z"),
"Val" : 2.0
}
/* 4 */
{
"Timestamp" : ISODate("2017-01-01T00:04:00.000Z"),
"Val" : 3.0
}
/* 5 */
{
"Timestamp" : ISODate("2017-01-01T00:05:00.000Z"),
"Val" : 0.0
}
/* 6 */
{
"Timestamp" : ISODate("2017-01-01T00:06:00.000Z"),
"Val" : 1.0
}

Answer 2

Eventuell könntest du mir nochmal helfen, auch ein Tipp würde reichen #dnickless.. Ich bräuchte eine Query die mir den Betrag der Differenz zum vorherig gemessenen Wert gibt (in einem bestimmten Zeitraum, zu einer bestimmten ID).
Also als Beispiel:
Timestamp_day: ISODate("2017-01-01T01:00:00.000Z"),
Hours: [
[ 1.0, 1.0, -1.0, null, 2.0, 2.0, 3.0, ... ],
[ ... ],
...
]
Und dann als output:
{
'Timestamp' : ISODate("2017-01-01T00:00:00.000Z"),
'Val' : 0.0 /* nix - 1.0 */
}
{
'Timestamp' : ISODate("2017-01-01T00:01:00.000Z"),
'Val' : 0.0 /* 1.0 - 1.0 */
}
{
'Timestamp' : ISODate("2017-01-01T00:02:00.000Z"),
'Val' : 2.0 /* 1.0 - -1.0 */
}
{
'Timestamp' : ISODate("2017-01-01T00:04:00.000Z"),
'Val' : 3.0 /* -1.0 - (null) - 2.0 */
}
{
'Timestamp' : ISODate("2017-01-01T00:05:00.000Z"),
'Val' : 0.0 /* 2.0 - 2.0 */
}
{
'Timestamp' : ISODate("2017-01-01T00:06:00.000Z"),
'Val' : 1.0 /* 2.0 - 3.0 */
}
Hoffe es ist einigermaßen verständlich was ich meine