I have an array = [ 'something', 'other' ]
And I want to retrieve only the values of those 2 ids from Firebase, which contains more than 2 items ( potentially millions ), but if I do this:
var questionRef = new Firebase(fireBaseURL+"/morethanamillionitems/");
loadUID.once('value', function (dataSnapshot) {
dataSnapshot.forEach(function(childSnapshot) { // Firebase method
console.log(dataSnapshot.numChildren()); // potentially outputs 1.000.000 +
var uid = childSnapshot.name();
var childData = childSnapshot.val();
console.log(uid.indexOf('something'));
result.push(uid)
});
}
I first basically load the whole database, which is not that efficient
Now I could do:
array.forEach(key, function() {
var questionRef = new Firebase(fireBaseURL+"/morethanamillionitems/"+key);
refID = questionRef.val();
result.push(refID);
})
Or maybe:
questionRef = new Firebase(fireBaseURL+"/morethanamillionitems/");
array.forEach(key, function() {
if ( questionRef.child(key) !== null ){
refID = questionRef.val();
result.push(refID);
}
})
The last one seems the nicest, the previous one seems a bit expensive on the old RAM.
However, I apparently have to call questionRef.once('value', function(){}) each time, hence already loading the whole document-root...
Or am I misunderstanding how Firebase handles these requests? is the .numChildren() just an answer directly from the server?
Is the .forEach actually remotely executed?
I'm wondering if there is any other way to reduce traffic per request. Which brings me to another question: it seems that firebase searches locally first, but eventually will search remotely, but it's not clear when this exactly happens. Does it periodically check if something has changed? Or will that only happend when I use .on() and not .once().
Or am I using the wrong backend for this purpose? Any other suggestions? I tried hood.ie which is still very beta, looked at Parse but firebase seemed to have the simplicity I need.
(sorry for the sloppy syntax, but you can see what I intended)
[update]
I now have this:
load: function(uids){
var FB = new Firebase(URL);
uids.map(function(uid) {
var currentRef = FB.child( uid+"/_current" );
currentRef.once('value', function (each) {
eachVal = each.val()
if (eachVal !== null){
var localSave = {};
localSave[uid] = eachVal;
this.saveLocal(localSave)
} else {
console.error("Not found: [%s]", uid)
}}, function (err) { });
});
}
But I'm still wondering when the request actually happens, on .child()? or in .once() and if the latter, what is the use of .child() exactly? It seems it's only used for referencing.
Then the second thing, if I want to retrieve an array of a hundred items, this would still mean a hundred seperate requests? or does Firebase have a way of collecting requests and then send them in a batch?
In that last case .once would be a more 'conservative' option for initial retrieval, then later you could attach a .on listener if you need real-time updates.
Related
I have created a webpage with Node JS, Express JS, Mongoose and D3 JS.
In the webpage, it contains 3 pull down menus: Department, Employee, Week.
The usage of the webpage is as follows:
When 'Department' is selected, 'Employee' menu will be filtered to show only those from the selected 'Department'. The same goes to 'Week' after 'Employee' is selected.
After the 3 menus are selected and 'PLOT' button is clicked, a line chart (using d3.js) will be plotted to show the employee working hours for the month.
MongoDB Json
{ dep: '1',
emp: 'Mr A',
week: 1,
hrs: [{
{1,8},
{2,10},
...
}]
}
Here are the snippets of my codes:
routes.js
// Connect the required database and collection
var dataAll = require('./models/dataModel');
module.exports = function(app) {
app.get('/api/data', function(req, res) {
dataAll.find({}, {}, function(err, dataRes) {
res.json(dataRes);
});
}
app.get('*', function(req,res) {
res.sendfile('./index.html');
}
}
index.html
... // More codes
<div id="menuSelect1"></div>
<div id="menuSelect2"></div>
<div id="menuSelect3"></div>
...
<script src="./display.js" type='text/javascript'></script>
... // More codes
display.js
//Menu (Department,Employee,Week) Information is gathered here
queue()
.defer(d3.json, "/api/data")
.await(createPlot);
function createPlot(error, plotData) {
var myData = plotData;
var depData = d3.nest()
.key(function(d) {return d.dep;})
.rollup(function(v) {return v.length;})
.entries(myData);
selectField1 = d3.select('#menuSelect1')
.append("select")
.on("change", menu1change)
.selectAll(depData)
.enter()
.append("option")
.attr("value", function(d) {return d.key;})
.text(function(d) {return d.key;});
function menu1Change() {
//Filter Next Menu with the option chosen in this menu
... // More codes
var selectedVal = this.options[this.selectedIndex].value;
var empData = dataSet.filter(function(d) { return d.emp = selectString; });
... // More codes
}
... // More codes
}
Problem:
Functionally, it is working as expected. Problem is when the database is getting larger and larger, the loading of the page becomes very very slow (mins to load). I believe it should be due to the routing where all data is retrieved (.find({},{})) but I thought I need it because I am using it in 'display.js' to filter my menu options.
Is there a better way to do this to resolve the performance issue?
It is rarely necessary to send all the data to the client. In fact, I haven't seen an API with a single endpoint that returns the entire database to everyone.
It's hard to give you any specific solution not knowing how your data looks like, how large it is, how fast it grows etc. The performance issues may be related to querying the database, to large data transfer, or large JSON to parse by the browser.
In any case, you shouldn't send all your database to the client with no limits. Usually it is implemented with a number of records to skip and a maximum number of records to return.
Some frameworks like LoopBack does it for you, see:
https://docs.strongloop.com/display/public/LB/Skip+filter
https://docs.strongloop.com/display/public/LB/Limit+filter
If you're using Express then you'll have to implement the limits yourself.
To test the bottleneck, you can run the Mongo shell and try to run the .find({},{}) query from there to see how long it takes. You can see the transfer size and time in the browser's developer tools. This may find you narrow down the place that needs most attention, but returning the entire database no matter how large it is, is already a good place to start.
current the only way I can fetch the current component and its name from the current route is as following.
match({routes, location: request.url.path }, (error, redirectLocation, renderProps) => {
var componentName = renderProps.components[1].displayName.replace(/Connect\(|\)/g, '');
var componentObj = renderProps.components[1].WrappedComponent;
});
is there a simpler way I can do this in a simpler way?
my main reason to do this is to prefetch data for rendering serverside, its works fairly well as above but looks quite hacky.
I want to create a form on an index page that can store data via session storage. I also want to make sure that whatever data(let's say name) ... is remembered and used throughout the site with angular. I have researched pieces of this process but I do not understand how to write it or really even what it's called.
Any help in the right direction would be useful as I am in the infant stages of all of this angular business. Let me know.
The service you want is angular-local-storage.
Just configure it in your app.js file:
localStorageServiceProvider
.setStorageType('sessionStorage');
And then use it in the controller that contains whatever data you want to remember. Here is an example of a controller that loads the session storage data on initialization, and saves it when a user fires $scope.doSearch through the UI. This should give you a good place to start.
(function () {
angular.module("pstat")
.controller("homeCtrl", homeCtrl);
homeCtrl.$inject = ['$log', 'dataService', 'localStorageService', '$http'];
function homeCtrl ($log, dataService, localStorageService, $http) { {
if (localStorageService.get("query")) { //Returns null for missing 'query' cookie
//Or store the results directly if they aren't too large
//Do something with your saved query on page load, probably get data
//Example:
dataService.getData(query)
.success( function (data) {})
.error( function (err) {})
}
$scope.doSearch = function (query) {
vm.token = localStorageService.set("query", query);
//Then actually do your search
}
})
}()
There aren't many examples demonstrating indexedDB in a ServiceWorker yet, but the ones I saw were all structured like this:
const request = indexedDB.open( 'myDB', 1 );
var db;
request.onupgradeneeded = ...
request.onsuccess = function() {
db = this.result; // Average 8ms
};
self.onfetch = function(e)
{
const requestURL = new URL( e.request.url ),
path = requestURL.pathname;
if( path === '/test' )
{
const response = new Promise( function( resolve )
{
console.log( performance.now(), typeof db ); // Average 15ms
db.transaction( 'cache' ).objectStore( 'cache' ).get( 'test' ).onsuccess = function()
{
resolve( new Response( this.result, { headers: { 'content-type':'text/plain' } } ) );
}
});
e.respondWith( response );
}
}
Is this likely to fail when the ServiceWorker starts up, and if so what is a robust way of accessing indexedDB in a ServiceWorker?
Opening the IDB every time the ServiceWorker starts up is unlikely to be optimal, you'll end up opening it even when it isn't used. Instead, open the db when you need it. A singleton is really useful here (see https://github.com/jakearchibald/svgomg/blob/master/src/js/utils/storage.js#L5), so you don't need to open IDB twice if it's used twice in its lifetime.
The "activate" event is a great place to open IDB and let any "onupdateneeded" events run, as the old version of ServiceWorker is out of the way.
You can wrap a transaction in a promise like so:
var tx = db.transaction(scope, mode);
var p = new Promise(function(resolve, reject) {
tx.onabort = function() { reject(tx.error); };
tx.oncomplete = function() { resolve(); };
});
Now p will resolve/reject when the transaction completes/aborts. So you can do arbitrary logic in the tx transaction, and p.then(...) and/or pass a dependent promise into e.respondWith() or e.waitUntil() etc.
As noted by other commenters, we really do need to promisify IndexedDB. But the composition of its post-task autocommit model and the microtask queues that Promises use make it... nontrivial to do so without basically completely replacing the API. But (as an implementer and one of the spec editors) I'm actively prototyping some ideas.
I don't know of anything special about accessing IndexedDB from the context of a service worker via accessing IndexedDB via a controlled page.
Promises obviously makes your life much easier within a service worker, so I've found using something like, e.g., https://gist.github.com/inexorabletash/c8069c042b734519680c to be useful instead of the raw IndexedDB API. But it's not mandatory as long as you create and manage your own promises to reflect the state of the asynchronous IndexedDB operations.
The main thing to keep in mind when writing a fetch event handler (and this isn't specific to using IndexedDB), is that if you call event.respondWith(), you need to pass in either a Response object or a promise that resolves with a Response object. As long as you're doing that, it shouldn't matter whether your Response is constructed from IndexedDB entries or the Cache API or elsewhere.
Are you running into any actual problems with the code you posted, or was this more of a theoretical question?
Is it possible to reset a resolved jQuery object to an 'unresolved' state and kick off it's initialization and callbacks all over again?
The specific thing I'm doing is that I have a jQuery deferred wrapper over the local file system api. From there I build up higher level deferreds for the things I care about:
var getFs = defFs.requestQuota(PERSISTENT, 1024*1024)
.pipe (bytes) -> defFs.requestFs(PERSISTENT, bytes)
var getCacheContents = getFs.pipe (fileSystem) ->
defFs.getDirectory('Cache', fileSystem.root).pipe (dir) ->
defFs.readEntries(dir)
Now most of the time, when I call getCacheContents I don't mind the memo-ized values being returned, in fact I prefer it. But, on the occasion when I want to write to the cache I really would like the ability to reset that pipe and have it re-select and rescan the cache next time its accessed.
I could cobble something together from $.Callbacks but a deferred-based solution would really be ideal.
No. A Promise is by definition a thing that resolves only once - from unresolved to fulfilled OR to rejected. You will not be able to do this with jQuery's Deferreds.
What you are actually searching for are Signals. They are to be fired more than once, but provide a similiar interface. There are some implementations around, you might ceck out js-signals or wire.js.
The only solution I could find is to reset the $.Deferred object and return new Promise from that one. It works together with some internal API dirty checking (if something gets edited / deleted), but would be more performant to just reset the existing $.Deferred and let it re-resolve on the next Promise request.
An example of a possible solution is:
$.myDeferredList = [];
$.createRestorableDeferred = function(a,b) {
// JUST BY SIMPLE $.when().then();
$.myDeferredList[a] = {
deferred: $.Deferred()
, then: b
,restore : function() {
$.myDeferredList['myReady'].deferred = $.Deferred();
$.when($.myDeferredList['myReady'].deferred).then($.myDeferredList['myReady'].then);
}
,resolve : function() {
$.myDeferredList['myReady'].deferred.resolve();
}
}
$.when($.myDeferredList['myReady'].deferred).then($.myDeferredList['myReady'].then);
window[a] = $.myDeferredList['myReady'];
}
var counter = 0;
$.createRestorableDeferred('myReady', function () {
console.log('>> myReady WHEN called',++counter);
$.myDeferredList['myReady'].restore();
});
// RESOLVING ways
$.myDeferredList['myReady'].deferred.resolve();
$.myDeferredList.myReady.deferred.resolve();
myReady.resolve();
Results in console:
>> myReady WHEN called 1
>> myReady WHEN called 2
>> myReady WHEN called 3