When to close a MongoDB database connection in Nodejs

Question

Working with Nodejs and MongoDB through Node is the native MongoDB driver. You need to get some documents and make changes, and then save them back. That's an example:

db.open(function (err, db) { db.collection('foo', function (err, collection) { var cursor = collection.find({}); cursor.each(function (err, doc) { if (doc != null) { doc.newkey = 'foo'; // Make some changes db.save(doc); // Update the document } else { db.close(); // Closing the connection } }); }); }); 

With an asynchronous nature, if the process of updating a document takes longer, then when the cursor reaches the end of the documents, the database connection is closed. Not all updates are stored in the database.

If the db.close() parameter is omitted, all documents will be updated correctly, but the application freezes, never exits.

I saw a message suggesting to use the counter to track the number of updates when it returns to zero and then closes db. But am I doing something wrong? What is the best way to handle this situation? Use db.close() to free a resource? Or do you need to open a new db connection?

Answer 1

Here is a potential solution based on a counting approach (I have not tested it and found no errors, but I must convey the idea).

The main strategy: Get a counter of the number of records that need to be updated, save each record asynchronously and a success callback, which will decrease the counter and close the database if the counter reaches 0 (when the last update is completed). Using {safe:true} , we can guarantee that every update is successful.

The mongo server will use one thread for each connection, so it is good for: a) closing unused connections or b) pool / reuse.

 db.open(function (err, db) { db.collection('foo', function (err, collection) { var cursor = collection.find({}); cursor.count(function(err,count)){ var savesPending = count; if(count == 0){ db.close(); return; } var saveFinished = function(){ savesPending--; if(savesPending == 0){ db.close(); } } cursor.each(function (err, doc) { if (doc != null) { doc.newkey = 'foo'; // Make some changes db.save(doc, {safe:true}, saveFinished); } }); }) }); }); 

Answer 2

It is best to use a join connection and then call the db.close () function in the cleanup function at the end of the application's lifetime:

 process.on('SIGINT', cleanup); process.on('SIGTERM', cleanup); 

See http://mongodb.imtqy.com/node-mongodb-native/driver-articles/mongoclient.html

A bit old stream, but anyway.

Answer 3

I found that using a counter can be applied to a simple scenario, but can be difficult in complex situations. Here is the solution that I raise when closing the database connection when the database connection is idle:

 var dbQueryCounter = 0; var maxDbIdleTime = 5000; //maximum db idle time var closeIdleDb = function(connection){ var previousCounter = 0; var checker = setInterval(function(){ if (previousCounter == dbQueryCounter && dbQueryCounter != 0) { connection.close(); clearInterval(closeIdleDb); } else { previousCounter = dbQueryCounter; } }, maxDbIdleTime); }; MongoClient.connect("mongodb://127.0.0.1:27017/testdb", function(err, connection)( if (err) throw err; connection.collection("mycollection").find({'a':{'$gt':1}}).toArray(function(err, docs) { dbQueryCounter ++; }); //do any db query, and increase the dbQueryCounter closeIdleDb(connection); )); 

This can be a general solution for any database connections. maxDbIdleTime can be set to the same value as db request timeout or longer.

It is not very elegant, but I cannot think of a better way to do this. I use NodeJs to run a script that queries MongoDb and Mysql, and the script hangs there forever if the database connections are not closed properly.

Answer 4

Based on the assumption from @mpobrien above, I found the async module to be incredibly useful in this regard. Here is an example of a template that I came up with:

 const assert = require('assert'); const async = require('async'); const MongoClient = require('mongodb').MongoClient; var mongodb; async.series( [ // Establish Covalent Analytics MongoDB connection (callback) => { MongoClient.connect('mongodb://localhost:27017/test', (err, db) => { assert.equal(err, null); mongodb = db; callback(null); }); }, // Insert some documents (callback) => { mongodb.collection('sandbox').insertMany( [{a : 1}, {a : 2}, {a : 3}], (err) => { assert.equal(err, null); callback(null); } ) }, // Find some documents (callback) => { mongodb.collection('sandbox').find({}).toArray(function(err, docs) { assert.equal(err, null); console.dir(docs); callback(null); }); } ], () => { mongodb.close(); } ); 

Answer 5

I came up with a solution that uses a counter. This is independent of the call to count () and does not wait for a timeout. It will close db after all documents in each () have been exhausted.

 var mydb = {}; // initialize the helper object. mydb.cnt = {}; // init counter to permit multiple db objects. mydb.open = function(db) // call open to inc the counter. { if( !mydb.cnt[db.tag] ) mydb.cnt[db.tag] = 1; else mydb.cnt[db.tag]++; }; mydb.close = function(db) // close the db when the cnt reaches 0. { mydb.cnt[db.tag]--; if ( mydb.cnt[db.tag] <= 0 ) { delete mydb.cnt[db.tag]; return db.close(); } return null; }; 

So that every time you make a call like db.each () or db.save (), you should use these methods to ensure db is ready during operation and close when finished.

Example from OP:

 foo = db.collection('foo'); mydb.open(db); // *** Add here to init the counter.** foo.find({},function(err,cursor) { if( err ) throw err; cursor.each(function (err, doc) { if( err ) throw err; if (doc != null) { doc.newkey = 'foo'; mydb.open(db); // *** Add here to prevent from closing prematurely ** foo.save(doc, function(err,count) { if( err ) throw err; mydb.close(db); // *** Add here to close when done. ** }); } else { mydb.close(db); // *** Close like this instead. ** } }); }); 

Now this assumes that from the second to the last callback from each one it passes through mydb.open () before the last callback from each one goes to mydb.close () .... so of course let me know if this is a problem.

So: put mydb.open (db) before calling db and put mydb.close (db) at the callback return point or after calling db (depending on the type of call).

It seems to me that this counter should be maintained inside the db object, but this is my current solution. Perhaps we could create a new object that takes db in the constructor and wraps mongodb functions to better handle the result.

Answer 6

Here is the solution I came up with. It avoids using toArray, and it is rather short and sweet:

 var MongoClient = require('mongodb').MongoClient; MongoClient.connect("mongodb://localhost:27017/mydb", function(err, db) { let myCollection = db.collection('myCollection'); let query = {}; // fill in your query here let i = 0; myCollection.count(query, (err, count) => { myCollection.find(query).forEach((doc) => { // do stuff here if (++i == count) db.close(); }); }); });