Nodes send partial response

Question

There are 20,000 entries in the mongodb collection. I export all these entries to csv. I am sending a partial response using this:

res.writeHead(200, { "Content-Type": "application/csv", "Content-disposition": "attachment; filename='import.csv'" }); res.write(data + '0', "binary"); 

The above code is executed in batch 500. I end using this code when all records are processed.

 if (++responseCount == loopCount) { res.end(); } 

But I got this error:

Cannot set headers after sending them.

But I get a file loaded with 500 entries.

Here is my complete code.

 var exportData = function (req, res, next) { var limit = 500; var responseCount = 0; var loopCount = 1; var size = 30000; //Get 500 records at one time var getData = function (req, start, cb) { req.db.collection('items').find().skip(start).limit(limit).toArray(function (err, records) { if (err) throw err; cb(null, records); }); }; if (size > limit) { loopCount = parseInt(req.size / limit); if ((req.size % limit) != 0) { loopCount += 1; } } for (var j = 0; j < loopCount; j++) { getData(req, limit * j, function (err, records) { if (err) throw err; records.forEach(function (record) { //Process record one by one }); res.write(records); if (++responseCount == loopCount) { res.setHeader('Content-type', 'application/csv'); res.setHeader("Content-disposition", 'attachment; filename="import.csv"'); res.end(); } }); } }; 

Answer 1

Why not just flood the data? You can use the mongoose query.stream function. Example from the docs:

 // follows the nodejs 0.8 stream api Thing.find({ name: /^hello/ }).stream().pipe(res) 

The stream will follow the data stream for you. As a node.js stream, you can also listen for events:

 // manual streaming var stream = Thing.find({ name: /^hello/ }).stream(); stream.on('data', function (doc) { // do something with the mongoose document }).on('error', function (err) { // handle the error }).on('close', function () { // the stream is closed }); 

Answer 2

This statement

 res.writeHead(200, { "Content-Type": "application/csv", "Content-disposition": "attachment; filename='import.csv'" }); 

belongs to the header section when you post a response. So it sends the header.
res.end () also sends a header. This way you send the header again.

Please refer to this stackoverlflow at
EDIT CODE:

 data.pipe(resp); resp.end(); 

Please refer to this section for more pipe.

Answer 3

 var exportData = function (req, res, next) { res.writeHead(200, { 'Content-Type': 'application/csv', 'Content-Disposition': 'attachment; filename="import.csv"' }); var limit = 500; var responseCount = 0; var loopCount = 1; var size = 30000; //Get 500 records at one time var getData = function (req, start, cb) { req.db.collection('items').find().skip(start).limit(limit).toArray(function (err, records) { if (err) throw err; cb(null, records); }); }; if (size > limit) { loopCount = parseInt(req.size / limit); if ((req.size % limit) != 0) { loopCount += 1; } } for (var j = 0; j < loopCount; j++) { getData(req, limit * j, function (err, records) { if (err) throw err; records.forEach(function (record) { //Process record one by one }); res.write(records); if (++responseCount == loopCount) { res.end(); } }); } };