Node.js extracts the csv http file and uploads to mongoose

Question

I am very new to coding in general, so I apologize in advance if this question should be pretty obvious. Here is what I want to do, and after that I will post the code that I have used so far.

I am trying to get gzip'd csv ranking data from a website and store it in a database for a clan site that I am working on developing. As soon as I find out, I will need to receive data every 5 minutes. Capturing csv data that I was able to execute, although it stores it in a text file, and I need to save it in mongodb.

Here is my code:

var DB = require('../modules/db-settings.js'); var http = require('http'); var zlib = require('zlib'); var fs = require('fs'); var mongoose = require('mongoose'); var db = mongoose.createConnection(DB.host, DB.database, DB.port, {user: DB.user, pass: DB.password}); var request = http.get({ host: 'www.earthempires.com', path: '/ranks_feed?apicode=myapicode', port: 80, headers: { 'accept-encoding': 'gzip' } }); request.on('response', function(response) { var output = fs.createWriteStream('./output'); switch (response.headers['content-encoding']) { // or, just use zlib.createUnzip() to handle both cases case 'gzip': response.pipe(zlib.createGunzip()).pipe(output); break; default: response.pipe(output); break; } }); db.on('error', console.error.bind(console, 'connection error:')); db.once('open', function callback () { var rankSchema = new mongoose.Schema({ serverid: Number, resetid: Number, rank: Number, countryNumber: Number, name: String, land: Number, networth: Number, tag: String, gov: String, gdi: Boolean, protection: Boolean, vacation: Boolean, alive: Boolean, deleted: Boolean }) }); 

Here is an example of what csv will look like (first 5 lines of the file):

 9,386,1,451,Super Kancheong Style,22586,318793803,LaF,D,1,0,0,1,0 9,386,2,119,Storm of Swords,25365,293053897,LaF,D,1,0,0,1,0 9,386,3,33,eug gave it to mak gangnam style,43501,212637806,LaF,H,1,0,0,1,0 9,386,4,128,Justpickupgirlsdotcom,22628,201606479,LaF,H,1,0,0,1,0 9,386,5,300,One and Done,22100,196130870,LaF,H,1,0,0,1,0 

Answer 1

Hope this is not too late to help, but here is what I will do:

• Request data in CSV format and save it in memory or file.
• Parsing CSV data to convert each row into an object.
• For each object, use Model.create() to create a new record.

First you need to create a model from your schema:

 var Rank = db.model('Rank', rankSchema); 

Then you can analyze the block of CSV text (whether you read it from a file or execute it immediately after your request is up to you.) I created my own dummy variable data , since I do not have access to api, but so far your the data is CSV newline delimited strings, this should work:

 /* Data is just a block of CSV formatted text. This can be read from a file or retrieved right in the response. */ var data = '' + '9,386,1,451,Super Kancheong Style,22586,318793803,LaF,D,1,0,0,1,0\n' + '9,386,2,119,Storm of Swords,25365,293053897,LaF,D,1,0,0,1,0\n' + '9,386,3,33,eug gave it to mak gangnam style,43501,212637806,LaF,H,1,0,0,1,0\n' + '9,386,4,128,Justpickupgirlsdotcom,22628,201606479,LaF,H,1,0,0,1,0\n' + '9,386,5,300,One and Done,22100,196130870,LaF,H,1,0,0,1,0\n'; data = data.split('\n'); data.forEach(function(line) { line = line.split(','); if (line.length != 14) return; /* Create an object representation of our CSV data. */ var new_rank = { serverid: line[0], resetid: line[1], rank: line[2], countryNumber: line[3], name: line[4], land: line[5], networth: line[6], tag: line[7], gov: line[8], gdi: line[9], protection: line[10], vacation: line[11], alive: line[12], deleted: line[13] }; /* Store the new entry in MongoDB. */ Rank.create(new_rank, function(err, rank) { console.log('Created new rank!', rank); }); }); 

You can put this in a script and run it every 5 minutes using the cron job. On my Mac, I would edit my cron file with crontab -e , and I would set up a job with this line:

 */5 * * * * /path/to/node /path/to/script.js > /dev/null