Using data from an HTTP request in node.js

Question

My task is to take the data from http://services.swpc.noaa.gov/text/ace-swepam.txt and break / sort it into something useful. To start, I try to categorize the data so that I can use it in chart.js or something later, but when I try to print a field, it just appears as [].

var options = { host: 'services.swpc.noaa.gov', path: '/text/ace-swepam.txt', port: 80, method: 'POST' }; var req = http.request(options, function (res) { res.on('data', function (chunk) { // console.log('BODY: ' + chunk); results += chunk.toString(); //split results into an array by each new line lines = results.split("\n"); //delete header lines lines.splice(0, 18); if (lines.length <= 20) { return; } console.log(lines); }); res.on('end', function (e) { callback(); }); }); req.on('error', function (e) { console.log('problem with request: ' + e.message); }); req.end(); function callback() { for (var line in lines) { var x = []; x = lines[line].split(" "); var statuscode = x[14]; if (statuscode == 0) { if (lines[line].indexOf('-') === -1) { year.push(x[0]); month.push(x[1]); day.push(x[2]); time.push(x[4]); statusno.push(statuscode); proton.push(x[22]); bulksp.push(x[28]); iontemp.push(x[33]); } } } // console.log(year, month, day, time, statusno, proton, bulksp, iontemp) } 

Answer 1

Data rows are not displayed with tab delimiters. I expect them to be of fixed length.

Here is the first line I get. "2015 08 18 1708 57252 61680 0 2.6 45"

Instead of trying to break this line into a tab.

  fields = line.split("\t"); 

Create an array of the lengths of each field and split it using the substring method.

Here is the complete code to analyze the returned data. It gives 119 lines or 6-7 has the status! = 0 (and therefore is skipped). Then your variables each have 112 entries.

  res.on('data', function (chunk) { var fieldLengths = [0, 4, 7, 10, 16, 24, 32, 37, 48, 59, 72]; // console.log('BODY: ' + chunk); results += chunk.toString(); //split results into an array by each new line lines = results.split("\n"); // for me, the first "chunk" is incomplete. Throw it away and just use the second chunk. if (lines.length <= 20) { return; } //delete header lines lines.splice(0, 18); for (var line in lines) { console.log("entry: " + lines[line]); //split into data fields var lineText = lines[line]; var fields = []; for (var i = 0; i <= fieldLengths.length -1; i++) { fields.push(lineText.substring(fieldLengths[i], fieldLengths[i + 1])); } //if there are no problems (status code 0) //add the data to their respective fields if (fields[6] == 0) { year.push(fields[0]); month.push(fields[1]); day.push(fields[2]); time.push(fields[3]); statusno.push(fields[6]); proton.push(fields[7]); bulksp.push(fields[8]); iontemp.push(fields[9]); } } }); res.on('end', function (e) { console.log(year); }); 

});

This is easy to debug if you are using Visual Studio (the free community version will work) and add node tools for visual studio.

Let me know if the data is not entirely correct. I understand what you are trying to do, and you can customize the code if necessary.

Answer 2

 function callback() { for (var line in lines) { //split into data fields year.push(lines[line].substring(x,y));//fill in x and y //month.push.. //day.push.. } } 

or

 callback() { var x = []; for (var line in lines){ x = lines[line].split(" "); console.log(x); year.push(x[index]) // index being where year was split into x } } 

just put this function in res.on ('end'). I am not 100% sure what you are doing, hope this helps.

EDIT:

  var options = { host: 'services.swpc.noaa.gov', path: '/text/ace-swepam.txt', port: 80, method: 'POST' }; var req = http.request(options, function (res) { res.on('data', function (chunk) { // console.log('BODY: ' + chunk); results += chunk.toString(); //split results into an array by each new line lines = results.split("\n"); //delete header lines lines.splice(0, 18); }); res.on('end', function (e) { callback(); }); }); req.on('error', function (e) { console.log('problem with request: ' + e.message); }); req.end(); function callback() { for (var line in lines) { var x = []; x = lines[line].split(" "); //console.log(x); Print x and see which index of x has the vaule you want. Constant for all year.push(x[0]); month.push(x[1]); day.push(x[2]); time.push(x[4]); } //console.log(year,month,day,time); Check final result }