Read the original http post in Nodejs

Question

I am sending an HTTP request using the http.request function, and I would like to read the entire HTTP response, such as text; that is, the raw text of the HTTP protocol. Is it possible? I wrote the code below, but it does not work.

// Set up the request console.log('Sending request'); var post_req = http.request(post_options, function(res) { res.setEncoding('utf8'); console.log('Response statusCode: ' + res.statusCode); // res.on('data', function (chunk) { // console.log('Response: ' + chunk); // }); // res.on('end', function() {}); }); post_req.on('socket', function (socket) { var response = ""; socket.on('data', function(chunk){ console.log(chunk); }); }); // post the data post_req.write(post_data); post_req.end(); 

Answer 1

If you want to access the raw HTTP message, I would suggest using net module instead and writing the request yourself. Something like this for a simple GET request:

 var net = require('net'); var host = 'stackoverflow.com', port = 80, socket = net.connect(port, host, function() { var request = "GET / HTTP/1.1\r\nHost: " + host + "\r\n\r\n", rawResponse = ""; // send http request: socket.end(request); // assume utf-8 encoding: socket.setEncoding('utf-8'); // collect raw http message: socket.on('data', function(chunk) { rawResponse += chunk; }); socket.on('end', function(){ console.log(rawResponse); }); }); 

For a POST request sending application/x-www-form-urlencoded data, you can write a request using something like:

 function writePOSTRequest (data, host, path) { return "POST " + path + " HTTP/1.1\r\n" + "Host: " + host + "\r\n" + "Content-Type: application/x-www-form-urlencoded\r\n" + "Content-Length: " + Buffer.byteLength(data) + "\r\n\r\n" + data + "\r\n\r\n"; } var data = "name1=value1&name2=value2", request = writePOSTRequest(data, host, "/path/to/resource"); 

where I use Buffer.byteLength because Content-Length requires a length in bytes rather than characters. Also, remember that data must be URL encoded.

If you know little about the format of HTTP messages, then this is a decent place to run:

http://jmarshall.com/easy/http/

In addition, if you do not know what the response encoding will be, you will have to parse the headers first, but UTF -8 is by far the most common , so this is a fairly safe bet.

Answer 2

Streams2 and Streams1 are not always able to interact well, see "Problem: streams1 and streams2 duality" in this video .

I tried to listen to the data at a lower level than the streams, and this code prints an raw HTTP response with headers for me:

 var http = require('http'); var raw = ''; console.log('Sending request'); var req = http.request({host: 'stackoverflow.com'}, function(res) { watch(res, 'res'); res.on('end', function() { console.log(raw); }); res.on('data', function(data) { // if we don't attach 'data' handler here 'end' is not called }); }); req.on('socket', function (socket) { socket.resume(); var oldOndata = socket.ondata; socket.ondata = function(buf, start, end) { raw += buf.slice(start, end).toString(); oldOndata.call(socket, buf, start, end); }; }); req.end(); 

Answer 3

Assuming that these tools are enabled in your environment, you can run an HTTP debugging proxy such as Fiddler http://www.fiddler2.com/ , which allows you to check HTTP requests and responses.