Geek Answers Handbook

Nodejs, Cloud Firestore Upload Tasks - Authentication Error: Error: Socket Hangs

I am encoding a function that launches API calls and requests JSON from a huge database sequentially using offsets. The JSON response is parsed, and then subsequent data internally is uploaded to our Cloud Firestore server.

Nodejs (Node 6.11.3) and the latest Firebase Admin SDK

The information is parsed as expected and prints fine on the console. However, when data is being tried to load into our Firestore database, the console is sent as spam with an error message:

Authentication Error: Error: Socket Hang

(node: 846) UnhandledPromiseRejectionWarning: rejection of an unhandled promise (rejection id: -Number-): Error: failure to receive metadata from the plugin Error: socket hung up

and sometimes:

Authentication Error: Error: Read ECONNRESET

The forEach function collects elements from loaded JSON and processes the data before loading it into the Firestore database. Each JSON has up to 1000 data elements (1000 documents) for passing through the forEach function. I understand that this can be a problem if the function is repeated until the download installation is complete?

I am new to encoding and understand that the control flow of this function is not the best. However, I cannot find the error information that the console prints. I can find a lot of information about socket hangs, but there is not one in the Auth error section.

JSON , firebase-adminsdk. / , ( ).

:

Firebase

 const admin = require('firebase-admin');
    var serviceAccount = require("JSON");
    admin.initializeApp({
    credential: admin.credential.cert(serviceAccount),
    databaseURL: "URL"
    });
    var db = admin.firestore();
    var offset = 0;
    var failed = false;

HTTP

var runFunction = function runFunction() {
    var https = require('https');
    var options = {
        host: 'website.com',
        path: (path including an offset and 1000 row specifier),
        method: 'GET',
        json: true,
        headers: {
            'content-type': 'application/json',
            'Authorization': 'Basic ' + new Buffer('username' + ':' + 'password').toString('base64')
        }
    };

HTTP- , API

if (failed === false) {
        var req = https.request(options, function (res) {
            var body = '';
            res.setEncoding('utf8');
            res.on('data', function (chunk) {
                body += chunk;
            });
            res.on('end', () => {
                console.log('Successfully processed HTTPS response');
                body = JSON.parse(body);
                if (body.hasOwnProperty('errors')) {
                    console.log('Body ->' + body)
                    console.log('API Call failed due to server error')
                    console.log('Function failed at ' + offset)
                    req.end();
                    return
                } else {
                    if (body.hasOwnProperty('result')) {
                        let result = body.result;
                        if (Object.keys(result).length === 0) {
                            console.log('Function has completed');
                            failed = true;
                            return;
                        } else {
                            result.forEach(function (item) {
                                var docRef = db.collection('collection').doc(name);
                                console.log(name);
                                var upload = docRef.set({
                                    thing: data,
                                    thing2: data,
                                })
                            });
                            console.log('Finished offset ' + offset)
                            offset = offset + 1000;
                            failed = false;
                        }
                        if (failed === false) {
                            console.log('Function will repeat with new offset');
                            console.log('offset = ' + offset);
                            req.end();
                            runFunction();
                        } else {
                            console.log('Function will terminate');
                        }
                    }
                }
            });
        });
        req.on('error', (err) => {
            console.log('Error -> ' + err)
            console.log('Function failed at ' + offset)
            console.log('Repeat from the given offset value or diagnose further')
            req.end();
        });
        req.end();
    } else {
        req.end();
    }
    };
    runFunction();

!

UPDATE

JSON, , , - 1000 100. , .

, forEach .

# 2

, async.eachSeries . , ( 9 158 000 ). , :

async.eachSeries(result, function (item, callback) {
    // result.forEach(function (item) {
    var docRef = db.collection('collection').doc(name);
    console.log(name);
    var upload = docRef.set({
      thing: data,
      thing2: data,
    }, { merge: true }).then(ref => {
        counter = counter + 1
        if (counter == result.length) {
            console.log('Finished offset ' + offset)
            offset = offset + 1000;
            console.log('Function will repeat with new offset')
            console.log('offset = ' + offset);
            failed = false;
            counter = 0
            req.end();
            runFunction();
        }
        callback()
    });
});

, :

(node: 16168) UnhandledPromiseRejectionWarning: ( : -Number-): : .

, ... . - - , ?

+6
3

Firestore - , .

, 50 , Promises , .

→ 50 node.js :

async function uploadData(dataArray) {
  try {
    const chunks = chunkArray(dataArray, 50);
    for (const [index, chunk] of chunks.entries()) {
      console.log(` --- Uploading ${index + 1} chunk started ---`);
      await uploadDataChunk(chunk);
      console.log(`---Uploading ${index + 1} chunk finished ---`);
    }
  } catch (error) {
    console.log(error)
    // Catch en error here
  }
}

function uploadDataChunk(chunk) {
  return Promise.all(
    chunk.map((item) => new Promise((resolve, reject) => {
      setTimeout(
        () => {
          console.log(`Chunk item ${item} uploaded`);
          resolve();
        },
        Math.floor(Math.random() * 500)
      );
    }))
  );
}

function chunkArray(array, chunkSize) {
  return Array.from(
    { length: Math.ceil(array.length / chunkSize) },
    (_, index) => array.slice(index * chunkSize, (index + 1) * chunkSize)
  );
}

uploadData - uploadData (data); uploadDataChunk setTimeout ( resolve()) chunk.map.

+3

promises 50 .

function Wait() {
    return new Promise(r => setTimeout(r, 50))
}

function writeDataToFirestoreParentPhones(data) {
    let chain = Promise.resolve();
    for (let i = 0; i < data.length; ++i) {
        var docRef = db.collection('parent_phones').doc(data[i].kp_ID_for_Realm);
        chain = chain.then(()=> {
            var setAda = docRef.set({
                parent_id: data[i].kf_ParentID,
                contact_number: data[i].contact_number,
                contact_type: data[i].contact_type
            }).then(ref => {
                console.log(i + ' - Added parent_phones with ID: ', data[i].kp_ID_for_Realm);
            }).catch(function(error) {
                console.error("Error writing document: ", error);
            });
        })
        .then(Wait)
    }
}
0

.

180 000 10 000 , , , Wi-Fi, .

4G . , - , , .

0
source

More articles:


All Articles