Node JS: Make flat json from json tree

Question

I wrote a node.js script to merge all json files into a directory and save the result as a new json file. I tried to do this job to a large extent, but it has several drawbacks.

A.json

[
  {
    "id": "addEmoticon1",
    "description": "Message to greet the user.",
    "defaultMessage": "Hello, {name}!"
  },
  {
    "id": "addPhoto1",
    "description": "How are youu.",
    "defaultMessage": "How are you??"
  }
]

B.json

[
  {
    "id": "close1",
    "description": "Close it.",
    "defaultMessage": "Close!"
  }
]

In the end I need:

result.json

{
  "addEmoticon1": "Hello, {name}!",
  "addPhoto1": "How are you??",
  "close1": "Close!"
}

I wrote a node.js script:

var fs = require('fs');

function readFiles(dirname, onFileContent, onError) {
  fs.readdir(dirname, function(err, filenames) {
    if (err) {
      onError(err);
      return;
    }
    filenames.forEach(function(filename) {
      fs.readFile(dirname + filename, 'utf-8', function(err, content) {
        if (err) {
          onError(err);
          return;
        }
        onFileContent(filename, content);
      });
    });
  });
}

var data = {};
readFiles('C:/node/test/', function(filename, content) {
  data[filename] = content;
  var lines = content.split('\n');
  lines.forEach(function(line) {
    var parts = line.split('"');
    if (parts[1] == 'id') {
      fs.appendFile('result.json', parts[3]+': ', function (err) {});
    }
    if (parts[1] == 'defaultMessage') {
      fs.appendFile('result.json', parts[3]+',\n', function (err) {});
    }
  });
}, function(err) {
  throw err;
});

It retrieves the "id" and "defaultMessage", but cannot correctly add.

What I get:

result.json

addEmoticon1: addPhoto1: Hello, {name}!,
close1: How are you??,
Close!,

This output is different every time I run my script.

• Goal 1: environment elements in double quotes,

• Goal 2: Add curly brackets at the top and end

• Goal 3: no comma at the end of the last element

• Goal 4: The same output every time I run my script

Answer 1

:

function readFiles(dirname, onFileContent, onError) {

    fs.readdir(dirname, function(err, filenames) {

        /**
         * We'll store the parsed JSON data in this array
         * @type {Array}
         */
        var fileContent = [];

        if (err) {
            onError(err);
        } else {

            filenames.forEach(function(filename) {

                // Reading the file (synchronously) and storing the parsed JSON output (parsing from string to JSON object)
                var jsonObject = JSON.parse(fs.readFileSync(dirname + filename, 'utf-8'));

                // Pushing the parsed JSON output into array
                fileContent.push(jsonObject);
            });

            // Calling the callback
            onFileContent(fileContent);
        }
    });
}

readFiles('./files/',function(fileContent) {

    /**
     * We'll store the final output object here
     * @type {Object}
     */
    var output = {};

    // Loop over the JSON objects
    fileContent.forEach(function(each) {

        // Looping within each object
        for (var index in each) {

            // Copying the `id` as key and the `defaultMessage` as value and storing in output object
            output[each[index].id] = each[index].defaultMessage;
        }
    });

    // Writing the file (synchronously) after converting the JSON object back to string
    fs.writeFileSync('result.json', JSON.stringify(output));

}, function(err) {

    throw err;
});

, readFile writeFile, . JSON.parse JSON.stringify , OP.

---

UPDATE:

var fs = require('fs');

function readFiles(dirname, onEachFilename, onComplete) {

    fs.readdir(dirname, function(err, filenames) {
        if (err) {
            throw err;
        } else {

            // Prepending the dirname to each filename
            filenames.forEach(function(each, index, array) {
                array[index] = dirname + each;
            });

            // Calling aync.map which accepts these parameters:
            // filenames <-------- array of filenames
            // onEachFilename <--- function which will be applied on each filename
            // onComplete <------- function to call when the all elements of filenames array have been processed
            require('async').map(filenames, onEachFilename, onComplete);
        }
    });
}

readFiles('./files/', function(item, callback) {

    // Read the file asynchronously
    fs.readFile(item, function(err, data) {
        if (err) {
            callback(err);
        } else {
            callback(null, JSON.parse(data));
        }
    });

}, function(err, results) {

    /**
     * We'll store the final output object here
     * @type {Object}
     */
    var output = {};

    if (err) {
        throw err;
    } else {

        // Loop over the JSON objects
        results.forEach(function(each) {

            // Looping within each object
            for (var index in each) {

                // Copying the `id` as key and the `defaultMessage` as value and storing in output object
                output[each[index].id] = each[index].defaultMessage;
            }
        });

        // Writing the file (synchronously) after converting the JSON object back to string
        fs.writeFileSync('result.json', JSON.stringify(output));
    }
});

, readFile. async.map.

Answer 2

...

. .

readdirp('.')
  .fmap(filter(match(/\.json$/)))
  .fmap(map(readfilep))
  .fmap(map(fmap(JSON.parse)))
  .fmap(concatp)
  .fmap(flatten)
  .fmap(reduce(createMap)({}))
  .fmap(data=> JSON.stringify(data, null, '\t'))
  .fmap(writefilep(resolve(__dirname, 'result.json')))
  .then(filename=> console.log('wrote results to %s', filename), err=>console.error(err));

wrote results to /path/to/result.json

result.json ( c.json , , )

{
    "addEmoticon1": "Hello, {name}!",
    "addPhoto1": "How are you??",
    "close1": "Close!",
    "somethingelse": "Something!"
}

---

Promise readdir readFile writeFile

import {readdir, readFile, writeFile} from 'fs';

const readdirp = dir=>
  new Promise((pass,fail)=>
    readdir(dir, (err, filenames) =>
      err ? fail(err) : pass(mapResolve (dir) (filenames))));

const readfilep = path=>
  new Promise((pass,fail)=>
    readFile(path, 'utf8', (err,data)=>
      err ? fail(err) : pass(data)));

const writefilep = path=> data=>
  new Promise((pass,fail)=>
    writeFile(path, data, err=>
      err ? fail(err) : pass(path)));

Promises, fmap. , .

Promise.prototype.fmap = function fmap(f) {
  return new Promise((pass,fail) =>
    this.then(x=> pass(f(x)), fail));
};

const fmap = f=> x=> x.fmap(f);
const mapResolve = dir=> map(x=>resolve(dir,x));
const map = f=> xs=> xs.map(x=> f(x));
const filter = f=> xs=> xs.filter(x=> f(x));
const match = re=> s=> re.test(s);
const concatp = xs=> Promise.all(xs);
const reduce = f=> y=> xs=> xs.reduce((y,x)=> f(y)(x), y);
const flatten = reduce(y=> x=> y.concat(Array.isArray(x) ? flatten (x) : x)) ([]);

, ,

const createMap = map=> ({id, defaultMessage})=>
  Object.assign(map, {[id]: defaultMessage});

c.json

[
  {
    "id": "somethingelse",
    "description": "something",
    "defaultMessage": "Something!"
  }
]

---

" ?"

, , , . . , . , . , , .

; @BlazeSahlen . 60 , . , JSON. , / , 60 . - . ... , . / , ? -, .

---

...

() ,

import {readdir, readFile, writeFile} from 'fs';
import {resolve} from 'path';

// logp: Promise<Value> -> Void
const logp = p=> p.then(x=> console.log(x), x=> console.err(x));

// fmap : Promise<a> -> (a->b) -> Promise<b>
Promise.prototype.fmap = function fmap(f) {
  return new Promise((pass,fail) =>
    this.then(x=> pass(f(x)), fail));
};

// fmap : (a->b) -> F<a> -> F<b>
const fmap = f=> x=> x.fmap(f);

// readdirp : String -> Promise<Array<String>>
const readdirp = dir=>
  new Promise((pass,fail)=>
    readdir(dir, (err, filenames) =>
      err ? fail(err) : pass(mapResolve (dir) (filenames))));

// mapResolve : String -> Array<String> -> Array<String>
const mapResolve = dir=> map(x=>resolve(dir,x));

// map : (a->b) -> Array<a> -> Array<b>
const map = f=> xs=> xs.map(x=> f(x));

// filter : (Value -> Boolean) -> Array<Value> -> Array<Value>
const filter = f=> xs=> xs.filter(x=> f(x));

// match : RegExp -> String -> Boolean
const match = re=> s=> re.test(s);

// readfilep : String -> Promise<String>
const readfilep = path=>
  new Promise((pass,fail)=>
    readFile(path, 'utf8', (err,data)=>
      err ? fail(err) : pass(data)));

// concatp : Array<Promise<Value>> -> Array<Value>
const concatp = xs=> Promise.all(xs);

// reduce : (b->a->b) -> b -> Array<a> -> b
const reduce = f=> y=> xs=> xs.reduce((y,x)=> f(y)(x), y);

// flatten : Array<Array<Value>> -> Array<Value>
const flatten = reduce(y=> x=> y.concat(Array.isArray(x) ? flatten (x) : x)) ([]);

// writefilep : String -> Value -> Promise<String>
const writefilep = path=> data=>
  new Promise((pass,fail)=>
    writeFile(path, data, err=>
      err ? fail(err) : pass(path)));

// -----------------------------------------------------------------------------

// createMap : Object -> Object -> Object
const createMap = map=> ({id, defaultMessage})=>
  Object.assign(map, {[id]: defaultMessage});

// do it !
readdirp('.')
  .fmap(filter(match(/\.json$/)))
  .fmap(map(readfilep))
  .fmap(map(fmap(JSON.parse)))
  .fmap(concatp)
  .fmap(flatten)
  .fmap(reduce(createMap)({}))
  .fmap(data=> JSON.stringify(data, null, '\t'))
  .fmap(writefilep(resolve(__dirname, 'result.json')))
  .then(filename=> console.log('wrote results to %s', filename), err=>console.error(err));

---

- ?

, . , . , , , ! , , Promises JSON...

// Here we are reading directory '.'
// We will get a Promise<Array<String>>
// Let say the files are 'a.json', 'b.json', 'c.json', and 'run.js'
// Promise will look like this:
// Promise<['a.json', 'b.json', 'c.json', 'run.js']>
readdirp('.')

  // Now we're going to strip out any non-JSON files
  // Promise<['a.json', 'b.json', 'c.json']>
  .fmap(filter(match(/\.json$/)))

  // call `readfilep` on each of the files
  // We will get <Promise<Array<Promise<JSON>>>>
  // Don't freak out, it not that bad!
  // Promise<[Promise<JSON>, Promise<JSON>. Promise<JSON>]>
  .fmap(map(readfilep))

  // for each file Promise, we want to parse the data as JSON
  // JSON.parse returns an object, so the structure will be the same
  // except JSON will be an object!
  // Promise<[Promise<Object>, Promise<Object>, Promise<Object>]>
  .fmap(map(fmap(JSON.parse)))

  // Now we can start collapsing some of the structure
  // `concatp` will convert Array<Promise<Value>> to Array<Value>
  // We will get
  // Promise<[Object, Object, Object]>
  // Remember, we have 3 Objects; one for each parsed JSON file
  .fmap(concatp)

  // Your particular JSON structures are Arrays, which are also Objects
  // so that means `concatp` will actually return Promise<[Array, Array, Array]
  // but we'd like to flatten that
  // that way each parsed JSON file gets mushed into a single data set
  // after flatten, we will have
  // Promise<Array<Object>>
  .fmap(flatten)

  // Here where it all comes together
  // now that we have a single Promise of an Array containing all of your objects ...
  // We can simply reduce the array and create the mapping of key:values that you wish
  // `createMap` is custom tailored for the mapping you need
  // we initialize the `reduce` with an empty object, {}
  // after it runs, we will have Promise<Object>
  // where Object is your result
  .fmap(reduce(createMap)({}))

  // It all downhill from here
  // We currently have Promise<Object>
  // but before we write that to a file, we need to convert it to JSON
  // JSON.stringify(data, null, '\t') will pretty print the JSON using tab to indent
  // After this, we will have Promise<JSON>
  .fmap(data=> JSON.stringify(data, null, '\t'))

  // Now that we have a JSON, we can easily write this to a file
  // We'll use `writefilep` to write the result to `result.json` in the current working directory
  // I wrote `writefilep` to pass the filename on success
  // so when this finishes, we will have
  // Promise<Path>
  // You could have it return Promise<Void> like writeFile sends void to the callback. up to you.
  .fmap(writefilep(resolve(__dirname, 'result.json')))

  // the grand finale
  // alert the user that everything is done (or if an error occurred)
  // Remember `.then` is like a fork in the road:
  // the code will go to the left function on success, and the right on failure
  // Here, we're using a generic function to say we wrote the file out
  // If a failure happens, we write that to console.error
  .then(filename=> console.log('wrote results to %s', filename), err=>console.error(err));

!

Answer 3

- ; [a, b,...];

var res = {};
files.reduce((a, b) => a.concat(b), []).forEach(o => res[o.id] = o.defaultMessage);

---

.
onFileContent.

JSON.parse(fileContent).forEach(o => res[o.id] = o.defaultMessage);

, readFiles.
:

fs.writeFile('result.json', JSON.stringify(res));

---

, :

var fs = require('fs');

function task(dir, it, cb) {
  fs.readdir(dir, (err, names) => {
    if (err) return cb([err]);
    var errors = [], c = names.length;
    names.forEach(name => {
      fs.readFile(dir + name, 'utf-8', (err, data) => {
        if (err) return errors.push(err);
        try {
          it(JSON.parse(data)); // We get a file data!
        } catch(e) {
          errors.push('Invalid json in ' + name + ': '+e.message);
        }
        if (!--c) cb(errors); // We are finish
      });
    });
  });
}

var res = {};
task('C:/node/test/', (data) => data.forEach(o => res[o.id] = o.defaultMessage), (errors) => {
  // Some files can be wrong
  errors.forEach(err => console.error(err));
  // But we anyway write received data
  fs.writeFile('C:/node/test/result.json', JSON.stringify(res), (err) => {
    if (err) console.error(err);
    else console.log('Task finished. see results.json');
  })
});

Answer 4

, json a b:

var a = [
  {
    "id": "addEmoticon1",
    "description": "Message to greet the user.",
    "defaultMessage": "Hello, {name}!"
  },
  {
    "id": "addPhoto1",
    "description": "How are youu.",
    "defaultMessage": "How are you??"
  }
];

var b = [
  {
    "id": "close1",
    "description": "Close it.",
    "defaultMessage": "Close!"
  }
];

var c = a.concat(b);
var res = []

for (var i = 0; i < c.length; i++){
    res[ c[i].id ] = c[i].defaultMessage;
}

console.log(res);