From ed53d52c80d5deb263ddc443b6bb8690c620f674 Mon Sep 17 00:00:00 2001 From: Michael Rodrigues Date: Mon, 31 May 2021 05:58:11 +0000 Subject: [PATCH] Add `unwindIgnoreEmptyArrays` option. In order to convert the generated CSV back to the most accurate JSON possible, the module must still insert `[]` for unwound empty array values. However, there are some legitimate cases where users want to be able to have the module ignore these values so that the generated CSV is a bit more user-friendly, depending on the data being converted. This new option allows users the ability to achieve this without impacting existing use cases, as the user has opted to specify an option which may impact the ability to convert the CSV back to JSON later on. Fixes #168 --- lib/constants.json | 1 + lib/json2csv.js | 5 ++-- lib/utils.js | 8 ++--- test/config/testCsvFilesList.js | 1 + test/config/testJsonFilesList.js | 1 + test/data/csv/unwindIgnoreEmptyArrays.csv | 3 ++ test/data/json/unwindIgnoreEmptyArrays.json | 33 +++++++++++++++++++++ test/json2csv.js | 19 ++++++++---- 8 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 test/data/csv/unwindIgnoreEmptyArrays.csv create mode 100644 test/data/json/unwindIgnoreEmptyArrays.json diff --git a/lib/constants.json b/lib/constants.json index ddc4c7c..b49cb64 100644 --- a/lib/constants.json +++ b/lib/constants.json @@ -33,6 +33,7 @@ "checkSchemaDifferences": false, "expandArrayObjects": false, "unwindArrays": false, + "unwindIgnoreEmptyArrays": false, "useDateIso8601Format": false, "useLocaleFormat": false, "parseValue": null, diff --git a/lib/json2csv.js b/lib/json2csv.js index 008e18d..797f24e 100755 --- a/lib/json2csv.js +++ b/lib/json2csv.js @@ -13,7 +13,8 @@ const Json2Csv = function(options) { deeksOptions = { expandArrayObjects: expandingWithoutUnwinding, ignoreEmptyArraysWhenExpanding: expandingWithoutUnwinding, - escapeNestedDots: true + escapeNestedDots: true, + ignoreEmptyArrays: options.unwindIgnoreEmptyArrays }; /** HEADER FIELD FUNCTIONS **/ @@ -201,7 +202,7 @@ const Json2Csv = function(options) { // Unwind each of the documents at the given headerField params.headerFields.forEach((headerField) => { - params.records = utils.unwind(params.records, headerField); + params.records = utils.unwind(params.records, headerField, options); }); return retrieveHeaderFields(params.records) diff --git a/lib/utils.js b/lib/utils.js index 3671df6..ac3744a 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -216,7 +216,7 @@ function getNCharacters(str, start, n) { * @param item {any} * @param fieldPath {String} */ -function unwindItem(accumulator, item, fieldPath) { +function unwindItem(accumulator, item, fieldPath, options) { const valueToUnwind = path.evaluatePath(item, fieldPath); let cloned = deepCopy(item); @@ -227,7 +227,7 @@ function unwindItem(accumulator, item, fieldPath) { }); } else if (Array.isArray(valueToUnwind) && valueToUnwind.length === 0) { // Push an empty string so the value is empty since there are no values - path.setPath(cloned, fieldPath, ''); + path.setPath(cloned, fieldPath, options.unwindIgnoreEmptyArrays ? [] : ''); accumulator.push(cloned); } else { accumulator.push(cloned); @@ -240,10 +240,10 @@ function unwindItem(accumulator, item, fieldPath) { * @param field {String} * @returns {Array} */ -function unwind(array, field) { +function unwind(array, field, options) { const result = []; array.forEach((item) => { - unwindItem(result, item, field); + unwindItem(result, item, field, options); }); return result; } diff --git a/test/config/testCsvFilesList.js b/test/config/testCsvFilesList.js index cb115ec..117f4b9 100644 --- a/test/config/testCsvFilesList.js +++ b/test/config/testCsvFilesList.js @@ -32,6 +32,7 @@ const fs = require('fs'), {key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv'}, {key: 'unwind', file: '../data/csv/unwind.csv'}, {key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv'}, + {key: 'unwindIgnoreEmptyArrays', file: '../data/csv/unwindIgnoreEmptyArrays.csv'}, {key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv'}, {key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv'}, {key: 'localeFormat', file: '../data/csv/localeFormat.csv'}, diff --git a/test/config/testJsonFilesList.js b/test/config/testJsonFilesList.js index fb2f351..ef34bdd 100644 --- a/test/config/testJsonFilesList.js +++ b/test/config/testJsonFilesList.js @@ -25,6 +25,7 @@ module.exports = { csvEmptyLastValue: require('../data/json/csvEmptyLastValue'), unwind: require('../data/json/unwind'), unwindEmptyArray: require('../data/json/unwindEmptyArray'), + unwindIgnoreEmptyArrays: require('../data/json/unwindIgnoreEmptyArrays'), localeFormat: require('../data/json/localeFormat'), invalidParsedValues: require('../data/json/invalidParsedValues'), firstColumnWrapCRLF: require('../data/json/firstColumnWrapCRLF.json'), diff --git a/test/data/csv/unwindIgnoreEmptyArrays.csv b/test/data/csv/unwindIgnoreEmptyArrays.csv new file mode 100644 index 0000000..a4302a1 --- /dev/null +++ b/test/data/csv/unwindIgnoreEmptyArrays.csv @@ -0,0 +1,3 @@ +entities.id,entities.name,entities.siblings.entity.name,entities.siblings.relationship_set.relationships.relationship.name +4,Zeus,Exo,Senior +4,Zeus,Chrono, \ No newline at end of file diff --git a/test/data/json/unwindIgnoreEmptyArrays.json b/test/data/json/unwindIgnoreEmptyArrays.json new file mode 100644 index 0000000..0c77e43 --- /dev/null +++ b/test/data/json/unwindIgnoreEmptyArrays.json @@ -0,0 +1,33 @@ +[ + { + "entities": + { + "id": 4, + "name": "Zeus", + "siblings": [ + { + "entity": { + "name": "Exo" + }, + "relationship_set": { + "relationships": [ + { + "relationship": { + "name": "Senior" + } + } + ] + } + }, + { + "entity": { + "name": "Chrono" + }, + "relationship_set": { + "relationships": [] + } + } + ] + } + } +] \ No newline at end of file diff --git a/test/json2csv.js b/test/json2csv.js index d0a4530..5835046 100644 --- a/test/json2csv.js +++ b/test/json2csv.js @@ -656,15 +656,24 @@ function runTests(jsonTestData, csvTestData) { }); }); + // Test case for #168 + it('should ignore empty arrays when specified when unwinding arrays', (done) => { + converter.json2csv(jsonTestData.unwindIgnoreEmptyArrays, (err, csv) => { + if (err) done(err); + csv.should.equal(csvTestData.unwindIgnoreEmptyArrays); + done(); + }, { + expandArrayObjects: true, + unwindArrays: true, + unwindIgnoreEmptyArrays: true + }); + }); + // Test case for #184 it('should handle keys with nested dots when expanding and unwinding arrays', (done) => { converter.json2csv(jsonTestData.nestedDotKeysWithArrayExpandedUnwound, (err, csv) => { if (err) done(err); - - // Replace raw boolean values with quoted versions - let expectedCsv = csvTestData.nestedDotKeysWithArrayExpandedUnwound; - - csv.should.equal(expectedCsv); + csv.should.equal(csvTestData.nestedDotKeysWithArrayExpandedUnwound); done(); }, { expandArrayObjects: true,