diff --git a/package-lock.json b/package-lock.json index b9b01ea..1d63a01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1558,9 +1558,9 @@ "dev": true }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", diff --git a/package.json b/package.json index 4614c73..c675001 100755 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "compile": "tsc -p tsconfig.build.json", "coverage": "nyc npm run test", "lint": "eslint --ext .js,.ts src test", + "lint:fix": "eslint --fix --ext .js,.ts src test", "prepublishOnly": "npm run build", "test": "mocha -r ts-node/register test/index.ts" }, @@ -59,4 +60,4 @@ "node": ">= 16" }, "license": "MIT" -} +} \ No newline at end of file diff --git a/src/csv2json.ts b/src/csv2json.ts index e1c1734..c1c9268 100644 --- a/src/csv2json.ts +++ b/src/csv2json.ts @@ -5,7 +5,7 @@ import { excelBOM } from './constants'; import type { Csv2JsonParams, FullCsv2JsonOptions, HeaderField } from './types'; import * as utils from './utils'; -export const Csv2Json = function(options: FullCsv2JsonOptions) { +export const Csv2Json = function (options: FullCsv2JsonOptions) { const escapedWrapDelimiterRegex = new RegExp(options.delimiter.wrap + options.delimiter.wrap, 'g'), excelBOMRegex = new RegExp('^' + excelBOM), valueParserFn = options.parseValue && typeof options.parseValue === 'function' ? options.parseValue : JSON.parse; @@ -166,9 +166,18 @@ export const Csv2Json = function(options: FullCsv2JsonOptions) { if (utils.getNCharacters(csv, index + 1, eolDelimiterLength) === options.delimiter.eol) { index += options.delimiter.eol.length + 1; // Skip past EOL } - } - - else if ((charBefore !== options.delimiter.wrap || stateVariables.justParsedDoubleQuote && charBefore === options.delimiter.wrap) && + } else if (charBefore === options.delimiter.field && character === options.delimiter.wrap && charAfter === options.delimiter.eol) { + // We reached the start of a wrapped new field that begins with an EOL delimiter + + // Retrieve the remaining value and add it to the split line list of values + splitLine.push(csv.substring(stateVariables.startIndex, index - 1)); + + stateVariables.startIndex = index; + stateVariables.parsingValue = true; + stateVariables.insideWrapDelimiter = true; + stateVariables.justParsedDoubleQuote = true; + index += 1; + } else if ((charBefore !== options.delimiter.wrap || stateVariables.justParsedDoubleQuote && charBefore === options.delimiter.wrap) && character === options.delimiter.wrap && utils.getNCharacters(csv, index + 1, eolDelimiterLength) === options.delimiter.eol) { // If we reach a wrap which is not preceded by a wrap delim and the next character is an EOL delim (ie. *"\n) diff --git a/test/config/testCsvFilesList.ts b/test/config/testCsvFilesList.ts index d94010a..0b5f226 100644 --- a/test/config/testCsvFilesList.ts +++ b/test/config/testCsvFilesList.ts @@ -2,60 +2,61 @@ import { readFileSync } from 'fs'; import path from 'path'; const csvFileConfig = [ - {key: 'noData', file: '../data/csv/noData.csv'}, - {key: 'singleDocument', file: '../data/csv/singleDocument.csv'}, - {key: 'array', file: '../data/csv/array.csv'}, - {key: 'arrayObjects', file: '../data/csv/arrayObjects.csv'}, - {key: 'arrayMixedObjNonObj', file: '../data/csv/arrayMixedObjNonObj.csv'}, - {key: 'arraySingleArray', file: '../data/csv/arraySingleArray.csv'}, - {key: 'date', file: '../data/csv/date.csv'}, - {key: 'null', file: '../data/csv/null.csv'}, - {key: 'undefined', file: '../data/csv/undefined.csv'}, - {key: 'nested', file: '../data/csv/nested.csv'}, - {key: 'nestedMissingField', file: '../data/csv/nestedMissingField.csv'}, - {key: 'comma', file: '../data/csv/comma.csv'}, - {key: 'quotes', file: '../data/csv/quotes.csv'}, - {key: 'quotesHeader', file: '../data/csv/quotesHeader.csv'}, - {key: 'quotesAndCommas', file: '../data/csv/quotesAndCommas.csv'}, - {key: 'eol', file: '../data/csv/eol.csv'}, - {key: 'assortedValues', file: '../data/csv/assortedValues.csv'}, - {key: 'trimFields', file: '../data/csv/trimFields.csv'}, - {key: 'trimmedFields', file: '../data/csv/trimmedFields.csv'}, - {key: 'trimHeader', file: '../data/csv/trimHeader.csv'}, - {key: 'trimmedHeader', file: '../data/csv/trimmedHeader.csv'}, - {key: 'excelBOM', file: '../data/csv/excelBOM.csv'}, - {key: 'specifiedKeys', file: '../data/csv/specifiedKeys.csv'}, - {key: 'specifiedKeysNoData', file: '../data/csv/specifiedKeysNoData.csv'}, - {key: 'extraLine', file: '../data/csv/extraLine.csv'}, - {key: 'noHeader', file: '../data/csv/noHeader.csv'}, - {key: 'sortedHeader', file: '../data/csv/sortedHeader.csv'}, - {key: 'sortedHeaderCustom', file: '../data/csv/sortedHeaderCustom.csv'}, - {key: 'emptyFieldValues', file: '../data/csv/emptyFieldValues.csv'}, - {key: 'quotedEmptyFieldValue', file: '../data/csv/quotedEmptyFieldValue.csv'}, - {key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv'}, - {key: 'unwind', file: '../data/csv/unwind.csv'}, - {key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv'}, - {key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv'}, - {key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv'}, - {key: 'localeFormat', file: '../data/csv/localeFormat.csv'}, - {key: 'invalidParsedValues', file: '../data/csv/invalidParsedValues.csv'}, - {key: 'firstColumnWrapCRLF', file: '../data/csv/firstColumnWrapCRLF.csv'}, - {key: 'emptyLastFieldValue', file: '../data/csv/emptyLastFieldValue.csv'}, - {key: 'emptyLastFieldValueNoEol', file: '../data/csv/emptyLastFieldValueNoEol.csv'}, - {key: 'lastCharFieldDelimiter', file: '../data/csv/lastCharFieldDelimiter.csv'}, - {key: 'nativeMapMethod', file: '../data/csv/nativeMapMethod.csv'}, - {key: 'nestedDotKeys', file: '../data/csv/nestedDotKeys.csv'}, - {key: 'nestedDotKeysWithArray', file: '../data/csv/nestedDotKeysWithArray.csv'}, - {key: 'nestedDotKeysWithArrayExpandedUnwound', file: '../data/csv/nestedDotKeysWithArrayExpandedUnwound.csv'}, - {key: 'emptyColumns', file: '../data/csv/emptyColumns.csv'}, - {key: 'quotedFieldWithNewline', file: '../data/csv/quotedFieldWithNewline.csv'}, - {key: 'falsyValues', file: '../data/csv/falsyValues.csv'}, - {key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv'}, - {key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv'}, - {key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv'}, - {key: 'wildcardMatch', file: '../data/csv/wildcardMatch.csv'}, - {key: 'arrayIndexesAsKeys', file: '../data/csv/arrayIndexesAsKeys.csv'}, - {key: 'keyWithEndingDot', file:'../data/csv/keyWithEndingDot.csv'}, + { key: 'noData', file: '../data/csv/noData.csv' }, + { key: 'singleDocument', file: '../data/csv/singleDocument.csv' }, + { key: 'array', file: '../data/csv/array.csv' }, + { key: 'arrayObjects', file: '../data/csv/arrayObjects.csv' }, + { key: 'arrayMixedObjNonObj', file: '../data/csv/arrayMixedObjNonObj.csv' }, + { key: 'arraySingleArray', file: '../data/csv/arraySingleArray.csv' }, + { key: 'date', file: '../data/csv/date.csv' }, + { key: 'null', file: '../data/csv/null.csv' }, + { key: 'undefined', file: '../data/csv/undefined.csv' }, + { key: 'nested', file: '../data/csv/nested.csv' }, + { key: 'nestedMissingField', file: '../data/csv/nestedMissingField.csv' }, + { key: 'comma', file: '../data/csv/comma.csv' }, + { key: 'quotes', file: '../data/csv/quotes.csv' }, + { key: 'quotesHeader', file: '../data/csv/quotesHeader.csv' }, + { key: 'quotesAndCommas', file: '../data/csv/quotesAndCommas.csv' }, + { key: 'eol', file: '../data/csv/eol.csv' }, + { key: 'assortedValues', file: '../data/csv/assortedValues.csv' }, + { key: 'trimFields', file: '../data/csv/trimFields.csv' }, + { key: 'trimmedFields', file: '../data/csv/trimmedFields.csv' }, + { key: 'trimHeader', file: '../data/csv/trimHeader.csv' }, + { key: 'trimmedHeader', file: '../data/csv/trimmedHeader.csv' }, + { key: 'excelBOM', file: '../data/csv/excelBOM.csv' }, + { key: 'specifiedKeys', file: '../data/csv/specifiedKeys.csv' }, + { key: 'specifiedKeysNoData', file: '../data/csv/specifiedKeysNoData.csv' }, + { key: 'extraLine', file: '../data/csv/extraLine.csv' }, + { key: 'noHeader', file: '../data/csv/noHeader.csv' }, + { key: 'sortedHeader', file: '../data/csv/sortedHeader.csv' }, + { key: 'sortedHeaderCustom', file: '../data/csv/sortedHeaderCustom.csv' }, + { key: 'emptyFieldValues', file: '../data/csv/emptyFieldValues.csv' }, + { key: 'quotedEmptyFieldValue', file: '../data/csv/quotedEmptyFieldValue.csv' }, + { key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv' }, + { key: 'unwind', file: '../data/csv/unwind.csv' }, + { key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv' }, + { key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv' }, + { key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv' }, + { key: 'localeFormat', file: '../data/csv/localeFormat.csv' }, + { key: 'invalidParsedValues', file: '../data/csv/invalidParsedValues.csv' }, + { key: 'firstColumnWrapCRLF', file: '../data/csv/firstColumnWrapCRLF.csv' }, + { key: 'emptyLastFieldValue', file: '../data/csv/emptyLastFieldValue.csv' }, + { key: 'emptyLastFieldValueNoEol', file: '../data/csv/emptyLastFieldValueNoEol.csv' }, + { key: 'lastCharFieldDelimiter', file: '../data/csv/lastCharFieldDelimiter.csv' }, + { key: 'nativeMapMethod', file: '../data/csv/nativeMapMethod.csv' }, + { key: 'nestedDotKeys', file: '../data/csv/nestedDotKeys.csv' }, + { key: 'nestedDotKeysWithArray', file: '../data/csv/nestedDotKeysWithArray.csv' }, + { key: 'nestedDotKeysWithArrayExpandedUnwound', file: '../data/csv/nestedDotKeysWithArrayExpandedUnwound.csv' }, + { key: 'emptyColumns', file: '../data/csv/emptyColumns.csv' }, + { key: 'quotedFieldWithNewline', file: '../data/csv/quotedFieldWithNewline.csv' }, + { key: 'falsyValues', file: '../data/csv/falsyValues.csv' }, + { key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv' }, + { key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv' }, + { key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv' }, + { key: 'wildcardMatch', file: '../data/csv/wildcardMatch.csv' }, + { key: 'arrayIndexesAsKeys', file: '../data/csv/arrayIndexesAsKeys.csv' }, + { key: 'keyWithEndingDot', file: '../data/csv/keyWithEndingDot.csv' }, + { key: 'fieldEolAtStart', file: '../data/csv/fieldEolAtStart.csv' }, ]; function readCsvFile(filePath: string) { diff --git a/test/config/testJsonFilesList.ts b/test/config/testJsonFilesList.ts index 4a1e40f..42fd5cb 100644 --- a/test/config/testJsonFilesList.ts +++ b/test/config/testJsonFilesList.ts @@ -48,4 +48,5 @@ export default { wildcardMatch: require('../data/json/wildcardMatch.json'), arrayIndexesAsKeys: require('../data/json/arrayIndexesAsKeys.json'), keyWithEndingDot: require('../data/json/keyWithEndingDot.json'), + fieldEolAtStart: require('../data/json/fieldEolAtStart.json'), }; diff --git a/test/csv2json.ts b/test/csv2json.ts index 5c62016..91c3558 100644 --- a/test/csv2json.ts +++ b/test/csv2json.ts @@ -62,7 +62,7 @@ export function runTests() { it('should convert csv representing nested json with a missing field to the correct json structure', () => { const json = csv2json(csvTestData.nestedMissingField); - jsonTestData.nestedMissingField[0].level1.level2.level3 = {level: null}; + jsonTestData.nestedMissingField[0].level1.level2.level3 = { level: null }; assert.deepEqual(json, jsonTestData.nestedMissingField); }); @@ -177,6 +177,12 @@ export function runTests() { const json = csv2json(csvTestData.keyWithEndingDot); assert.deepEqual(json, jsonTestData.keyWithEndingDot); }); + + // Test case for #265 + it('should handle fields starting with an EOL delimiter', () => { + const json = csv2json(csvTestData.fieldEolAtStart); + assert.deepEqual(json, jsonTestData.fieldEolAtStart); + }); }); describe('Error Handling', () => { diff --git a/test/data/csv/fieldEolAtStart.csv b/test/data/csv/fieldEolAtStart.csv new file mode 100644 index 0000000..a7af24a --- /dev/null +++ b/test/data/csv/fieldEolAtStart.csv @@ -0,0 +1,6 @@ +id,name,age,city +1,John Doe,29,New York +2,Jane Smith,34," + Los Angeles" +3,Emily Davis,22,Chicago +4,Michael Brown,40,Houston \ No newline at end of file diff --git a/test/data/json/fieldEolAtStart.json b/test/data/json/fieldEolAtStart.json new file mode 100644 index 0000000..32db0c0 --- /dev/null +++ b/test/data/json/fieldEolAtStart.json @@ -0,0 +1,26 @@ +[ + { + "id": 1, + "name": "John Doe", + "age": 29, + "city": "New York" + }, + { + "id": 2, + "name": "Jane Smith", + "age": 34, + "city": "\n Los Angeles" + }, + { + "id": 3, + "name": "Emily Davis", + "age": 22, + "city": "Chicago" + }, + { + "id": 4, + "name": "Michael Brown", + "age": 40, + "city": "Houston" + } +] \ No newline at end of file