Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix 265 #268

Merged
merged 4 commits into from
Nov 24, 2024
Merged

Fix 265 #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d253b84
fix: resolve bug when a wrapped field value starts with an EOL delimiter
mrodrig Nov 24, 2024
643dcd9
test: add test case for #265 fix
mrodrig Nov 24, 2024
6a6b986
ci: add lint:fix script
mrodrig Nov 24, 2024
2155500
chore(deps): npm audit fix
mrodrig Nov 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"compile": "tsc -p tsconfig.build.json",
"coverage": "nyc npm run test",
"lint": "eslint --ext .js,.ts src test",
"lint:fix": "eslint --fix --ext .js,.ts src test",
"prepublishOnly": "npm run build",
"test": "mocha -r ts-node/register test/index.ts"
},
Expand Down Expand Up @@ -59,4 +60,4 @@
"node": ">= 16"
},
"license": "MIT"
}
}
17 changes: 13 additions & 4 deletions src/csv2json.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { excelBOM } from './constants';
import type { Csv2JsonParams, FullCsv2JsonOptions, HeaderField } from './types';
import * as utils from './utils';

export const Csv2Json = function(options: FullCsv2JsonOptions) {
export const Csv2Json = function (options: FullCsv2JsonOptions) {
const escapedWrapDelimiterRegex = new RegExp(options.delimiter.wrap + options.delimiter.wrap, 'g'),
excelBOMRegex = new RegExp('^' + excelBOM),
valueParserFn = options.parseValue && typeof options.parseValue === 'function' ? options.parseValue : JSON.parse;
Expand Down Expand Up @@ -166,9 +166,18 @@ export const Csv2Json = function(options: FullCsv2JsonOptions) {
if (utils.getNCharacters(csv, index + 1, eolDelimiterLength) === options.delimiter.eol) {
index += options.delimiter.eol.length + 1; // Skip past EOL
}
}

else if ((charBefore !== options.delimiter.wrap || stateVariables.justParsedDoubleQuote && charBefore === options.delimiter.wrap) &&
} else if (charBefore === options.delimiter.field && character === options.delimiter.wrap && charAfter === options.delimiter.eol) {
// We reached the start of a wrapped new field that begins with an EOL delimiter

// Retrieve the remaining value and add it to the split line list of values
splitLine.push(csv.substring(stateVariables.startIndex, index - 1));

stateVariables.startIndex = index;
stateVariables.parsingValue = true;
stateVariables.insideWrapDelimiter = true;
stateVariables.justParsedDoubleQuote = true;
index += 1;
} else if ((charBefore !== options.delimiter.wrap || stateVariables.justParsedDoubleQuote && charBefore === options.delimiter.wrap) &&
character === options.delimiter.wrap && utils.getNCharacters(csv, index + 1, eolDelimiterLength) === options.delimiter.eol) {
// If we reach a wrap which is not preceded by a wrap delim and the next character is an EOL delim (ie. *"\n)

Expand Down
109 changes: 55 additions & 54 deletions test/config/testCsvFilesList.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,60 +2,61 @@ import { readFileSync } from 'fs';
import path from 'path';

const csvFileConfig = [
{key: 'noData', file: '../data/csv/noData.csv'},
{key: 'singleDocument', file: '../data/csv/singleDocument.csv'},
{key: 'array', file: '../data/csv/array.csv'},
{key: 'arrayObjects', file: '../data/csv/arrayObjects.csv'},
{key: 'arrayMixedObjNonObj', file: '../data/csv/arrayMixedObjNonObj.csv'},
{key: 'arraySingleArray', file: '../data/csv/arraySingleArray.csv'},
{key: 'date', file: '../data/csv/date.csv'},
{key: 'null', file: '../data/csv/null.csv'},
{key: 'undefined', file: '../data/csv/undefined.csv'},
{key: 'nested', file: '../data/csv/nested.csv'},
{key: 'nestedMissingField', file: '../data/csv/nestedMissingField.csv'},
{key: 'comma', file: '../data/csv/comma.csv'},
{key: 'quotes', file: '../data/csv/quotes.csv'},
{key: 'quotesHeader', file: '../data/csv/quotesHeader.csv'},
{key: 'quotesAndCommas', file: '../data/csv/quotesAndCommas.csv'},
{key: 'eol', file: '../data/csv/eol.csv'},
{key: 'assortedValues', file: '../data/csv/assortedValues.csv'},
{key: 'trimFields', file: '../data/csv/trimFields.csv'},
{key: 'trimmedFields', file: '../data/csv/trimmedFields.csv'},
{key: 'trimHeader', file: '../data/csv/trimHeader.csv'},
{key: 'trimmedHeader', file: '../data/csv/trimmedHeader.csv'},
{key: 'excelBOM', file: '../data/csv/excelBOM.csv'},
{key: 'specifiedKeys', file: '../data/csv/specifiedKeys.csv'},
{key: 'specifiedKeysNoData', file: '../data/csv/specifiedKeysNoData.csv'},
{key: 'extraLine', file: '../data/csv/extraLine.csv'},
{key: 'noHeader', file: '../data/csv/noHeader.csv'},
{key: 'sortedHeader', file: '../data/csv/sortedHeader.csv'},
{key: 'sortedHeaderCustom', file: '../data/csv/sortedHeaderCustom.csv'},
{key: 'emptyFieldValues', file: '../data/csv/emptyFieldValues.csv'},
{key: 'quotedEmptyFieldValue', file: '../data/csv/quotedEmptyFieldValue.csv'},
{key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv'},
{key: 'unwind', file: '../data/csv/unwind.csv'},
{key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv'},
{key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv'},
{key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv'},
{key: 'localeFormat', file: '../data/csv/localeFormat.csv'},
{key: 'invalidParsedValues', file: '../data/csv/invalidParsedValues.csv'},
{key: 'firstColumnWrapCRLF', file: '../data/csv/firstColumnWrapCRLF.csv'},
{key: 'emptyLastFieldValue', file: '../data/csv/emptyLastFieldValue.csv'},
{key: 'emptyLastFieldValueNoEol', file: '../data/csv/emptyLastFieldValueNoEol.csv'},
{key: 'lastCharFieldDelimiter', file: '../data/csv/lastCharFieldDelimiter.csv'},
{key: 'nativeMapMethod', file: '../data/csv/nativeMapMethod.csv'},
{key: 'nestedDotKeys', file: '../data/csv/nestedDotKeys.csv'},
{key: 'nestedDotKeysWithArray', file: '../data/csv/nestedDotKeysWithArray.csv'},
{key: 'nestedDotKeysWithArrayExpandedUnwound', file: '../data/csv/nestedDotKeysWithArrayExpandedUnwound.csv'},
{key: 'emptyColumns', file: '../data/csv/emptyColumns.csv'},
{key: 'quotedFieldWithNewline', file: '../data/csv/quotedFieldWithNewline.csv'},
{key: 'falsyValues', file: '../data/csv/falsyValues.csv'},
{key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv'},
{key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv'},
{key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv'},
{key: 'wildcardMatch', file: '../data/csv/wildcardMatch.csv'},
{key: 'arrayIndexesAsKeys', file: '../data/csv/arrayIndexesAsKeys.csv'},
{key: 'keyWithEndingDot', file:'../data/csv/keyWithEndingDot.csv'},
{ key: 'noData', file: '../data/csv/noData.csv' },
{ key: 'singleDocument', file: '../data/csv/singleDocument.csv' },
{ key: 'array', file: '../data/csv/array.csv' },
{ key: 'arrayObjects', file: '../data/csv/arrayObjects.csv' },
{ key: 'arrayMixedObjNonObj', file: '../data/csv/arrayMixedObjNonObj.csv' },
{ key: 'arraySingleArray', file: '../data/csv/arraySingleArray.csv' },
{ key: 'date', file: '../data/csv/date.csv' },
{ key: 'null', file: '../data/csv/null.csv' },
{ key: 'undefined', file: '../data/csv/undefined.csv' },
{ key: 'nested', file: '../data/csv/nested.csv' },
{ key: 'nestedMissingField', file: '../data/csv/nestedMissingField.csv' },
{ key: 'comma', file: '../data/csv/comma.csv' },
{ key: 'quotes', file: '../data/csv/quotes.csv' },
{ key: 'quotesHeader', file: '../data/csv/quotesHeader.csv' },
{ key: 'quotesAndCommas', file: '../data/csv/quotesAndCommas.csv' },
{ key: 'eol', file: '../data/csv/eol.csv' },
{ key: 'assortedValues', file: '../data/csv/assortedValues.csv' },
{ key: 'trimFields', file: '../data/csv/trimFields.csv' },
{ key: 'trimmedFields', file: '../data/csv/trimmedFields.csv' },
{ key: 'trimHeader', file: '../data/csv/trimHeader.csv' },
{ key: 'trimmedHeader', file: '../data/csv/trimmedHeader.csv' },
{ key: 'excelBOM', file: '../data/csv/excelBOM.csv' },
{ key: 'specifiedKeys', file: '../data/csv/specifiedKeys.csv' },
{ key: 'specifiedKeysNoData', file: '../data/csv/specifiedKeysNoData.csv' },
{ key: 'extraLine', file: '../data/csv/extraLine.csv' },
{ key: 'noHeader', file: '../data/csv/noHeader.csv' },
{ key: 'sortedHeader', file: '../data/csv/sortedHeader.csv' },
{ key: 'sortedHeaderCustom', file: '../data/csv/sortedHeaderCustom.csv' },
{ key: 'emptyFieldValues', file: '../data/csv/emptyFieldValues.csv' },
{ key: 'quotedEmptyFieldValue', file: '../data/csv/quotedEmptyFieldValue.csv' },
{ key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv' },
{ key: 'unwind', file: '../data/csv/unwind.csv' },
{ key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv' },
{ key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv' },
{ key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv' },
{ key: 'localeFormat', file: '../data/csv/localeFormat.csv' },
{ key: 'invalidParsedValues', file: '../data/csv/invalidParsedValues.csv' },
{ key: 'firstColumnWrapCRLF', file: '../data/csv/firstColumnWrapCRLF.csv' },
{ key: 'emptyLastFieldValue', file: '../data/csv/emptyLastFieldValue.csv' },
{ key: 'emptyLastFieldValueNoEol', file: '../data/csv/emptyLastFieldValueNoEol.csv' },
{ key: 'lastCharFieldDelimiter', file: '../data/csv/lastCharFieldDelimiter.csv' },
{ key: 'nativeMapMethod', file: '../data/csv/nativeMapMethod.csv' },
{ key: 'nestedDotKeys', file: '../data/csv/nestedDotKeys.csv' },
{ key: 'nestedDotKeysWithArray', file: '../data/csv/nestedDotKeysWithArray.csv' },
{ key: 'nestedDotKeysWithArrayExpandedUnwound', file: '../data/csv/nestedDotKeysWithArrayExpandedUnwound.csv' },
{ key: 'emptyColumns', file: '../data/csv/emptyColumns.csv' },
{ key: 'quotedFieldWithNewline', file: '../data/csv/quotedFieldWithNewline.csv' },
{ key: 'falsyValues', file: '../data/csv/falsyValues.csv' },
{ key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv' },
{ key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv' },
{ key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv' },
{ key: 'wildcardMatch', file: '../data/csv/wildcardMatch.csv' },
{ key: 'arrayIndexesAsKeys', file: '../data/csv/arrayIndexesAsKeys.csv' },
{ key: 'keyWithEndingDot', file: '../data/csv/keyWithEndingDot.csv' },
{ key: 'fieldEolAtStart', file: '../data/csv/fieldEolAtStart.csv' },
];

function readCsvFile(filePath: string) {
Expand Down
1 change: 1 addition & 0 deletions test/config/testJsonFilesList.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,5 @@ export default {
wildcardMatch: require('../data/json/wildcardMatch.json'),
arrayIndexesAsKeys: require('../data/json/arrayIndexesAsKeys.json'),
keyWithEndingDot: require('../data/json/keyWithEndingDot.json'),
fieldEolAtStart: require('../data/json/fieldEolAtStart.json'),
};
8 changes: 7 additions & 1 deletion test/csv2json.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ export function runTests() {

it('should convert csv representing nested json with a missing field to the correct json structure', () => {
const json = csv2json(csvTestData.nestedMissingField);
jsonTestData.nestedMissingField[0].level1.level2.level3 = {level: null};
jsonTestData.nestedMissingField[0].level1.level2.level3 = { level: null };
assert.deepEqual(json, jsonTestData.nestedMissingField);
});

Expand Down Expand Up @@ -177,6 +177,12 @@ export function runTests() {
const json = csv2json(csvTestData.keyWithEndingDot);
assert.deepEqual(json, jsonTestData.keyWithEndingDot);
});

// Test case for #265
it('should handle fields starting with an EOL delimiter', () => {
const json = csv2json(csvTestData.fieldEolAtStart);
assert.deepEqual(json, jsonTestData.fieldEolAtStart);
});
});

describe('Error Handling', () => {
Expand Down
6 changes: 6 additions & 0 deletions test/data/csv/fieldEolAtStart.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
id,name,age,city
1,John Doe,29,New York
2,Jane Smith,34,"
Los Angeles"
3,Emily Davis,22,Chicago
4,Michael Brown,40,Houston
26 changes: 26 additions & 0 deletions test/data/json/fieldEolAtStart.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
[
{
"id": 1,
"name": "John Doe",
"age": 29,
"city": "New York"
},
{
"id": 2,
"name": "Jane Smith",
"age": 34,
"city": "\n Los Angeles"
},
{
"id": 3,
"name": "Emily Davis",
"age": 22,
"city": "Chicago"
},
{
"id": 4,
"name": "Michael Brown",
"age": 40,
"city": "Houston"
}
]
Loading