diff --git a/README.md b/README.md index f358b0d..802290d 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,14 @@ fs.createReadStream('scores.csv') # etl.csv([options]) -Parses incoming csv text into individual records. For parsing options see [csv-parser](https://www.npmjs.com/package/csv-parser). If `options` contains a `transform` object containing functions, those functions will be applied on the values of any matching keys in the data. If a key in the `transform` object is set to `null` then value with that key will not be included in the downstream packets. If option `santitize` is set to true, then headers will be trimmed, converted to lowercase, spaces converted to underscore and any blank values (empty strings) will be set to undefined. +Parses incoming csv text into individual records. For parsing options see [csv-parser](https://www.npmjs.com/package/csv-parser). If `options` contains a `transform` object containing functions, those functions will be applied on the values of any matching keys in the data. If a key in the `transform` object is set to `null` then value with that key will not be included in the downstream packets. If option `santitize` is set to true, then headers will be trimmed, converted to lowercase, spaces converted to underscore and any blank values (empty strings) will be set to undefined. The option `addLineNumbers` is `true` by default, if is set to `false` (boolean) the result won't have the `__line` numbers for each object. A `header` event will be emitted when headers have been parsed. An event listener can change the headers in-place before the stream starts piping out parsed data. Example ```js -// Here the test.csv is parsed but field dt is converted to date. Each packet will +// Here the test.csv is parsed but field dt is converted to date. Each packet will // contain the following properties: __filename, __path, __line and csv fields etl.file('test.csv') .pipe(etl.csv({ diff --git a/lib/csv_parser.js b/lib/csv_parser.js index 1b4868f..541c4d2 100644 --- a/lib/csv_parser.js +++ b/lib/csv_parser.js @@ -67,7 +67,11 @@ Csv_parser.prototype._push = function(d) { obj[key] = d[key]; } } - obj.__line = ++this.line; + + if (this.options.addLineNumbers !== false) { + obj.__line = ++this.line; + } + this.push(obj); }; diff --git a/test/csv-test.js b/test/csv-test.js index e41c48f..24f3fbe 100644 --- a/test/csv-test.js +++ b/test/csv-test.js @@ -4,34 +4,70 @@ const t = require('tap'); const data = require('./data'); t.test('csv',async t => { - const csv = etl.csv_parser({ - sanitize: true, - transform: { - dt: d => new Date(d) - } - }); - etl.file(path.join(__dirname,'test.csv')).pipe(csv); + t.test('parsing',async t => { + const csv = etl.csv_parser({ + sanitize: true, + transform: { + dt: d => new Date(d) + } + }); + + etl.file(path.join(__dirname,'test.csv')).pipe(csv); + + // Adjust expected values to the csv + const expected = data.copy().map(function(d) { + d.__line = d.__line +1; + d.__filename = 'test.csv'; - // Adjust expected values to the csv - const expected = data.copy().map(function(d) { - d.__line = d.__line +1; - d.__filename = 'test.csv'; + // Clear out __path and text as they are volatile + d.__path = undefined; + d.text = undefined; + + return d; + }); - // Clear out __path and text as they are volatile - d.__path = undefined; - d.text = undefined; - return d; - }); + const d = await csv.pipe(etl.expand()).promise(); + + d.forEach(d => { + d.__path = undefined; + d.text = undefined; + }); - - const d = await csv.pipe(etl.expand()).promise(); - - d.forEach(d => { - d.__path = undefined; - d.text = undefined; + t.same(d,expected,'parses data correctly'); }); - t.same(d,expected,'parses data correctly'); + t.test('when addLineNumbers is set to false',async t => { + const csv = etl.csv_parser({ + sanitize: true, + addLineNumbers: false, + transform: { + dt: d => new Date(d) + } + }); + + etl.file(path.join(__dirname,'test.csv')).pipe(csv); + + // Adjust expected values to the csv + const expected = data.copy().map(function(d) { + d.__filename = 'test.csv'; + + // Clear out __path and text as they are volatile + d.__path = undefined; + d.text = undefined; + delete d.__line; + + return d; + }); + + const d = await csv.pipe(etl.expand()).promise(); + + d.forEach(d => { + d.__path = undefined; + d.text = undefined; + }); + + t.same(d,expected,'parses data correctly'); + }); }); \ No newline at end of file