csvtojson

Version:

A tool concentrating on converting csv data to JSON with customised parser supporting

64 lines • 603 kB

HTML

<!doctype html> <html lang="en"> <head> <title>Code coverage report for csv2json/dist/csvtojson.min.js</title> <meta charset="utf-8" /> <link rel="stylesheet" href="../../prettify.css" /> <link rel="stylesheet" href="../../base.css" /> <meta name="viewport" content="width=device-width, initial-scale=1"> <style type='text/css'> .coverage-summary .sorter { background-image: url(../../sort-arrow-sprite.png); } </style> </head> <body> <div class='wrapper'> <div class='pad1'> <h1> <a href="../../index.html">All files</a> / <a href="index.html">csv2json/dist</a> csvtojson.min.js </h1> <div class='clearfix'> <div class='fl pad1y space-right2'> 0% Statements 0/5007 </div> <div class='fl pad1y space-right2'> 0% Branches 0/5826 </div> <div class='fl pad1y space-right2'> 0% Functions 0/1473 </div> <div class='fl pad1y space-right2'> 0% Lines 0/6 </div> </div> Press n or j to go to the next uncovered block, b, p or k for the previous block. </div> <div class='status-line low'></div> <pre><table class="coverage"> <tr><td class="line-count quiet"><a name='L1'></a><a href='#L1'>1</a> <a name='L2'></a><a href='#L2'>2</a> <a name='L3'></a><a href='#L3'>3</a> <a name='L4'></a><a href='#L4'>4</a> <a name='L5'></a><a href='#L5'>5</a> <a name='L6'></a><a href='#L6'>6</a> <a name='L7'></a><a href='#L7'>7</a> <a name='L8'></a><a href='#L8'>8</a></td><td class="line-coverage quiet">               </td><td class="text"><pre class="prettyprint lang-js">/*Automatically Generated. Do not modify.*/ !function a(b,c,d){function e(g,h){if(!c[g]){if(!b[g]){var i="function"==typeof require&&require;if(!h&&i)return i(g,!0);if(f)return f(g,!0);var j=new Error("Cannot find module '"+g+"'");throw j.code="MODULE_NOT_FOUND",j}var k=c[g]={exports:{}};b[g][0].call(k.exports,function(a){var c=b[g][1][a];return e(c?c:a)},k,k.exports,a,b,c,d)}return c[g].exports}for(var f="function"==typeof require&&require,g=0;g<d.length;g++)e(d[g]);return e}({1:[function(a,b,c){window&&(window.csvtojson=a("./index.js"),window.csvtojson.version=a("./package.json").version)},{"./index.js":2,"./package.json":134}],2:[function(a,b,c){b.exports=a("./libs/csv2json.js")},{"./libs/csv2json.js":24}],3:[function(a,b,c){function d(a,b,c){Error.call(this,""),this.err=a,this.line=b,this.extra=c,this.message="Error: "+a+". JSON Line number: "+b+(c?" near: "+c:""),this.name="CSV Error"}var e=a("util");b.exports=d,e.inherits(d,Error),d.prototype.toString=function(){return JSON.stringify([this.err,this.line,this.extra])},d.column_mismatched=function(a,b){return new d("column_mismatched",a,b)},d.unclosed_quote=function(a,b){return new d("unclosed_quote",a,b)},d.fromArray=function(a){return new d(a[0],a[1],a[2])}},{util:73}],4:[function(a,b,c){(function(c,d){function e(a,b){return j.call(this,b),this._options=b||{},this.param=n(a),this.param._options=this._options,this.started=!1,this.recordNum=0,this.lineNumber=0,this._csvLineBuffer="",this.lastIndex=0,this.param.forked&&(this.param.forked=!1,this.workerNum=2),this.flushCb=null,this.processEnd=!1,this.sequenceBuffer=[],this._needJson=null,this._needEmitResult=null,this._needEmitFinalResult=null,this._needEmitJson=null,this._needPush=null,this._needEmitCsv=null,this._csvTransf=null,this.finalResult=[],this.on("error",f(this)),this.on("end",f(this)),this.initWorker(),c.nextTick(function(){null===this._needEmitFinalResult&&(this._needEmitFinalResult=this.listeners("end_parsed").length>0),null===this._needEmitResult&&(this._needEmitResult=this.listeners("record_parsed").length>0),null===this._needEmitJson&&(this._needEmitJson=this.listeners("json").length>0),null===this._needEmitCsv&&(this._needEmitCsv=this.listeners("csv").length>0),null===this._needJson&&(this._needJson=this._needEmitJson||this._needEmitFinalResult||this._needEmitResult||this.transform||this._options.objectMode),null===this._needPush&&(this._needPush=this.listeners("data").length>0||this.listeners("readable").length>0),this.param._needParseJson=this._needJson||this._needPush}.bind(this)),this}function f(a){return function(b){a._hasDone||(a._hasDone=!0,c.nextTick(function(){a.emit("done",b)}))}}function g(a){return v?d.from(a,"utf8"):new d(a,"utf8")}function h(a){if(a._postIgnoreColumns){for(var b=0;b<a.ignoreColumns.length;b++){var c=a.ignoreColumns[b];if("string"==typeof c){var d=a._headers.indexOf(c);d>-1?a.ignoreColumns[b]=d:a.ignoreColumns[b]=-1}}a.ignoreColumns.sort(function(a,b){return b-a})}if(a._postIncludeColumns)for(var b=0;b<a.includeColumns.length;b++){var e=a.includeColumns[b];if("string"==typeof e){var d=a._headers.indexOf(e);d>-1?a.includeColumns[b]=d:a.includeColumns[b]=-1}}a.ignoreColumns=t.uniq(a.ignoreColumns),a.includeColumns=t.uniq(a.includeColumns)}var i=a("util"),j=a("stream").Transform,k=a("os"),l=a("strip-bom"),m=k.EOL,n=a("./defParam"),o=a("./fileline"),p=a("./fileLineToCSVLine"),q=a("./linesToJson"),r=a("./CSVError"),s=null,t=a("lodash"),u=a("./rowSplit");i.inherits(e,j);var v=c.versions.node.split(".")[0]>=6;e.prototype._transform=function(a,b,c){a=this.prepareData(a);var e=a.length-1,f=null;if(0!=(128&a[e])){for(;128===(192&a[e]);)e--;e--}if(e!=a.length-1){f=a.slice(e+1),a=a.slice(0,e+1);var h=c,i=this;c=function(){i._csvLineBuffer?i._csvLineBuffer=d.concat([g(i._csvLineBuffer,"utf8"),f]):i._csvLineBuffer=f,h()}}a=a.toString("utf8"),this.started===!1&&(this.started=!0,a=l(a),this.param.toArrayString&&this._needPush&&this.push("["+m,"utf8"));var i=this;this.preProcessRaw(a,function(a){a&&a.length>0?i.processData(a,c):c()})},e.prototype.prepareData=function(a){return this._csvLineBuffer&&this._csvLineBuffer.length>0?("string"==typeof this._csvLineBuffer&&(this._csvLineBuffer=g(this._csvLineBuffer)),d.concat([this._csvLineBuffer,a])):a},e.prototype.setPartialData=function(a){this._csvLineBuffer=a},e.prototype.processData=function(a,b){var c=this.param;c.ignoreEmpty&&!c._headers&&(a=a.trimLeft());var d=o(a,this.param);if(d.lines.length>0)if(this.preProcessLine&&"function"==typeof this.preProcessLine&&(d.lines=this._preProcessLines(d.lines,this.lastIndex)),c._headers)if(c.workerNum<=1){var e=p(d,c);this.setPartialData(e.partial);var f=q(e.lines,c,this.recordNum);this.processResult(f),this.lastIndex+=f.length,this.recordNum+=f.length,b()}else this.workerProcess(d,b);else this.processHead(d,b);else this.setPartialData(d.partial),b()},e.prototype._preProcessLines=function(a,b){for(var c=[],d=0,e=a.length;e>d;d++){var f=this.preProcessLine(a[d],b+d+1);"string"==typeof f?c.push(f):(c.push(a[d]),this.emit("error",new Error("preProcessLine should return a string but got: "+JSON.stringify(f))))}return c},e.prototype.initWorker=function(){var b=this.param.workerNum-1;b>0&&(s=a("./workerMgr"),this.workerMgr=s(),this.workerMgr.initWorker(b,this.param))},e.prototype.preRawData=function(a){return this.preProcessRaw=a,this},e.prototype.preFileLine=function(a){return this.preProcessLine=a,this},e.prototype.workerProcess=function(a,b){var c=this,d=a,e=this.getEol();this.setPartialData(d.partial),this.workerMgr.sendWorker(d.lines.join(e)+e,this.lastIndex,b,function(a,b){var d,e=c.sequenceBuffer[0];if(e.idx===b){e.result=a;for(var f=[];c.sequenceBuffer[0]&&c.sequenceBuffer[0].result;)d=c.sequenceBuffer.shift(),f=f.concat(d.result);c.processResult(f),c.recordNum+=f.length}else for(var g=0,h=c.sequenceBuffer.length;h>g;g++)if(d=c.sequenceBuffer[g],d.idx===b){d.result=a;break}}),this.sequenceBuffer.push({idx:this.lastIndex,result:null}),this.lastIndex+=d.lines.length},e.prototype.processHead=function(b,c){var d=this.param;if(d._headers)return c();d._needFilterRow=!1;var e=b.lines,f="",g=[];if(!d.noheader)for(;e.length;){var i=f+e.shift(),j=u(i,d);if(j.closed){g=j.cols,f="";break}f=i+this.getEol()}if(d._needFilterRow=!0,!d.noheader&&0===g.length)return this.setPartialData(f),c();d.noheader?d.headers?d._headers=d.headers:d._headers=[]:d.headers?d._headers=d.headers:d._headers=g,h(d),d._headers=a("./filterRow")(d._headers,d);var e=p(b,d);this.setPartialData(e.partial),this.param.workerNum>1&&this.workerMgr.setParams(d);var k=q(e.lines,d,0);this.processResult(k),this.lastIndex+=k.length,this.recordNum+=k.length,c()},e.prototype.processResult=function(a){for(var b=0,c=a.length;c>b;b++){var d=a[b];d.err?this.emit("error",d.err):this.emitResult(d)}},e.prototype.emitResult=function(a){var b=a.index,c=a.row,d=a.json,e=null,f=null;"string"==typeof d?f=d:e=d,null===e&&this._needJson&&(e=JSON.parse(f),"string"==typeof c&&(c=JSON.parse(c))),this.transform&&"function"==typeof this.transform&&(this.transform(e,c,b),f=null),this._needEmitJson&&this.emit("json",e,b),this._needEmitCsv&&("string"==typeof c&&(c=JSON.parse(c)),this.emit("csv",c,b)),this.param.constructResult&&this._needEmitFinalResult&&this.finalResult.push(e),this._needEmitResult&&this.emit("record_parsed",e,c,b),this.param.toArrayString&&b>0&&this._needPush&&this.push(","+m),this._options&&this._options.objectMode?this.push(e):this._needPush&&(null===f&&(f=JSON.stringify(e)),this.push(this.param.toArrayString?f:f+m,"utf8"))},e.prototype.preProcessRaw=function(a,b){b(a)},e.prototype.preProcessLine=function(a,b){return a},e.prototype._flush=function(a){var b=this;if(this.flushCb=function(){b.emit("end_parsed",b.finalResult),b.workerMgr&&b.workerMgr.destroyWorker(),a(),b._needPush||b.emit("end")},this._csvLineBuffer.length>0){var c=this.getEol();this._csvLineBuffer[this._csvLineBuffer.length-1]!==c&&(this._csvLineBuffer+=c),this.processData(this._csvLineBuffer,function(){this.checkAndFlush()}.bind(this))}else this.checkAndFlush()},e.prototype.checkAndFlush=function(){0!==this._csvLineBuffer.length&&this.emit("error",r.unclosed_quote(this.recordNum,this._csvLineBuffer),this._csvLineBuffer),this.param.toArrayString&&this._needPush&&this.push(m+"]","utf8"),this.workerMgr&&this.workerMgr.isRunning()?this.workerMgr.drain=function(){this.flushCb()}.bind(this):this.flushCb()},e.prototype.getEol=function(a){if(!this.param.eol&&a){for(var b=0,c=a.length;c>b;b++){if("\r"===a[b])return"\n"===a[b+1]?this.param.eol="\r\n":this.param.eol="\r",this.param.eol;if("\n"===a[b])return this.param.eol="\n",this.param.eol}this.param.eol=m}return this.param.eol||m},e.prototype.fromFile=function(b,c,d){var e=a("fs"),f=null;return"object"==typeof c&&"undefined"==typeof d&&(d=c,c=null),this.wrapCallback(c,function(){f&&f.destroy&&f.destroy()}),e.exists(b,function(a){a?(f=e.createReadStream(b,d),f.pipe(this)):this.emit("error",new Error("File not exists"))}.bind(this)),this},e.prototype.fromStream=function(a,b){return b&&"function"==typeof b&&this.wrapCallback(b),a.pipe(this),this},e.prototype.transf=function(a){return this.transform=a,this},e.prototype.fromString=function(a,b){return"string"!=typeof a?b(new Error("Passed CSV Data is not a string.")):(b&&"function"==typeof b&&this.wrapCallback(b,function(){}),c.nextTick(function(){this.end(a)}.bind(this)),this)},e.prototype.wrapCallback=function(a,b){void 0===b&&(b=function(){}),a&&"function"==typeof a&&this.once("end_parsed",function(b){this.hasError||a(null,b)}.bind(this)),this.once("error",function(c){this.hasError=!0,a&&"function"==typeof a&&a(c),b()}.bind(this))},b.exports=e}).call(this,a("_process"),a("buffer").Buffer)},{"./CSVError":3,"./defParam":6,"./fileLineToCSVLine":13,"./fileline":14,"./filterRow":15,"./linesToJson":19,"./rowSplit":22,"./workerMgr":23,_process:42,buffer:33,fs:31,lodash:129,os:41,stream:60,"strip-bom":133,util:73}],5:[function(a,b,c){var d=a("./getEol"),e=a("./rowSplit");b.exports=function(a,b){for(var c=[],f="";a.length;){var g=f+a.shift(),h=e(g,b);h.closed?(c.push(h.cols),f=""):f=g+(d(g,b)||"\n")}return{lines:c,partial:f}}},{"./getEol":17,"./rowSplit":22}],6:[function(a,b,c){(function(a){function c(b,c){return a.env[b]?a.env[b]:c}var d=/^[0-9]+$/;b.exports=function(a){var b={constructResult:!0,delimiter:",",ignoreColumns:[],includeColumns:[],quote:'"',trim:!0,checkType:!1,toArrayString:!1,ignoreEmpty:!1,workerNum:c("CSV_WORKER",1),fork:!1,noheader:!1,headers:null,flatKeys:!1,maxRowLength:0,checkColumn:!1,escape:'"',colParser:{},_columnConv:[],_headerType:[],_headerTitle:[],_headerFlag:[],_headers:null,_needFilterRow:!1};a||(a={});for(var e in a)a.hasOwnProperty(e)&&(b[e]=a[e]);return b.ignoreColumns.length>0&&!d.test(b.ignoreColumns.join(""))&&(b._postIgnoreColumns=!0),b.includeColumns.length>0&&!d.test(b.includeColumns.join(""))&&(b._postIncludeColumns=!0),(b.ignoreColumns.length||b.includeColumns.length)&&(b._needFilterRow=!0,b._postIgnoreColumns||b.ignoreColumns.sort(function(a,b){return b-a})),b}}).call(this,a("_process"))},{_process:42}],7:[function(a,b,c){b.exports=[a("./parser_array.js"),a("./parser_json.js"),a("./parser_omit.js"),a("./parser_jsonarray.js"),a("./parser_flat.js")]},{"./parser_array.js":8,"./parser_flat.js":9,"./parser_json.js":10,"./parser_jsonarray.js":11,"./parser_omit.js":12}],8:[function(a,b,c){b.exports={name:"array",processSafe:!0,regExp:/^\*array\*/,parserFunc:function(a){var b=a.head.replace(this.regExp,"");void 0===a.resultRow[b]&&(a.resultRow[b]=[]),a.resultRow[b].push(a.item)}}},{}],9:[function(a,b,c){b.exports={name:"flat",processSafe:!0,regExp:/^\*flat\*/,parserFunc:function(a){var b=this.getHeadStr(),c=a.item;a.resultRow[b]=c}}},{}],10:[function(a,b,c){function d(a,b,c,d){for(var e,f,g;b.length>1;)e=b.shift(),f=d?!1:e.match(c),f?(void 0===a[e.replace(f[0],"")]&&(a[e.replace(f[0],"")]=[]),g=f[1],a=a[e.replace(f[0],"")],""===g&&(g=a.length),a[g]||(a[g]={}),a=a[g]):(void 0===a[e]&&(a[e]={}),a=a[e]);return a}var e=/\[([0-9]*)\]/;b.exports={name:"json",processSafe:!0,regExp:/^\*json\*/,parserFunc:function(a){var b,c,f,g=this.getHeadStr(),h=a.config&&a.config.flatKeys?[g]:g.split("."),i=d(a.resultRow,h,e,a.config&&a.config.flatKeys);if(f=h.shift(),b=a.config&&a.config.flatKeys?!1:f.match(e))try{f=f.replace(b[0],""),i[f]&&i[f]instanceof Array||(i[f]=[]),i[f]?(c=b[1],""===c&&(c=i[f].length),i[f][c]=a.item):a.resultRow[g]=a.item}catch(j){a.resultRow[g]=a.item}else"string"==typeof i?a.resultRow[g]=a.item:i[f]=a.item}}},{}],11:[function(a,b,c){b.exports={name:"jsonarray",processSafe:!0,regExp:/^\*jsonarray\*/,parserFunc:function(a){for(var b=a.head.replace(this.regExp,""),c=b.split("."),d=a.resultRow;c.length>1;){var e=c.shift();void 0===d[e]&&(d[e]={}),d=d[e]}var f=c.shift();void 0===d[f]&&(d[f]=[]),d[f].push(a.item)}}},{}],12:[function(a,b,c){b.exports={name:"omit",regExp:/^\*omit\*/,processSafe:!0,parserFunc:function(){}}},{}],13:[function(a,b,c){var d=a("./csvline");b.exports=function(a,b){var c=a.lines,e=d(c,b);return{lines:e.lines,partial:e.partial+a.partial}}},{"./csvline":5}],14:[function(a,b,c){var d=a("./getEol");b.exports=function(a,b){var c=d(a,b),e=a.split(c),f=e.pop();return{lines:e,partial:f}}},{"./getEol":17}],15:[function(a,b,c){b.exports=function(a,b){if(b.ignoreColumns instanceof Array&&b.ignoreColumns.length>0)for(var c=0,d=b.ignoreColumns.length;d>c;c++)b.ignoreColumns[c]>=0&&a.splice(b.ignoreColumns[c],1);if(b.includeColumns instanceof Array&&b.includeColumns.length>0){for(var e=[],f=0,g=b.includeColumns.length;g>f;f++)b.includeColumns[f]>=0&&e.push(a[b.includeColumns[f]]);a=e}return a}},{}],16:[function(a,b,c){function d(a,b){var c;if("auto"===b.delimiter)c=e;else{if(!(b.delimiter instanceof Array))return b.delimiter;c=b.delimiter}var d=0,f=",";return c.forEach(function(b){var c=a.split(b).length;c>d&&(f=b,d=c)}),f}b.exports=d;var e=[",","|"," ",";",":"]},{}],17:[function(a,b,c){a("os").EOL;b.exports=function(a,b){if(!b.eol&&a)for(var c=0,d=a.length;d>c;c++){if("\r"===a[c])return"\n"===a[c+1]?b.eol="\r\n":a[c+1]&&(b.eol="\r"),b.eol;if("\n"===a[c])return b.eol="\n",b.eol}return b.eol}},{os:41}],18:[function(a,b,c){function d(a,c){return new b.exports.Converter(a,c)}b.exports=d,b.exports.Converter=a("./Converter.js")},{"./Converter.js":4}],19:[function(a,b,c){function d(a,b,c){for(var d=[],e=0,f=a.length;f>e;e++)d.push({err:null,json:{},index:c++,row:a[e]});return d}function e(a,b,c){for(var d=[],e=0,g=a.length;g>e;e++){var h=f(a[e],b,c++);h&&d.push(h)}return d}function f(a,b,c){var d=b.parseRules;if(b.checkColumn&&a.length!==d.length)return{err:s.column_mismatched(c)};var e=b._headers,f=g(a,e,b);return f?{json:f,index:c,row:a}:null}function g(a,b,c){for(var d=!1,e={},f=0,g=a.length;g>f;f++){var m,n,o;if(o=a[f],!c.ignoreEmpty||""!==o){d=!0,n=b[f],n&&""!==n||(n=b[f]="field"+(f+1));var p=h(n,f,c);if(p){var q=p(o,n,e,a,f);void 0!==q&&i(e,n,q)}else{var r=j(n,f,c);if("omit"===r)continue;c.checkType&&(m=l(o,n,f,c),o=m(o));var s=k(n,f,c);"flat"===r||c.flatKeys?e[s]=o:i(e,s,o)}}}return d?e:!1}function h(a,b,c){if(void 0!==c._columnConv[b])return c._columnConv[b];var d=c.colParser[a];if(void 0===d)return c._columnConv[b]=!1;if("string"==typeof d){d=d.trim().toLowerCase();var e=u[d];return e?c._columnConv[b]=e:c._columnConv[b]=!1}return"function"==typeof d?c._columnConv[b]=d:c._columnConv[b]=!1}function i(b,c,d){var e=a("lodash/set"),f=c.split(".");1===f.length?b[c]=d:e(b,c,d)}function j(a,b,c){return"string"==typeof c._headerFlag[b]?c._headerFlag[b]:a.indexOf("*omit*")>-1?c._headerFlag[b]="omit":a.indexOf("*flat*")>-1?c._headerFlag[b]="flat":c._headerFlag[b]=""}function k(a,b,c){if(c._headerTitle[b])return c._headerTitle[b];var d=(j(a,b,c),a.replace("*flat*","").replace("string#!","").replace("number#!",""));return c._headerTitle[b]=d}function l(a,b,c,d){return d._headerType[c]?d._headerType[c]:b.indexOf("number#!")>-1?d._headerType[c]=m:b.indexOf("string#!")>-1?d._headerType[c]=n:d.checkType?d._headerType[c]=o:d._headerType[c]=n}function m(a){var b=parseFloat(a);return isNaN(b)?a:b}function n(a){return a.toString()}function o(a){var b=a.trim();return""===b?n(a):t.test(b)?m(a):5===b.length&&"false"===b.toLowerCase()||4===b.length&&"true"===b.toLowerCase()?p(a):"{"===b[0]&&"}"===b[b.length-1]||"["===b[0]&&"]"===b[b.length-1]?q(a):n(a)}function p(a){var b=a.trim();return 5===b.length&&"false"===b.toLowerCase()?!1:!0}function q(a){try{return JSON.parse(a)}catch(b){return a}}var r=a("./parserMgr.js"),s=a("./CSVError"),t=/^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/;b.exports=function(a,b,c){if(b._needParseJson){if(b._headers||(b._headers=[]),!b.parseRules){var f=b._headers;b.parseRules=r.initParsers(f,b)}return e(a,b,c)}return d(a,b,c)};var u={string:n,number:m,omit:function(){}}},{"./CSVError":3,"./parserMgr.js":21,"lodash/set":131}],20:[function(a,b,c){function d(a,b,c,d){this.name="undefined"==typeof a?"Default":a,this.regExp=null,this.type="",this.processSafe=d,"undefined"!=typeof b&&("string"==typeof b?this.regExp=new RegExp(b):this.regExp=b),"undefined"!=typeof c&&(this.parse=c)}var e=["number","string"];d.prototype.convertType=function(a){var b=this.type;if("number"===b){var c=parseFloat(a);return isNaN(c)?0:c}if(this.param&&this.param.checkType&&""===b){var d=a.trim();if(""===d)return d;if(!isNaN(d))return parseFloat(d);if(5===d.length&&"false"===d.toLowerCase())return!1;if(4===d.length&&"true"===d.toLowerCase())return!0;if(!("{"===d[0]&&"}"===d[d.length-1]||"["===d[0]&&"]"===d[d.length-1]))return a;try{return JSON.parse(d)}catch(e){return a}}return a},d.prototype.setParam=function(a){this.param=a},d.prototype.test=function(a){return this.regExp&&this.regExp.test(a)},d.prototype.parse=function(a){a.resultRow[a.head]=a.item},d.prototype.getHeadStr=<span class="f