with-csv
Version:
A CSV file manipulation library with a fluent API à la Lodash
295 lines • 10.9 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.withCSV = void 0;
const crypto_1 = require("crypto");
const fs_1 = require("fs");
const lodash_pick_1 = __importDefault(require("lodash.pick"));
const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
const lodash_isstring_1 = __importDefault(require("lodash.isstring"));
const lodash_isarray_1 = __importDefault(require("lodash.isarray"));
const csv_parser_1 = __importDefault(require("csv-parser"));
const csv_stringify_1 = require("csv-stringify");
const stream_1 = require("stream");
const hashCache = new Set();
function getInterface(csvSource, options) {
const stream = (() => {
if (csvSource instanceof fs_1.ReadStream) {
return csvSource;
}
if (typeof csvSource === 'string') {
return (0, fs_1.createReadStream)(csvSource, { encoding: 'utf-8' });
}
return stream_1.Readable.from(csvSource);
})();
if (!stream) {
throw new Error('Input should be a string path, a Buffer or a ReadStream');
}
return stream.pipe((0, csv_parser_1.default)(options));
}
function hashRecord(record) {
const hash = (0, crypto_1.createHash)('sha1');
hash.update(JSON.stringify(record), 'utf8');
return hash.digest('hex');
}
function withCSV(csvFileOrBuffer, options) {
const readInterface = getInterface(csvFileOrBuffer, options);
const pipeline = [];
function getQueryChain() {
async function applyPipeline(row, idx) {
let value = row;
// First apply the pipeline to each row to filter/map it
for (const operation of pipeline) {
try {
value = await operation(value, idx);
}
catch (e) {
throw "Didn't make it to the end of the pipeline";
}
}
return value;
}
async function toDataset(limit) {
const dataSet = [];
let idx = 0;
const maxIdx = limit ? limit - 1 : Infinity;
for await (const row of readInterface) {
try {
const value = await applyPipeline(row, idx);
dataSet.push(value);
}
catch (e) { }
if (idx >= maxIdx) {
break;
}
idx++;
}
return dataSet;
}
const queryChain = {
/**
* Chainable methods
*/
map(callback) {
pipeline.push(async function map_(value, index) {
return await callback(value, index);
});
return getQueryChain();
},
pick(keys) {
pipeline.push(async function map_(value) {
return await (0, lodash_pick_1.default)(value, keys);
});
return getQueryChain();
},
filter(callback) {
pipeline.push(async function filter_(value, index) {
if (await callback(value, index)) {
return value;
}
throw 'Filtered out';
});
return getQueryChain();
},
forEach(callback) {
pipeline.push(async function forEach_(value, index) {
await callback(value, index);
return value;
});
return getQueryChain();
},
uniq(iterator) {
const callback = (() => {
if ((0, lodash_isstring_1.default)(iterator) || (0, lodash_isarray_1.default)(iterator)) {
return function uniqMap_(value) {
return (0, lodash_pick_1.default)(value, iterator);
};
}
return iterator;
})();
pipeline.push(async function uniq_(value, index) {
const rowToHash = callback ? await callback(value, index) : value;
const hashedRow = hashRecord(rowToHash);
if (hashCache.has(hashedRow)) {
throw 'Duplicate';
}
hashCache.add(hashedRow);
return value;
});
hashCache.clear();
return getQueryChain();
},
/**
* Terminator methods
*/
async process() {
let idx = 0;
for await (const row of readInterface) {
await applyPipeline(row, idx);
idx++;
}
},
async find(callback) {
let idx = 0;
for await (const row of readInterface) {
try {
const value = await applyPipeline(row, idx);
const result = await callback(value);
if (result) {
return value;
}
}
catch (e) { }
idx++;
}
return null;
},
async findIndex(callback) {
let idx = 0;
for await (const row of readInterface) {
try {
const value = await applyPipeline(row, idx);
const result = await callback(value);
if (result) {
return idx;
}
}
catch (e) { }
idx++;
}
return -1;
},
async every(callback) {
let idx = 0;
for await (const row of readInterface) {
try {
const value = await applyPipeline(row, idx);
const result = await callback(value);
if (!result) {
return false;
}
}
catch (e) { }
idx++;
}
return true;
},
async some(callback) {
let idx = 0;
for await (const row of readInterface) {
try {
const value = await applyPipeline(row, idx);
const result = await callback(value);
if (result) {
return true;
}
}
catch (e) { }
idx++;
}
return false;
},
async includes(searchedValue) {
let idx = 0;
for await (const row of readInterface) {
try {
const value = await applyPipeline(row, idx);
if ((0, lodash_isequal_1.default)(value, searchedValue)) {
return true;
}
}
catch (e) { }
idx++;
}
return false;
},
async rows() {
return toDataset();
},
async first(limit) {
return toDataset(limit);
},
async last(limit) {
const dataset = await toDataset();
return dataset.slice(-1 * limit);
},
async skip(offset) {
const dataset = await toDataset();
return dataset.slice(offset);
},
async count() {
let idx = 0;
let count = 0;
for await (const row of readInterface) {
try {
await applyPipeline(row, idx);
count++;
}
catch (e) { }
idx++;
}
return count;
},
async key(property, filterUndefined) {
const dataset = await toDataset();
const values = dataset.map(row => row[property]);
if (filterUndefined) {
return values.filter(Boolean);
}
return values;
},
async toJSON(replacer, space) {
const dataset = await toDataset();
return JSON.stringify(dataset, replacer, space);
},
async toCSV(csvTarget, stringifyOptions) {
const outputStream = (() => {
if (csvTarget instanceof fs_1.WriteStream) {
return csvTarget;
}
return (0, fs_1.createWriteStream)(csvTarget, { encoding: 'utf-8' });
})();
let stringifier = null;
let idx = 0;
for await (const row of readInterface) {
try {
const value = await applyPipeline(row, idx);
if (!stringifier) {
const options = (() => {
if (stringifyOptions) {
return stringifyOptions;
}
return {
header: true,
columns: Object.keys(value),
};
})();
stringifier = (0, csv_stringify_1.stringify)(options);
}
stringifier.write(Object.values(value));
}
catch (e) { }
idx++;
}
if (!!stringifier) {
stringifier.end();
stringifier.pipe(outputStream);
}
},
};
return queryChain;
}
return {
columns(columns) {
const queryChain = getQueryChain();
if (columns && columns.length > 0) {
queryChain.map(value => (0, lodash_pick_1.default)(value, columns));
}
return queryChain;
},
};
}
exports.withCSV = withCSV;
//# sourceMappingURL=index.js.map