@beenotung/tslib
Version:
utils library in Typescript
172 lines • 6.88 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.read_csv_file_sync = exports.read_csv_file = exports.write_csv_file_sync = exports.write_csv_file = void 0;
const tslib_1 = require("tslib");
/**
* for processing large data set, then will run out of memory if use csv.ts directly
* */
const fs_1 = tslib_1.__importDefault(require("fs"));
const csv_1 = require("./csv");
const generator_1 = require("./generator");
function wrapLine(s) {
return s.replace(/\n/g, '\\n');
}
function unwrapLine(s) {
return s.replace(/\\n/g, '\n');
}
function wrapObjectLine(o, titles) {
const res = {};
for (const key of titles) {
let value = o[key];
if (typeof value === 'string') {
value = wrapLine(value);
}
res[key] = value;
}
return res;
}
function unwrapObjectLine(o, titles) {
for (const key of titles) {
let value = o[key];
if (typeof value === 'string') {
value = unwrapLine(value);
o[key] = value;
}
}
}
function write_csv_file(filename, records, options) {
return new Promise((resolve, reject) => {
let stream;
try {
stream = fs_1.default.createWriteStream(filename);
const is_wrap_line = options === null || options === void 0 ? void 0 : options.wrap_line;
const separator = options === null || options === void 0 ? void 0 : options.separator;
const delimiter = options === null || options === void 0 ? void 0 : options.delimiter;
let titles = (options === null || options === void 0 ? void 0 : options.titles) || csv_1.json_to_csv_titles(records);
if (is_wrap_line) {
titles = titles.map(wrapLine);
}
let first = true;
for (let record of records) {
if (is_wrap_line) {
record = wrapObjectLine(record, titles);
}
let rows = csv_1.json_to_csv([record], titles);
if (first) {
first = false; // include titles
if (options === null || options === void 0 ? void 0 : options.skip_titles) {
rows = [rows[1]]; // skip titles
}
}
else {
rows = [rows[1]]; // skip titles
}
const text = csv_1.to_csv(rows, separator, delimiter);
stream.write(text);
}
stream.close();
stream.on('close', resolve);
stream.on('error', reject);
}
catch (e) {
stream === null || stream === void 0 ? void 0 : stream.close();
reject(e);
}
});
}
exports.write_csv_file = write_csv_file;
function write_csv_file_sync(filename, records, options) {
const is_wrap_line = options === null || options === void 0 ? void 0 : options.wrap_line;
const separator = options === null || options === void 0 ? void 0 : options.separator;
const delimiter = options === null || options === void 0 ? void 0 : options.delimiter;
let titles = (options === null || options === void 0 ? void 0 : options.titles) || csv_1.json_to_csv_titles(records);
if (is_wrap_line) {
titles = titles.map(wrapLine);
}
let first = true;
fs_1.default.writeFileSync(filename, '');
for (let record of records) {
if (is_wrap_line) {
record = wrapObjectLine(record, titles);
}
let rows = csv_1.json_to_csv([record], titles);
if (first) {
first = false; // include titles
if (options === null || options === void 0 ? void 0 : options.skip_titles) {
rows = [rows[1]]; // skip titles
}
}
else {
rows = [rows[1]]; // skip titles
}
const text = csv_1.to_csv(rows, separator, delimiter);
fs_1.default.appendFileSync(filename, text);
}
}
exports.write_csv_file_sync = write_csv_file_sync;
function read_csv_file(filename, options) {
return tslib_1.__asyncGenerator(this, arguments, function* read_csv_file_1() {
var e_1, _a;
const separator = options === null || options === void 0 ? void 0 : options.separator;
const delimiter = options === null || options === void 0 ? void 0 : options.delimiter;
const is_wrap_line = options === null || options === void 0 ? void 0 : options.wrap_line;
const stream = fs_1.default.createReadStream(filename);
let titles;
let first = true;
try {
for (var _b = tslib_1.__asyncValues(generator_1.stream_lines(stream)), _c; _c = yield tslib_1.__await(_b.next()), !_c.done;) {
const line = _c.value;
if (first) {
titles = csv_1.from_csv(line, separator, delimiter)[0];
if (is_wrap_line) {
titles = titles.map(unwrapLine);
}
first = false;
continue;
}
const text = line + '\n';
const records = csv_1.csv_to_json(csv_1.from_csv(text, separator, delimiter), titles);
const record = records[0];
if (record) {
if (is_wrap_line) {
unwrapObjectLine(record, titles);
}
yield yield tslib_1.__await(record);
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) yield tslib_1.__await(_a.call(_b));
}
finally { if (e_1) throw e_1.error; }
}
stream.close();
});
}
exports.read_csv_file = read_csv_file;
function* read_csv_file_sync(filename, options) {
const separator = options === null || options === void 0 ? void 0 : options.separator;
const delimiter = options === null || options === void 0 ? void 0 : options.delimiter;
const is_wrap_line = options === null || options === void 0 ? void 0 : options.wrap_line;
const text = fs_1.default.readFileSync(filename).toString();
const lines = text.split('\n');
let titles = csv_1.from_csv(lines[0])[0];
if (is_wrap_line) {
titles = titles.map(unwrapLine);
}
for (let i = 1; i < lines.length; i++) {
const text = lines[i] + '\n';
const records = csv_1.csv_to_json(csv_1.from_csv(text, separator, delimiter), titles);
const record = records[0];
if (record) {
if (is_wrap_line) {
unwrapObjectLine(record, titles);
}
yield record;
}
}
}
exports.read_csv_file_sync = read_csv_file_sync;
//# sourceMappingURL=csv-file.js.map