excel2json
Version:
excel2json
204 lines (203 loc) • 8.28 kB
JavaScript
;
/**
* @fileOverview excel parser
* @name parser
* @author Yuhei Aihara
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const child_process_1 = __importDefault(require("child_process"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const zlib_1 = __importDefault(require("zlib"));
const lodash_1 = __importDefault(require("lodash"));
const libxmljs_1 = __importDefault(require("libxmljs"));
const jszip_1 = __importDefault(require("jszip"));
const cell_1 = __importDefault(require("./cell"));
const logger_1 = __importDefault(require("../logger"));
const XML_NS = { a: 'http://schemas.openxmlformats.org/spreadsheetml/2006/main' };
async function asText(zip, files, sheetNum) {
const raw = zip.files[`xl/worksheets/sheet${sheetNum}.xml`];
const contents = raw && await raw.async('text');
if (!contents) {
return;
}
files.sheets.push({
num: sheetNum,
contents,
});
await asText(zip, files, sheetNum + 1);
}
class ExcelParser {
constructor() {
this.processes = 5;
}
/**
* @param {String} filepath
* @param {Array} sheets
*/
async extractFiles(filepath, sheets) {
const files = {
strings: { contents: '' },
book: { contents: '' },
sheets: [],
};
return new Promise((resolve, reject) => {
fs_1.default.readFile(filepath, 'binary', async (err, data) => {
if (err || !data) {
return reject(err || new Error('data not exists'));
}
let zip;
try {
zip = await jszip_1.default.loadAsync(data, { base64: false });
}
catch (e) {
logger_1.default.error(e.stack);
return reject(e);
}
const stringsRaw = zip && zip.files && zip.files['xl/sharedStrings.xml'];
const stringsContents = stringsRaw && await stringsRaw.async('text');
if (!stringsContents) {
return reject(new Error('xl/sharedStrings.xml not exists (maybe not xlsx file)'));
}
files.strings.contents = stringsContents;
const bookRaw = zip && zip.files && zip.files['xl/workbook.xml'];
const bookContents = bookRaw && await bookRaw.async('text');
if (!bookContents) {
return reject(new Error('xl/workbook.xml not exists (maybe not xlsx file)'));
}
files.book.contents = bookContents;
if (sheets && sheets.length) {
await Promise.all(lodash_1.default.map(sheets, (sheetNum) => {
return (async () => {
const raw = zip.files[`xl/worksheets/sheet${sheetNum}.xml`];
const contents = raw && await raw.async('text');
if (!contents) {
return reject(new Error(`sheet ${sheetNum} not exists`));
}
files.sheets.push({
num: sheetNum,
contents,
});
})();
}));
}
else {
await asText(zip, files, 1);
}
resolve(files);
});
});
}
/**
* @param {Object} files
*/
async extractData(files) {
let strings;
let sheetNames = [];
let sheets;
try {
const book = libxmljs_1.default.parseXml(files.book.contents);
strings = libxmljs_1.default.parseXml(files.strings.contents);
sheetNames = lodash_1.default.map(book.find('//a:sheets//a:sheet', XML_NS), (tag) => {
const name = tag.attr('name');
return name ? name.value() : '';
});
// sheets and sheetNames were retained the arrangement.
sheets = lodash_1.default.map(files.sheets, (sheetObj) => {
return {
num: sheetObj.num,
name: sheetNames[sheetObj.num - 1],
contents: sheetObj.contents,
xml: libxmljs_1.default.parseXml(sheetObj.contents),
};
});
}
catch (e) {
logger_1.default.error(e.stack);
throw e;
}
const result = [];
await Promise.all(lodash_1.default.map(sheets, (sheetObj) => {
return (async () => {
const sheet = sheetObj.xml;
const cellNodes = sheet.find('/a:worksheet/a:sheetData/a:row/a:c', XML_NS);
const tasks = [];
if (cellNodes.length < 20000) {
result.push({
num: sheetObj.num,
name: sheetObj.name,
cells: cell_1.default(cellNodes, strings, XML_NS),
});
return;
}
const nodes = Math.floor(cellNodes.length / this.processes);
lodash_1.default.times(this.processes, (i) => {
tasks.push({
start: nodes * i,
end: i + 1 === this.processes ? cellNodes.length : nodes * (i + 1),
});
});
const stringsBuf = await new Promise((resolve, reject) => {
zlib_1.default.deflate(files.strings.contents, (err, buf) => {
if (err) {
return reject(err);
}
resolve(buf);
});
});
const sheetsBuf = await new Promise((resolve, reject) => {
zlib_1.default.deflate(sheetObj.contents, (err, buf) => {
if (err) {
return reject(err);
}
resolve(buf);
});
});
const sendData = {
strings: stringsBuf.toString('base64'),
sheets: sheetsBuf.toString('base64'),
ns: XML_NS,
};
const cells = await Promise.all(lodash_1.default.map(tasks, (task) => {
const _cellConverter = child_process_1.default.fork(path_1.default.join(__dirname, './cell'));
let _err;
let _result;
return new Promise((resolve, reject) => {
_cellConverter.on('message', (data) => {
_err = data.err;
if (data.result) {
_result = data.result;
}
_cellConverter.send({ exit: true });
});
_cellConverter.on('exit', (code) => {
if (code !== 0) {
return reject(_err || code);
}
resolve(_result);
});
_cellConverter.send(lodash_1.default.assign({
start: task.start,
end: task.end,
}, sendData));
});
}));
result.push({
num: sheetObj.num,
name: sheetObj.name,
cells: lodash_1.default.flatten(cells),
});
})();
}));
return result;
}
async execute(filePath, sheets) {
const files = await this.extractFiles(filePath, sheets);
const result = await this.extractData(files);
return result;
}
}
exports.default = new ExcelParser();