datax
Version:
Wrapper fo Buffer for nodejs.
556 lines (507 loc) • 13.7 kB
JavaScript
;
var Promise = require('bluebird');
var AdmZip = require('adm-zip');
var fs = require('fs');
var Buffer = require('buffer').Buffer;
var crc = require('crc');
var crypto = require('crypto');
var fs = require('fs');
var http = require('http');
var https = require('https');
var mkdirp = require('mkdirp');
var path = require('path');
var tmp = require('tmp');
var URL = require('url');
var unrar = require('unrar-js');
var zlib = require('zlib');
class Data {
constructor(buffer) {
this._ = {
buffer: buffer ? (Buffer.isBuffer(buffer) ? buffer : new Buffer(buffer)) : new Buffer('')
};
}
get buffer() {
return this._.buffer;
}
get crc32() {
if (this._.crc32 === undefined) {
this._.crc32 = crc.crc32(this.buffer);
}
return this._.crc32;
}
get md5() {
if (this._.md5 === undefined) {
var hash = crypto.createHash('md5');
hash.update(this._.buffer);
this._.md5 = hash.digest('hex');
}
return this._.md5;
}
get sha256() {
if (this._.sha256 === undefined) {
var hash = crypto.createHash('sha256');
hash.update(this._.buffer);
this._.sha256 = hash.digest('hex');
}
return this._.sha256;
}
toString() {
return this._.buffer.toString();
}
}
module.exports = Data;
/**
* setCachePolicy() sepcifies cache location and expiry time.
*
* @param {Object} policy - the cache policy
*/
Data.setCachePolicy = function(policy) {
if (policy === undefined || policy === null) {
Data.cachePolicy = null;
return;
}
if (typeof(policy) != 'object') {
throw new TypeError('Invalid policy');
}
if (!policy.location) {
throw new TypeError('Missing cache location');
}
if (!policy.location || typeof(policy.location) != 'string') {
throw new TypeError('Invalid cache location');
}
if (policy.duration && typeof(policy.duration) != 'number') {
throw new TypeError('Invalid cache duration');
}
Data.cachePolicy = {
location: policy.location,
duration: policy.duration || 360000
};
};
/**
* from() loads data from a given uri
*
* @param {String} uri - the uri to get data from.
* @param (Object) options - options for http or fs
* @return {Promise} a promise of the Buffer instance
*/
Data.from = function(uri, options = { }) {
var url = URL.parse(uri);
if (url && url.protocol) {
if (url.protocol == 'http:' || url.protocol == 'https:') {
return fromHTTP(uri, options);
} else {
return Promise.reject(new Error(`Invalid or unsupported URI ${uri}`));
}
} else {
return fromFS(uri, options);
}
};
function fromFS(filename, options) {
return new Promise((resolve, reject) => {
fs.stat(filename, (error, stats) => {
if (error) {
reject(error);
return;
}
fs.readFile(filename, options, (error, buffer) => {
if (error) {
reject(error);
return;
}
var data = new Data(buffer);
data.stats = stats;
resolve(data);
});
});
});
}
function fromHTTP(urlText, options) {
var urlTextHash = (new Data(urlText)).md5;
// Cache is diabled or do not use cache.
if (!Data.cachePolicy || (options && options.cache === false)) {
return fromHTTPDirect(urlText, urlTextHash, options);
}
// Get data from cache.
return fromFS(Data.cachePolicy.location+'/'+urlTextHash)
// Cache is not available, get from server.
.catch(() => {
return fromHTTPDirect(urlText, urlTextHash, options);
})
// Cache is available
.then((cache) => {
// This is from server, not cache
if (cache.httpStatusCode) {
return Promise.resolve(cache);
}
// Cache is diabled while retrieving data from file system.
if (!Data.cachePolicy) {
return fromHTTPDirect(urlText, urlTextHash, options);
}
// Return data from cache if it is still fresth or no expriy time.
if (Data.cachePolicy.duration < 0 ||
Date.now() - cache.stats.mtime.getTime() <= Data.cachePolicy.duration) {
cache.httpStatusCode = 200;
return Promise.resolve(cache);
}
// Get data from server, return data from cache
// if it failed or http status is not 200 (ok)
return fromHTTPDirect(urlText, urlTextHash, options)
.then((data) => {
if (data.httpStatusCode !== 200) {
cache.httpStatusCode = 200;
return Promise.resolve(cache);
}
return Promise.resolve(data);
})
.catch((error) => {
cache.httpStatusCode = 200;
cache.refreshError = error;
return Promise.resolve(cache);
});
});
}
function fromHTTPDirect(urlText, urlTextHash, options) {
return new Promise((resolve, reject) => {
fromHTTPDirectCallback(urlText, options, (error, data) => {
if (error) {
reject(error);
} else if (data.httpStatusCode == 200 && Data.cachePolicy) {
data.writeToFile(Data.cachePolicy.location + '/' + urlTextHash)
.then(() => {
resolve(data);
return null;
})
.catch(() => {
resolve(data);
return null;
})
} else {
resolve(data);
}
});
});
}
function fromHTTPDirectCallback(urlText, options, callback) {
var url = URL.parse(urlText);
var params = {
path: options.proxy ? URL.format(url) : url.path,
headers: options.headers || httpHeaders,
agent: options.agent || false
};
if (url.protocol == 'https:') {
if (options.rejectUnauthorized !== undefined) {
params.rejectUnauthorized = options.rejectUnauthorized;
}
if (options.requestCert !== undefined) {
params.requestCert = options.requestCert;
}
}
if (options.proxy) {
if (typeof(options.proxy) !== 'string' ) {
callback(new Error('Invalid option proxy'));
return;
}
let colonIndex = options.proxy.lastIndexOf(':');
if (colonIndex < 0) {
callback(new Error('Missing port in option proxy'));
return;
}
let hostname = options.proxy.substring(0, colonIndex);
let port = parseInt(options.proxy.substring(colonIndex+1));
if (isNaN(port)) {
callback(new Error('Invalid port in option proxy'));
return;
}
params.hostname = hostname;
params.port = port;
} else {
params.hostname = url.hostname;
if (url.port) {
params.port = url.port;
}
}
if (options.localAddress) {
params.localAddress = options.localAddress;
}
var req = (url.protocol == 'https:' ? https : http)
.get(params, (res) => {
res.on('error', (error) => {
callback(error);
});
var chunks = [];
res.on('data', (chunk) => {
chunks.push(chunk);
});
res.on('end', () => {
var encoding = res.headers['content-encoding'];
var buffer = Buffer.concat(chunks);
if (encoding == 'gzip') {
zlib.gunzip(buffer, function(error, decoded) {
if (error) {
callback(error);
} else {
callback(null, dataOfHTTP(decoded, res.statusCode));
}
});
}
else if (encoding == 'deflate') {
zlib.inflate(buffer, function(error, decoded) {
if (error) {
callback(error);
} else {
callback(null, dataOfHTTP(decoded, res.statusCode));
}
});
}
else {
callback(null, dataOfHTTP(buffer, res.statusCode));
}
});
});
req.on('socket', (socket) => {
socket.setTimeout(options.timeout || 60000);
socket.on('timeout', () => {
req.abort();
});
});
req.on('error', (error) => {
callback(error);
});
req.end();
}
var httpHeaders = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"accept-encoding": "gzip, deflate",
"accept-language": "en-US",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"
};
function dataOfHTTP(buffer, httpStatusCode) {
var data = new Data(buffer);
Object.defineProperty(data, 'httpStatusCode', {
value: httpStatusCode
});
return data;
}
/**
* writeTemporaryFile() writes data into a temporary file.
*
* @return {Promise} a promise of the generated filename.
*/
Data.prototype.writeToTemporaryFile = function() {
var data = this;
return new Promise((resolve, reject) => {
tmp.tmpName((error, filename) => {
if (error) {
reject(error);
} else {
resolve(filename);
}
});
})
.then((filename) => {
return data.writeToFile(filename)
.then(() => {
return Promise.resolve(filename);
});
});
};
Data.writeToTemporaryFile = function(data) {
return data.writeToTemporaryFile();
};
/**
* writeToFile() writes data into a temporary file.
*
* @param {String} filename - the full path to file.
* @return {Promise} a empty promise.
*/
Data.prototype.writeToFile = function(filename) {
var buffer = this.buffer;
var dirname = path.dirname(filename);
if (!dirname) {
return Promise.reject(new Error(`Invalid filename ${filename}`));
}
return new Promise((resolve, reject) => {
mkdirp(dirname, (error) => {
if (error) {
reject(error);
} else {
fs.writeFile(filename, buffer, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
});
}
});
});
};
/**
* unpack() unpacks the data to list of items.
*
* @param {String} format - the format zip or rar
* @return {Promise} a promise of the entries
*/
Data.prototype.unpack = function(format) {
switch (format) {
case 'zip':
return this.unpackZip();
case 'rar':
return this.unpackRar();
default:
return Promise.reject(new Error('Unsupported format '+ format));
}
};
/**
* unpackZip() unpacks the data in zip to list of items.
*
* @return {Promise} a promise of the entries
*/
Data.prototype.unpackZip = function() {
try {
var zip = new AdmZip(this.buffer);
} catch (msg) {
return Promise.reject(new Error(msg));
}
return Promise.resolve(zip)
.then((zip) => {
var entries = zip.getEntries();
if (!entries) {
return Promise.reject(new Error('Unable to unzip the data'));
} else {
var items = [];
for (let i = 0; i < entries.length; i++) {
var entry = entries[i];
if (!entry.isDirectory) {
try {
var buffer = entry.getData();
} catch (msg) {
return Promise.reject(new Error(msg));
}
items.push({
name: entry.entryName,
data: new Data(buffer)
});
}
}
return Promise.resolve(items);
}
})
.catch((error) => {
return Promise.reject(error);
});
};
Data.unpackZip = function(data) {
return data.unpackZip();
};
/**
* unpackRar() unpacks the data in rar to list of items.
*
* @return {Promise} a promise of the entries
*/
Data.prototype.unpackRar = function() {
return this.writeToTemporaryFile()
.then((filename) => {
try {
var entries = unrar.unrarSync(filename);
} catch (msg) {
return removeFile(filename)
.then(() => {
return Promise.reject(new Error(msg));
});
}
if (!entries) {
return removeFile(filename)
.then(() => {
return Promise.reject(new Error('Unable to unpack rar'));
});
}
return removeFile(filename)
.then(() => {
var items = [];
entries.forEach((entry) => {
items.push({
name: entry.filename,
data: new Data(entry.fileData)
});
})
return Promise.resolve(items);
});
});
};
Data.unpackRar = function(data) {
return data.unpackRar();
}
function removeFile(filename) {
return new Promise((resolve) => {
fs.unlink(filename, () => {
resolve();
});
});
}
/**
* compress() compresses the data in given encoding
*
* @param {String} encoding - the encoding gzip | (zlib = deflate).
* @return {Promise} a promise of the compressed data
*/
Data.prototype.compress = function(encoding) {
switch (encoding) {
case 'zlib':
case 'deflate':
return new Promise((resolve, reject) => {
zlib.deflate(this.buffer, (error, buffer) => {
if (error) {
reject(error);
} else {
resolve(new Data(buffer));
}
});
});
case 'gzip':
return new Promise((resolve, reject) => {
zlib.gzip(this.buffer, (error, buffer) => {
if (error) {
reject(error);
} else {
resolve(new Data(buffer));
}
});
});
default:
return Promise.reject(new Error('Unsupported encoding ' + encoding));
}
};
/**
* decompress() decompresses the data in given encoding
*
* @param {String} encoding - the encoding gzip | (zlib = deflate).
* @return {Promise} a promise of the decompressed data
*/
Data.prototype.decompress = function(encoding) {
switch (encoding) {
case 'zlib':
case 'deflate':
return new Promise((resolve, reject) => {
zlib.inflate(this.buffer, (error, buffer) => {
if (error) {
reject(error);
} else {
resolve(new Data(buffer));
}
});
});
case 'gzip':
return new Promise((resolve, reject) => {
zlib.gunzip(this.buffer, (error, buffer) => {
if (error) {
reject(error);
} else {
resolve(new Data(buffer));
}
});
});
default:
return Promise.reject(new Error('Unsupported encoding ' + encoding));
}
};