itsa-fileuploadhandler
Version:
Handles fileuploads and streams the progress to the client
386 lines (371 loc) • 21.5 kB
JavaScript
;
var fileUtils = require('./file-utils'),
ACCESS_CONTROL_ALLOW_ORIGIN,
fs = require('fs'),
fsp = require('fs-promise'),
utils = require('itsa-utils'),
idGenerator = utils.idGenerator,
DEF_NS_CLIENT_ID = 'ITSA_CL_ID',
DEF_MAX_FILESIZE = 100*1024*1024, // 100Mb
TMP_FILE = 'tmp-file',
REVIVER = function(key, value) {
return ((typeof value==='string') && value.itsa_toDate()) || value;
},
getFns;
require('itsa-jsext');
require('fs-extra');
require('itsa-writestream-promise'); // extends fs.WriteStream with .endPromise
/**
* The modules.export-function, which returns an object with 2 properties: `generateClientId` and `recieveFile`,
* which are both functions.
*
* `generateClientId` generates an unique clientId, which clients should use to identify themselves during fileuploads.
*
* `recieveFile` should be invoked for every filechunk that is send to the server.
*
* Both methods expect the client to follow specific rules, as specified by http://itsa.io/docs/io/index.html#io-filetransfer
* Therefore, this module is best used together with the ITSA-framework (http://itsa.io)
*
* IMPORTANT NOTE: this method is build for usage with hapijs.
*
* @method getFns
* @param [tempdir] {String} the folder where the temporarely-file should be created. If not specified,
* then nodejs its temp environment's variable will be used
* @param [maxFileSize] {Number} the max upload filesize to be accepted. Can be overrules per route (when specifying `recieveFile`).
* if not specified, then a value of 10Mb is used
* @param [nsClientId] {String} the namespace that is used as prefix for every unique generated clientId (generated by `generateClientId`).
* if not specified, then `ITSA_CL_ID` is used.
* @return {Object} Object with the properties: `generateClientId` and `recieveFile`
* @since 0.0.1
*/
getFns = function(tempdir, maxFileSize, accessControlAllowOrigin, nsClientId) {
var TMP_DIR = tempdir || process.env.TMP || process.env.TEMP || '/tmp',
NS_CLIENT_ID = nsClientId || DEF_NS_CLIENT_ID,
FILE_TRANSMISSIONS = {},
globalMaxFileSize = maxFileSize,
tmpDirCreated;
TMP_DIR.itsa_endsWith('/') || (TMP_DIR=TMP_DIR+'/');
tmpDirCreated = fileUtils.createDir(TMP_DIR);
ACCESS_CONTROL_ALLOW_ORIGIN = (accessControlAllowOrigin===true) ? '*' : (accessControlAllowOrigin || '');
/**
* Object that holds all transmission id's. The object gets a structure like this:
*
* {
* "ITSA_CL_ID-1": { // client number "1"
* "ITSA-FILETRANS-1": { // transmission number "1", for this client
* cummulatedSize: {Number}, // the total size of all recieved chinks of this transmission
* count: {Number}, // the total amont of chunks that are send for this file (only available when the last chunk-part is recieved)
* filename: {String}, // the client;s filename of the sent file (only available when the last chunk-part is recieved)
* data: {Object}, // additional params that are sent with the request (only available when the last chunk-part is recieved)
* '1': {String}, // the filename of the temporarely written 1st chunk
* '2': {String}, // the filename of the temporarely written 2nd chunk
* '3': {String}, // the filename of the temporarely written 3th chunk
* etc...
* },
* "ITSA-FILETRANS-2": { // transmission number "2", for this client
* cummulatedSize: {Number},
* count: {Number},
* filename: {String},
* data: {Object},
* '1': {String},
* '2': {String},
* '3': {String},
* etc...
* },
* etc...
* },
* "ITSA_CL_ID-2": { // client number "2"
* "ITSA-FILETRANS-1": { // transmission number "1", for this client
* cummulatedSize: {Number},
* count: {Number},
* filename: {String},
* data: {Object},
* '1': {String},
* '2': {String},
* '3': {String},
* etc...
* },
* etc...
* },
* etc...
* }
*
* @property FILE_TRANSMISSIONS
* @type Object
* @default {}
* @private
* @since 0.0.1
*/
return {
/**
* Generates an unique clientId, which clients should use to identify themselves during fileuploads.
* Will invoke `reply` internally.
*
* This methods expects the client to follow specific rules, as specified by http://itsa.io/docs/io/index.html#io-filetransfer
* Therefore, it is best used together with the ITSA-framework (http://itsa.io)
*
* IMPORTANT NOTE: this method is build for usage with hapijs.
*
* @method generateClientId
* @for init
* @param request {Object} hapijs its request-object
* @param response {Object} hapijs its reply-object
* @return serverresponse, with the unique clientId as text/html
* @since 0.0.1
*/
generateClientId: function(request, response) {
var headers = {'Content-Type': 'text/plain'};
ACCESS_CONTROL_ALLOW_ORIGIN && (headers['access-control-allow-origin']=ACCESS_CONTROL_ALLOW_ORIGIN);
response.set(headers)
.status(200)
.send(idGenerator(NS_CLIENT_ID));
},
responseOptions: function(request, reply) {
var requestHeaders = request.headers['access-control-request-headers'];
reply().header('access-control-allow-origin', ACCESS_CONTROL_ALLOW_ORIGIN)
.header('access-control-allow-methods', 'PUT,GET,POST')
.header('access-control-allow-headers', requestHeaders)
.header('access-control-max-age', '1728000')
.header('content-length', '0').code(200);
},
recieveFormFiles: function(request, reply, maxFileSize, callback, waitForCb) {
var form, replyInstance;
if (typeof maxFileSize==='function') {
callback = maxFileSize;
maxFileSize = null;
}
if (!maxFileSize) {
maxFileSize = maxFileSize || globalMaxFileSize || DEF_MAX_FILESIZE;
}
form = new multiparty.Form({
autoFiles: true,
uploadDir: TMP_DIR,
maxFilesSize: maxFileSize
});
return new Promise(function(fulfill, reject) {
form.parse(request.payload, function(err, fields, payload) {
var files,
wrapper;
if (err) {
reply({status: 'Error: max filesize exceeded'}).code(403);
reject();
}
else {
files = payload.uploadfiles.map(function(item) {
return {
fullFilename: item.path,
originalFilename: item.originalFilename
};
});
(typeof callback==='function') && (wrapper=callback(files));
if (waitForCb===false) {
// either intermediate response, or the final response when `callback` did no reply() invocation
replyInstance = reply('OK').code(200);
ACCESS_CONTROL_ALLOW_ORIGIN && (replyInstance.header('access-control-allow-origin', ACCESS_CONTROL_ALLOW_ORIGIN));
Promise.resolve(wrapper).then(function() {
files.forEach(function(item) {
return fileUtils.removeFile(item.fullFilename);
});
}).catch(function(err) {
console.error(err);
});
fulfill();
}
else {
return Promise.resolve(wrapper).then(function() {
if (!reply._replied) {
// either intermediate response, or the final response when `callback` did no reply() invocation
replyInstance = reply('OK').code(200);
ACCESS_CONTROL_ALLOW_ORIGIN && (replyInstance.header('access-control-allow-origin', ACCESS_CONTROL_ALLOW_ORIGIN));
}
files.forEach(function(item) {
return fileUtils.removeFile(item.fullFilename);
});
}).then(fulfill, reject);
}
}
});
});
},
/**
* Recieves and processes filechunks from a client's fileupload.
*
* This methods expects the client to follow specific rules, as specified by http://itsa.io/docs/io/index.html#io-filetransfer
* Therefore, it is best used together with the ITSA-framework (http://itsa.io)
*
* IMPORTANT NOTE: this method is build for usage with hapijs.
*
* @method recieveFile
* @param request {Object} hapijs its request-object
* @param reply {Object} hapijs its reply-object
* @param [maxFileSize] {Number} the max upload filesize to be accepted. If not specified, then the global value
* as set during `import` is being used.
* @param [callback] {Function} the function that should be invoked once all chunks have been processed and the final temporarely
* file has been created. The caalbackFn will be invoked with 2 arguments: `tmpBuildFilename` and `originalFilename`
* `tmpBuildFilename` is the FULL path to the temporarely file
* `originalFilename` is just a filename (without path), as selected on the client
* AFTER the callback gets invoked, tmpBuildFilename will be removed automaticly. Therefore, if you want to
* perform any processing, the callbackFn SHOULD return a Promise: removal will wait for the Promise to be resolved.
* The callbackFn may (but not necessarily) invoke `reply(object)`, which is handy if you want to return any data.
* If so, than reply MUST be invoked with an object, because the client expects this.
* If not, than reply gets invoked automaticly after the callback.
*
* IMPORTANT NOTE: If the callback replies by itself, than it will also need to set the
* 'access-control-allow-origin' headers (if needed). That is: the 5th argument of this method
* is not being used when you manually are replying. These headers will only be needed when using CORS.
* @param [waitForCb=true] {Boolean} whether to wait with the response until the callback has finished
* @return serverresponse, with the unique clientId as text/html
* @since 0.0.1
*/
recieveFile: function(request, response, maxFileSize, callback, waitForCb) {
var filedata = request.payload,
filedataSize = filedata.length,
originalFilename, transId, clientId, partialId, promise, data, totalSize, headers, errorMsg;
originalFilename = request.headers['x-filename'];
transId = request.headers['x-transid'];
clientId = request.headers['x-clientid'];
partialId = request.headers['x-partial'];
totalSize = request.headers['x-total-size'];
// create clientid if not defined:
FILE_TRANSMISSIONS[clientId] || (FILE_TRANSMISSIONS[clientId]={});
// create transid if not defined, and fill the property `cummulatedSize`:
if (!FILE_TRANSMISSIONS[clientId][transId]) {
FILE_TRANSMISSIONS[clientId][transId] = {
cummulatedSize: filedataSize
};
}
else {
FILE_TRANSMISSIONS[clientId][transId].cummulatedSize += filedataSize;
}
if (typeof maxFileSize==='function') {
callback = maxFileSize;
maxFileSize = null;
}
if (!maxFileSize) {
maxFileSize = maxFileSize || globalMaxFileSize || DEF_MAX_FILESIZE;
}
// Abort if the total filesize (of all chunks) exceeds max.
// check for `totalSize`, which can abort every single chunk --> note: not 100% safe,
// a user could manipulate request.headers['x-total-size'] manually.
// Therefore, also check for FILE_TRANSMISSIONS[clientId][transId].cummulatedSize, which is
// more safe method, but can only abort as soon as the cummulated size exceeds.
if ((totalSize>maxFileSize) || (FILE_TRANSMISSIONS[clientId][transId].cummulatedSize>maxFileSize)) {
errorMsg = "Error: max filesize exceeded";
delete FILE_TRANSMISSIONS[clientId][transId];
// to keep memory clean: also remove the clientid when there are no current transmissions
if (FILE_TRANSMISSIONS[clientId].itsa_size()===0) {
delete FILE_TRANSMISSIONS[clientId];
}
headers = {};
ACCESS_CONTROL_ALLOW_ORIGIN && (headers['access-control-allow-origin']=ACCESS_CONTROL_ALLOW_ORIGIN);
response.set(headers)
.status(403)
.send();
return Promise.reject(errorMsg);
}
else {
return tmpDirCreated
.then(fileUtils.getUniqueFilename)
.then(function(fullFilename) {
// fullFilename is the an unique filename that can be used to store the chunk.
var partCount,
wstream = fs.createWriteStream(fullFilename);
// write the chunk:
wstream.write(filedata);
// close the stream: wait until all has finished before continue: use wstream.endPromise():
return wstream.itsa_endPromise().then(function() {
// now save the chunk's filename:
FILE_TRANSMISSIONS[clientId][transId][partialId] = fullFilename;
// if the last part is send, then `originalFilename` and posible additional data `x-data` is defined.
// in which case we can set the property: `count`
// Be aware: the last part me arive sooner than other parts!
if (originalFilename) {
FILE_TRANSMISSIONS[clientId][transId].count = parseInt(partialId, 10);
FILE_TRANSMISSIONS[clientId][transId].filename = originalFilename;
// store any params that might have been sent with the request:
data = request.headers['x-data'];
if (data) {
try {
FILE_TRANSMISSIONS[clientId][transId].data = JSON.parse(data, REVIVER);
}
catch(err) {
console.log(err);
FILE_TRANSMISSIONS[clientId][transId].data = {};
}
}
else {
FILE_TRANSMISSIONS[clientId][transId].data = {};
}
}
// if all parts are processed, we can build the final file:
partCount = FILE_TRANSMISSIONS[clientId][transId].count;
if (partCount && (FILE_TRANSMISSIONS[clientId][transId].itsa_size()===(partCount+4))) {
// define any params (stored at FILE_TRANSMISSIONS[clientId][transId].data)
// and make them available at request.params:
request.params || (request.params={});
request.params.itsa_merge(FILE_TRANSMISSIONS[clientId][transId].data);
// return a Promise, that resolves with the unique filename of the rebuild file
// `fileUtils.getFinalFile` will rebuild and take care of removal of the intermediate chunk-files:
promise = fileUtils.getFinalFile(TMP_DIR, FILE_TRANSMISSIONS[clientId][transId]);
}
else {
// intermediate response:
// resolve without any data
promise = Promise.resolve();
}
return promise.then(function(filedata) {
// if `filedata` is there, than it is the full-filename of the build-file.
// In which case all has been processed.
// If there is no `filedata` than it is an intermediate request, and we should reply without calling the callback
var wrapper = ((filedata && (typeof callback==='function')) ? callback(filedata.tmpBuildFilename, filedata.originalFilename) : null);
// depending on the configuration, we response to the client immediately, or we wait for the callback to finish:
if (waitForCb===false) {
headers = {'Content-Type': 'text/plain'};
ACCESS_CONTROL_ALLOW_ORIGIN && (headers['access-control-allow-origin']=ACCESS_CONTROL_ALLOW_ORIGIN);
response.set(headers)
.status(200)
.send({status: filedata ? 'OK' : 'BUSY'});
if (filedata) {
Promise.resolve(wrapper).then(function() {
delete FILE_TRANSMISSIONS[clientId][transId];
// to keep memory clean: also remove the clientid when there are no current transmissions
if (FILE_TRANSMISSIONS[clientId].itsa_size()===0) {
delete FILE_TRANSMISSIONS[clientId];
}
fileUtils.removeFile(filedata.tmpBuildFilename);
}).catch(function(err) {
console.error(err);
});
}
}
else {
return Promise.resolve(wrapper).then(function() {
headers = {'Content-Type': 'text/plain'};
if (!response.headerSent) {
ACCESS_CONTROL_ALLOW_ORIGIN && (headers['access-control-allow-origin']=ACCESS_CONTROL_ALLOW_ORIGIN);
response.set(headers)
.status(200)
.send({status: filedata ? 'OK' : 'BUSY'});
}
if (filedata) {
delete FILE_TRANSMISSIONS[clientId][transId];
// to keep memory clean: also remove the clientid when there are no current transmissions
if (FILE_TRANSMISSIONS[clientId].itsa_size()===0) {
delete FILE_TRANSMISSIONS[clientId];
}
return fileUtils.removeFile(filedata.tmpBuildFilename);
}
});
}
});
});
})
.catch(function(err) {
console.error(err);
throw new Error(err);
});
}
}
};
};
module.exports = getFns;