tiny-storage-client
Version:
Tiny node client to request distributed AWS S3 or the OpenStack Swift Object Storage.
1,154 lines (1,053 loc) • 112 kB
JavaScript
/** Init retry delay for "Rock-req" tests */
const rock = require('rock-req');
const _rockDefaults = { ...rock.defaults };
_rockDefaults.retryDelay = 200;
global.rockReqConf = _rockDefaults;
const s3 = require('../index');
const assert = require('assert');
const nock = require('nock');
const fs = require('fs');
const path = require('path');
var stream = require('stream');
let storage = {};
const url1S3 = 'https://s3.gra.first.cloud.test';
const url2S3 = 'https://s3.de.first.cloud.test';
let dataStream = '';
const outputStreamFunction = function () {
dataStream = '';
const outputStream = new stream.Writable();
outputStream._write = function (chunk, encoding, done) {
dataStream += chunk;
done();
};
outputStream.on('error', (err) => {
console.log('Error Stream:', err.toString());
dataStream = '';
});
return outputStream;
};
/** ASSETS for download/upload */
const fileTxtPath = path.join(__dirname, 'assets', 'file.txt');
const fileTxt = fs.readFileSync(fileTxtPath).toString();
const fileXmlPath = path.join(__dirname, 'assets', 'files.xml');
const fileXml = fs.readFileSync(fileXmlPath).toString();
/** ASSETS for List objects Requests */
const _listObjectsResponseXML = fs.readFileSync(path.join(__dirname, "./assets", 'listObjects.response.xml'));
const _listObjectsResponseJSON = require('./assets/listObjects.response.json');
describe('S3 SDK', function () {
beforeEach(function() {
storage = s3([{
accessKeyId : '2371bebbe7ac4b2db39c09eadf011661',
secretAccessKey: '9978f6abf7f445566a2d316424aeef2',
url : url1S3.replace('https://', ''),
region : 'gra',
buckets : {
invoices : "invoices-gra-1234"
}
},
{
accessKeyId : '2371bebbe7ac4b2db39c09eadf011661',
secretAccessKey: '9978f6abf7f445566a2d316424aeef2',
url : url2S3.replace('https://', ''),
region : 'de',
buckets : {
invoices : "invoices-de-8888"
}
}]);
});
describe('constructor/getConfig/setConfig/setTimeout', function () {
it("should create a new s3 instance if the authentication is provided as Object", function () {
const _authS3 = { accessKeyId: '-', secretAccessKey: '-', region: '-', url: '-' };
const _storage = s3(_authS3);
const _config = _storage.getConfig();
assert.strictEqual(_config.timeout, 5000);
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(_config.storages.length, 1);
assert.strictEqual(JSON.stringify(_config.storages[0]), JSON.stringify(_authS3));
});
it("should create a new s3 instance if the authentication is provided as List of objects", function () {
const _authS3 = [{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, region: 7, url: 8 }, { accessKeyId: 9, secretAccessKey: 10, region: 11, url: 12 }];
const _storage = s3(_authS3);
const _config = _storage.getConfig();
assert.strictEqual(_config.timeout, 5000);
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(_config.storages.length, 3);
assert.strictEqual(JSON.stringify(_config.storages), JSON.stringify(_authS3));
});
it("should throw an error if authentication values are missing", function() {
assert.throws(function(){ s3({}); }, Error);
/** As object */
assert.throws(function(){ s3({accessKeyId: '', secretAccessKey: '', url: ''}); }, Error); // missing region
assert.throws(function(){ s3({accessKeyId: '', secretAccessKey: '', region: ''}); }, Error); // missing url
assert.throws(function(){ s3({accessKeyId: '', url: '', region: ''}); }, Error); // missing secretAccessKey
assert.throws(function(){ s3({secretAccessKey: '', url: '', region: ''}); }, Error); // missing accessKeyId
/** As array */
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, url: 8 }]); }, Error); // missing region
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, region: 8 }]); }, Error); // missing url
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, region: 6, url: 8 }]); }, Error); // missing secretAccessKey
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { secretAccessKey: 5, region: 6, url: 8 }]); }, Error); // missing accessKeyId
});
it("should set a new config", function () {
const _storage = s3({ accessKeyId: '-', secretAccessKey: '-', region: '-', url: '-' });
const _authS3 = [{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, region: 7, url: 8 }];
_storage.setConfig(_authS3);
const _config = _storage.getConfig();
assert.strictEqual(_config.timeout, 5000);
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(_config.storages.length, 2);
assert.strictEqual(JSON.stringify(_config.storages), JSON.stringify(_authS3));
});
it("should create a new instance", function () {
const _storage = s3({ accessKeyId: '1234', secretAccessKey: '4567', region: 'gra', url: 'url.gra.s3.ovh.io' });
const _storage2 = s3({ accessKeyId: 'abcd', secretAccessKey: 'efgh', region: 'sbg', url: 'url.sbg.s3.ovh.io' });
assert.strictEqual(_storage.getConfig().storages.length, 1);
assert.strictEqual(_storage2.getConfig().storages.length, 1);
assert.strictEqual(_storage.getConfig().storages[0].accessKeyId, '1234');
assert.strictEqual(_storage.getConfig().storages[0].secretAccessKey, '4567');
assert.strictEqual(_storage.getConfig().storages[0].region, 'gra');
assert.strictEqual(_storage.getConfig().storages[0].url, 'url.gra.s3.ovh.io');
assert.strictEqual(_storage2.getConfig().storages[0].accessKeyId, 'abcd');
assert.strictEqual(_storage2.getConfig().storages[0].secretAccessKey, 'efgh');
assert.strictEqual(_storage2.getConfig().storages[0].region, 'sbg');
assert.strictEqual(_storage2.getConfig().storages[0].url, 'url.sbg.s3.ovh.io');
});
it("should set a new timeout value", function() {
const _storage = s3({ accessKeyId: '-', secretAccessKey: '-', region: '-', url: '-' });
assert.strictEqual(_storage.getConfig().timeout, 5000);
_storage.setTimeout(10000);
assert.strictEqual(_storage.getConfig().timeout, 10000);
});
});
describe('headBucket', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should return code 200, and request signed with AWS4', function (done) {
const nockRequest = nock(url1S3,
{
reqheaders: {
'x-amz-content-sha256': () => true,
'x-amz-date': () => true,
'authorization': () => true,
'host': () => true
}
}).intercept("/customBucket", "HEAD").reply(200, '');
storage.headBucket('customBucket', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should return code 200 and request a bucket as ALIAS', function (done) {
const nockRequest = nock(url1S3,
{
reqheaders: {
'x-amz-content-sha256': () => true,
'x-amz-date': () => true,
'authorization': () => true,
'host': () => true
}
}).intercept("/invoices-gra-1234", "HEAD").reply(200, '');
storage.headBucket('invoices', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should return code 403 Forbidden', function (done) {
const nockRequest = nock(url1S3).intercept("/customBucket", "HEAD").reply(403, '');
storage.headBucket('customBucket', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 403);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
});
describe("SWITCH TO CHILD STORAGE", function () {
it('should switch to the child storage and return code 200 with bucket as ALIAS', function (done) {
const nockRequestS1 = nock(url1S3).intercept("/invoices-gra-1234", "HEAD").reply(500, '');
const nockRequestS2 = nock(url2S3).intercept("/invoices-de-8888", "HEAD").reply(200, '');
const nockRequestS3 = nock(url1S3).get('/').reply(500);
storage.headBucket('invoices', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
});
});
describe('listBuckets', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should fetch a list of buckets', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '366',
'x-amz-id-2': 'tx606add09487142fa88e67-00641aacf4',
'x-amz-request-id': 'tx606add09487142fa88e67-00641aacf4',
'x-trans-id': 'tx606add09487142fa88e67-00641aacf4',
'x-openstack-request-id': 'tx606add09487142fa88e67-00641aacf4',
date: 'Wed, 22 Mar 2023 07:23:32 GMT',
connection: 'close'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/')
.reply(200, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>89123456:user-feiowjfOEIJW</ID><DisplayName>12345678:user-feiowjfOEIJW</DisplayName></Owner><Buckets><Bucket><Name>invoices</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket><Bucket><Name>www</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
});
storage.listBuckets((err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
"bucket": [
{ "name": "invoices", "creationdate": "2023-02-27T11:46:24.000Z" },
{ "name": "www", "creationdate": "2023-02-27T11:46:24.000Z" }
]
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should return an error if credentials are not correct', function (done) {
const _header = {
'x-amz-request-id': 'BEFVPYB9PM889VMS',
'x-amz-id-2': 'Pnby9XcoK7X/GBpwr+vVV/X3XyadxsUkTzGdSJS5zRMhs2RvZDGroWleytOYGmYRSszFbsaZWUo=',
'content-type': 'application/xml',
'transfer-encoding': 'chunked',
date: 'Wed, 22 Mar 2023 07:37:22 GMT',
server: 'AmazonS3',
connection: 'close'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/')
.reply(403, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>InvalidAccessKeyId</Code><Message>The AWS Access Key Id you provided does not exist in our records.</Message><AWSAccessKeyId>AKIAUO7WHYLVFADDFL57e</AWSAccessKeyId><RequestId>BSTT951V1FREKS2X</RequestId><HostId>zWFC8ZOiZvyxTUgcYjHDD9rmPDG81TCJHkZhAv4zgguuR5I9aeqSFA9Ns4r5PdKy9+9o+xDLpOk=</HostId></Error>";
});
storage.listBuckets((err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 403);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
error: {
code: 'InvalidAccessKeyId',
message: 'The AWS Access Key Id you provided does not exist in our records.',
awsaccesskeyid: 'AKIAUO7WHYLVFADDFL57e',
requestid: 'BSTT951V1FREKS2X',
hostid: 'zWFC8ZOiZvyxTUgcYjHDD9rmPDG81TCJHkZhAv4zgguuR5I9aeqSFA9Ns4r5PdKy9+9o+xDLpOk='
}
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
});
describe("SWITCH TO CHILD STORAGE", function () {
it('should fetch a list of buckets', function (done) {
const nockRequestS1 = nock(url1S3)
.get('/')
.reply(500, '');
const nockRequestS2 = nock(url2S3)
.get('/')
.reply(200, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>89123456:user-feiowjfOEIJW</ID><DisplayName>12345678:user-feiowjfOEIJW</DisplayName></Owner><Buckets><Bucket><Name>invoices</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket><Bucket><Name>www</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.listBuckets((err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
"bucket": [
{ "name": "invoices", "creationdate": "2023-02-27T11:46:24.000Z" },
{ "name": "www", "creationdate": "2023-02-27T11:46:24.000Z" }
]
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}));
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
});
describe("Options 'requestStorageIndex'", function() {
it("should request the first storage and should return an error if the first storage is not available", function(done) {
const nockRequestS1 = nock(url1S3)
.get('/')
.reply(500, '');
storage.listBuckets({ requestStorageIndex: 0 }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 500);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}));
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
done();
});
});
it("should request the second storage and should return an error if the second storage is not available", function(done) {
const nockRequestS1 = nock(url2S3)
.get('/')
.reply(500, '');
storage.listBuckets({ requestStorageIndex: 1 }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 500);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}));
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
done();
});
});
it("should request the second storage and get a list of buckets", function(done) {
const nockRequestS1 = nock(url2S3)
.get('/')
.reply(200, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>89123456:user-feiowjfOEIJW</ID><DisplayName>12345678:user-feiowjfOEIJW</DisplayName></Owner><Buckets><Bucket><Name>invoices</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket><Bucket><Name>www</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
});
storage.listBuckets({ requestStorageIndex: 1 }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
"bucket": [
{ "name": "invoices", "creationdate": "2023-02-27T11:46:24.000Z" },
{ "name": "www", "creationdate": "2023-02-27T11:46:24.000Z" }
]
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}));
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
done();
});
});
});
});
describe('listFiles', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should fetch a list of objects', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '1887',
'x-amz-id-2': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-amz-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-trans-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-openstack-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
date: 'Mon, 20 Mar 2023 07:14:31 GMT',
connection: 'close'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket')
.query({ 'list-type' : 2 })
.reply(200, () => {
return _listObjectsResponseXML;
});
storage.listFiles('bucket', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should fetch a list of objects from a bucket as ALIAS', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '1887',
'x-amz-id-2': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-amz-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-trans-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-openstack-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
date: 'Mon, 20 Mar 2023 07:14:31 GMT',
connection: 'close'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/invoices-gra-1234')
.query({ 'list-type' : 2 })
.reply(200, () => {
return _listObjectsResponseXML;
});
storage.listFiles('invoices', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should fetch a list of objects with query parameters (prefix & limit)', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '1887',
'x-amz-id-2': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-amz-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-trans-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-openstack-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
date: 'Mon, 20 Mar 2023 07:14:31 GMT',
connection: 'close'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket')
.query({
"list-type" : 2,
"prefix" : "document",
"max-keys" : 2
})
.reply(200, () => {
return _listObjectsResponseXML;
});
storage.listFiles('bucket', { queries: { "prefix": "document", "max-keys": 2 } }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it("should return an error if the bucket does not exist", function (done) {
const _headers = {
'content-type': 'application/xml',
'x-amz-id-2': 'tx8fa5f00b19af4756b9ef3-0064184d77',
'x-amz-request-id': 'tx8fa5f00b19af4756b9ef3-0064184d77',
'x-trans-id': 'tx8fa5f00b19af4756b9ef3-0064184d77',
'x-openstack-request-id': 'tx8fa5f00b19af4756b9ef3-0064184d77',
date: 'Mon, 20 Mar 2023 12:11:35 GMT',
'transfer-encoding': 'chunked',
connection: 'close'
};
const _expectedBody = {
error: {
code: 'NoSuchBucket',
message: 'The specified bucket does not exist.',
requestid: 'txe285e692106542e88a2f5-0064184e80',
bucketname: 'buckeeeet'
}
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_headers)
.get('/buckeeeet')
.query({
"list-type" : 2
})
.reply(404, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>NoSuchBucket</Code><Message>The specified bucket does not exist.</Message><RequestId>txe285e692106542e88a2f5-0064184e80</RequestId><BucketName>buckeeeet</BucketName></Error>";
});
storage.listFiles('buckeeeet', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_expectedBody));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
});
describe("SWITCH TO CHILD STORAGE", function () {
it('should fetch a list of objects', function (done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket')
.query({ 'list-type' : 2 })
.reply(500, '');
const nockRequestS2 = nock(url2S3)
.get('/bucket')
.query({ 'list-type' : 2 })
.reply(200, () => {
return _listObjectsResponseXML;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.listFiles('bucket', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}));
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
});
});
describe('downloadFile', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should download a file', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should download a file as STREAM', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
storage.downloadFile('bucket', 'file.docx', { output: outputStreamFunction }, function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(dataStream, fileTxt);
assert.strictEqual(nockRequest.pendingMocks().length, 0);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
done();
});
});
it('should download a file from an alias', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
};
const nockRequest = nock(url1S3, {
reqheaders: {
'x-amz-content-sha256': () => true,
'x-amz-date': () => true,
'authorization': () => true,
'host': () => true
}
})
.defaultReplyHeaders(_header)
.get('/invoices-gra-1234/file.docx')
.reply(200, () => {
return fileTxt;
});
storage.downloadFile('invoices', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should download a file with options', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
};
const _options = {
headers: {
"x-amz-server-side-encryption-customer-key-MD5": "SSECustomerKeyMD5",
"x-amz-checksum-mode": "ChecksumMode"
},
queries: {
test : "2",
partNumber : "PartNumber"
}
};
const nockRequest = nock(url1S3, {
reqheaders: {
'x-amz-server-side-encryption-customer-key-MD5': () => true,
'x-amz-checksum-mode': () => true
}
})
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.query(_options.queries)
.reply(200, () => {
return fileTxt;
});
storage.downloadFile('bucket', 'file.docx', _options, function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should return code 404 if the file does not exist', function(done) {
const _header = {'content-type': 'application/xml'};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(404, "<?xml version='1.0' encoding='UTF-8'?><Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><RequestId>txc03d49a36c324653854de-006408d963</RequestId><Key>template222.odt</Key></Error>");
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
error: {
code: 'NoSuchKey',
message: 'The specified key does not exist.',
requestid: 'txc03d49a36c324653854de-006408d963',
key: 'template222.odt'
}
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('[STREAM] should return code 404 if the file does not exist and should convert the XML as JSON', function(done) {
const _header = {'content-type': 'application/xml'};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(404, "<?xml version='1.0' encoding='UTF-8'?><Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><RequestId>txc03d49a36c324653854de-006408d963</RequestId><Key>template222.odt</Key></Error>");
storage.downloadFile('bucket', 'file.docx', { output: outputStreamFunction }, function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(JSON.stringify(storage.xmlToJson(dataStream)), JSON.stringify({
error: {
code: 'NoSuchKey',
message: 'The specified key does not exist.',
requestid: 'txc03d49a36c324653854de-006408d963',
key: 'template222.odt'
}
}));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
done();
});
});
it("should return an error if the bucket does not exist", function (done) {
const _header = {
'content-type': 'application/xml',
'x-amz-id-2': 'txfa644d038be848a9938e3-00641850f0',
'x-amz-request-id': 'txfa644d038be848a9938e3-00641850f0',
'x-trans-id': 'txfa644d038be848a9938e3-00641850f0',
'x-openstack-request-id': 'txfa644d038be848a9938e3-00641850f0',
date: 'Mon, 20 Mar 2023 12:26:24 GMT',
'transfer-encoding': 'chunked',
connection: 'close'
};
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/buckeeeet/file.docx')
.reply(404, () => {
return "<?xml version='1.0' encoding='UTF-8'?><Error><Code>NoSuchBucket</Code><Message>The specified bucket does not exist.</Message><RequestId>txfa644d038be848a9938e3-00641850f0</RequestId><BucketName>buckeeeet</BucketName></Error>";
});
storage.downloadFile('buckeeeet', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
error: {
code: 'NoSuchBucket',
message: 'The specified bucket does not exist.',
requestid: 'txfa644d038be848a9938e3-00641850f0',
bucketname: 'buckeeeet'
}
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
});
describe("SWITCH TO CHILD STORAGE", function () {
it('should download a file from the second storage if the main storage returns a 500 error', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should download a file from the second storage if the main storage returns a 500 error and the container is an ALIAS', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/invoices-gra-1234/file.docx')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/invoices-de-8888/file.docx')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.downloadFile('invoices', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should download a file from the second storage if the main storage returns a 500 error, then should RECONNECT to the main storage', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(200);
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should download a file from the second storage if the authentication on the main storage is not allowed', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(401, 'Unauthorized');
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(401, 'Unauthorized');
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should download a file as Stream from the second storage if the authentication on the main storage is not allowed', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(401, 'Unauthorized');
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(401, 'Unauthorized');
storage.downloadFile('bucket', 'file.docx', { output: outputStreamFunction }, function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(dataStream, fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should download a file from the second storage if the main storage timeout', function(done) {
storage.setTimeout(200);
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.delayConnection(500)
.reply(200, {});
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.delayConnection(500)
.reply(200, {});
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should download a file from the second storage if the main storage returns any kind of error', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.replyWithError('Error Message 1234');
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.replyWithError('Error Message 1234');
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should return an error if all storage are not available, and reset the active storage to the main', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(500, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err.toString(), 'Error: All S3 storages are not available');
assert.strictEqual(resp, undefined);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
it('should return an error and should not create an infinite loop of retry', function(done) {
storage.setTimeout(200);
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.replyWithError(new Error('ERR_STREAM_PREMATURE_CLOSE'))
.get('/')
.reply(200)
.get('/bucket/file.docx')
.replyWithError(new Error('TIMEOUT'));
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.delayConnection(400)
.replyWithError(new Error('TIMEOUT'));
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err.toString(), 'Error: All S3 storages are not available');
assert.strictEqual(resp, undefined);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
done();
});
});
});
describe("PARALLEL REQUESTS", function () {
function getDownloadFilePromise() {
return new Promise((resolve, reject) => {
try {
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
if (err) {
return reject(err);
}
return resolve(resp);
});
} catch(err) {
return reject(err);
}
});
}
it('should fallback to a child if the main storage return any kind of errors, then should reconnect to the main storage automatically', function (done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.odt')
.reply(500)
.get('/bucket/file.odt')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
})
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(200);
const nockRequestS4 = nock(url1S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
});
const promise1 = getDownloadFilePromise();
const promise2 = getDownloadFilePromise();
Promise.all([promise1, promise2]).then(results => {
assert.strictEqual(results.length, 2);
assert.strictEqual(results[0].body.toString(), fileTxt);
assert.strictEqual(results[0].statusCode, 200);
assert.strictEqual(results[1].body.toString(), fileTxt);
assert.strictEqual(results[1].statusCode, 200);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
/** Last batch requesting the main storage, everything is ok */
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(nockRequestS4.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
done();
});
}).catch(err => {
assert.strictEqual(err, null);
done();
});
});
it('should fallback to a child if the main storage return any kind of errors, then should reconnect to the main storage after multiple try', function (done) {
/** First Batch */
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.odt')
.reply(500)
.get('/bucket/file.odt')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
})
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
/** Second Batch */
const nockRequestS4 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
})
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
});
const nockRequestS5 = nock(url1S3)
.get('/')
.reply(500);
/** Third Batch */
const nockRequestS6 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
});
const nockRequestS7 = nock(url1S3)
.get('/')
.reply(200);
/** Fourth Batch */
const nockRequestS8 = nock(url1S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt;
});
/** First batch of requests > S3 main return error > Child storage response ok */
const promise1 = getDownloadFilePromise();
const promise2 = getDownloadFilePromise();
Promise.all([promise1, promise2]).then(function (results) {
assert.strictEqual(results.length, 2);
assert.strictEqual(results[0].body.toString(), fileTxt);
assert.strictEqual(results[0].statusCode, 200);
assert.strictEqual(results[1].body.toString(), fileTxt);
assert.strictEqual(results[1].statusCode, 200);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
/** Second batch of requests > Still requesting the child storage, the main storage is still not available */
const promise3 = getDownloadFilePromise();
const promise4 = getDownloadFilePromise();
Promise.all([promise3, promise4]).then(function (results) {
assert.strictEqual(results.length, 2);
assert.strictEqual(results[0].body.toString(), fileTxt);
assert.strictEqual(results[0].statusCode, 200);
assert.strictEqual(results[1].body.toString(), fileTxt);
assert.strictEqual(results[1].statusCode, 200);
assert.strictEqual(nockRequestS4.pendingMocks().length, 0);
assert.strictEqual(nockRequestS5.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
/** Third batch of requests > Still requesting the child storage, the main storage is now Available! Active storage is reset to the main storage */
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(nockRequestS6.pendingMocks().length, 0);
assert.strictEqual(nockRequestS7.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
/** Fourth batch requesting the main storage, everything is ok */
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(resp.statusCode, 200);
assert.str