@ikigai-gfc/tiny-storage-client
Version:
Tiny node client to request distributed AWS S3 or the OpenStack Swift Object Storage.
1,168 lines (1,065 loc) • 105 kB
JavaScript
const s3 = require('../s3.js');
const assert = require('assert');
const nock = require('nock');
const fs = require('fs');
const path = require('path');
let storage = {};
const url1S3 = 'https://s3.gra.first.cloud.test';
const url2S3 = 'https://s3.de.first.cloud.test';
/** ASSETS for download/upload */
const fileTxtPath = path.join(__dirname, 'assets', 'file.txt');
const fileTxt = fs.readFileSync(fileTxtPath).toString();
const fileXmlPath = path.join(__dirname, 'assets', 'files.xml');
const fileXml = fs.readFileSync(fileXmlPath).toString();
/** ASSETS for List objects Requests */
const _listObjectsResponseXML = fs.readFileSync(path.join(__dirname, "./assets", 'listObjects.response.xml'));
const _listObjectsResponseJSON = require('./assets/listObjects.response.json');
describe('S3 SDK', function () {
beforeEach(function() {
storage = s3([{
accessKeyId : '2371bebbe7ac4b2db39c09eadf011661',
secretAccessKey: '9978f6abf7f445566a2d316424aeef2',
url : url1S3.replace('https://', ''),
region : 'gra',
buckets : {
invoices : "invoices-gra-1234"
}
},
{
accessKeyId : '2371bebbe7ac4b2db39c09eadf011661',
secretAccessKey: '9978f6abf7f445566a2d316424aeef2',
url : url2S3.replace('https://', ''),
region : 'de',
buckets : {
invoices : "invoices-de-8888"
}
}]);
})
describe('constructor/getConfig/setConfig/setTimeout', function () {
it("should create a new s3 instance if the authentication is provided as Object", function () {
const _authS3 = { accessKeyId: '-', secretAccessKey: '-', region: '-', url: '-' }
const _storage = s3(_authS3);
const _config = _storage.getConfig()
assert.strictEqual(_config.timeout, 5000);
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(_config.storages.length, 1);
assert.strictEqual(JSON.stringify(_config.storages[0]), JSON.stringify(_authS3))
})
it("should create a new s3 instance if the authentication is provided as List of objects", function () {
const _authS3 = [{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, region: 7, url: 8 }, { accessKeyId: 9, secretAccessKey: 10, region: 11, url: 12 }]
const _storage = s3(_authS3);
const _config = _storage.getConfig()
assert.strictEqual(_config.timeout, 5000);
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(_config.storages.length, 3);
assert.strictEqual(JSON.stringify(_config.storages), JSON.stringify(_authS3))
})
it("should throw an error if authentication values are missing", function() {
assert.throws(function(){ s3({}) }, Error);
/** As object */
assert.throws(function(){ s3({accessKeyId: '', secretAccessKey: '', url: ''}) }, Error); // missing region
assert.throws(function(){ s3({accessKeyId: '', secretAccessKey: '', region: ''}) }, Error); // missing url
assert.throws(function(){ s3({accessKeyId: '', url: '', region: ''}) }, Error); // missing secretAccessKey
assert.throws(function(){ s3({secretAccessKey: '', url: '', region: ''}) }, Error); // missing accessKeyId
/** As array */
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, url: 8 }]) }, Error); // missing region
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, region: 8 }]) }, Error); // missing url
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, region: 6, url: 8 }]) }, Error); // missing secretAccessKey
assert.throws(function(){ s3([{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { secretAccessKey: 5, region: 6, url: 8 }]) }, Error); // missing accessKeyId
});
it("should set a new config", function () {
const _storage = s3({ accessKeyId: '-', secretAccessKey: '-', region: '-', url: '-' });
const _authS3 = [{ accessKeyId: 1, secretAccessKey: 2, region: 3, url: 4 }, { accessKeyId: 5, secretAccessKey: 6, region: 7, url: 8 }]
_storage.setConfig(_authS3)
const _config = _storage.getConfig()
assert.strictEqual(_config.timeout, 5000);
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(_config.storages.length, 2);
assert.strictEqual(JSON.stringify(_config.storages), JSON.stringify(_authS3))
})
it("should set a new timeout value", function() {
const _storage = s3({ accessKeyId: '-', secretAccessKey: '-', region: '-', url: '-' });
assert.strictEqual(_storage.getConfig().timeout, 5000);
_storage.setTimeout(10000);
assert.strictEqual(_storage.getConfig().timeout, 10000);
});
})
describe('headBucket', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should return code 200, and request signed with AWS4', function (done) {
const nockRequest = nock(url1S3,
{
reqheaders: {
'x-amz-content-sha256': () => true,
'x-amz-date': () => true,
'authorization': () => true,
'host': () => true
}
}).intercept("/customBucket", "HEAD").reply(200, '');
storage.headBucket('customBucket', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should return code 200 and request a bucket as ALIAS', function (done) {
const nockRequest = nock(url1S3,
{
reqheaders: {
'x-amz-content-sha256': () => true,
'x-amz-date': () => true,
'authorization': () => true,
'host': () => true
}
}).intercept("/invoices-gra-1234", "HEAD").reply(200, '');
storage.headBucket('invoices', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
it('should return code 403 Forbidden', function (done) {
const nockRequest = nock(url1S3).intercept("/customBucket", "HEAD").reply(403, '');
storage.headBucket('customBucket', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 403);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
});
});
});
describe("SWITCH TO CHILD STORAGE", function () {
it('should switch to the child storage and return code 200 with bucket as ALIAS', function (done) {
const nockRequestS1 = nock(url1S3).intercept("/invoices-gra-1234", "HEAD").reply(500, '');
const nockRequestS2 = nock(url2S3).intercept("/invoices-de-8888", "HEAD").reply(200, '');
const nockRequestS3 = nock(url1S3).get('/').reply(500);
storage.headBucket('invoices', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
});
});
})
});
describe('listBuckets', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should fetch a list of buckets', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '366',
'x-amz-id-2': 'tx606add09487142fa88e67-00641aacf4',
'x-amz-request-id': 'tx606add09487142fa88e67-00641aacf4',
'x-trans-id': 'tx606add09487142fa88e67-00641aacf4',
'x-openstack-request-id': 'tx606add09487142fa88e67-00641aacf4',
date: 'Wed, 22 Mar 2023 07:23:32 GMT',
connection: 'close'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/')
.reply(200, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>89123456:user-feiowjfOEIJW</ID><DisplayName>12345678:user-feiowjfOEIJW</DisplayName></Owner><Buckets><Bucket><Name>invoices</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket><Bucket><Name>www</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
});
storage.listBuckets((err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
"bucket": [
{ "name": "invoices", "creationdate": "2023-02-27T11:46:24.000Z" },
{ "name": "www", "creationdate": "2023-02-27T11:46:24.000Z" }
]
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it('should return an error if credentials are not correct', function (done) {
const _header = {
'x-amz-request-id': 'BEFVPYB9PM889VMS',
'x-amz-id-2': 'Pnby9XcoK7X/GBpwr+vVV/X3XyadxsUkTzGdSJS5zRMhs2RvZDGroWleytOYGmYRSszFbsaZWUo=',
'content-type': 'application/xml',
'transfer-encoding': 'chunked',
date: 'Wed, 22 Mar 2023 07:37:22 GMT',
server: 'AmazonS3',
connection: 'close'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/')
.reply(403, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>InvalidAccessKeyId</Code><Message>The AWS Access Key Id you provided does not exist in our records.</Message><AWSAccessKeyId>AKIAUO7WHYLVFADDFL57e</AWSAccessKeyId><RequestId>BSTT951V1FREKS2X</RequestId><HostId>zWFC8ZOiZvyxTUgcYjHDD9rmPDG81TCJHkZhAv4zgguuR5I9aeqSFA9Ns4r5PdKy9+9o+xDLpOk=</HostId></Error>";
});
storage.listBuckets((err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 403);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
error: {
code: 'InvalidAccessKeyId',
message: 'The AWS Access Key Id you provided does not exist in our records.',
awsaccesskeyid: 'AKIAUO7WHYLVFADDFL57e',
requestid: 'BSTT951V1FREKS2X',
hostid: 'zWFC8ZOiZvyxTUgcYjHDD9rmPDG81TCJHkZhAv4zgguuR5I9aeqSFA9Ns4r5PdKy9+9o+xDLpOk='
}
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
});
describe("SWITCH TO CHILD STORAGE", function () {
it('should fetch a list of buckets', function (done) {
const nockRequestS1 = nock(url1S3)
.get('/')
.reply(500, '');
const nockRequestS2 = nock(url2S3)
.get('/')
.reply(200, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>89123456:user-feiowjfOEIJW</ID><DisplayName>12345678:user-feiowjfOEIJW</DisplayName></Owner><Buckets><Bucket><Name>invoices</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket><Bucket><Name>www</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.listBuckets((err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
"bucket": [
{ "name": "invoices", "creationdate": "2023-02-27T11:46:24.000Z" },
{ "name": "www", "creationdate": "2023-02-27T11:46:24.000Z" }
]
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}))
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
});
describe("Options 'requestStorageIndex'", function() {
it("should request the first storage and should return an error if the first storage is not available", function(done) {
const nockRequestS1 = nock(url1S3)
.get('/')
.reply(500, '');
storage.listBuckets({ requestStorageIndex: 0 }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 500);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}))
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
done();
})
})
it("should request the second storage and should return an error if the second storage is not available", function(done) {
const nockRequestS1 = nock(url2S3)
.get('/')
.reply(500, '');
storage.listBuckets({ requestStorageIndex: 1 }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 500);
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}))
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
done();
})
})
it("should request the second storage and get a list of buckets", function(done) {
const nockRequestS1 = nock(url2S3)
.get('/')
.reply(200, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>89123456:user-feiowjfOEIJW</ID><DisplayName>12345678:user-feiowjfOEIJW</DisplayName></Owner><Buckets><Bucket><Name>invoices</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket><Bucket><Name>www</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
});
storage.listBuckets({ requestStorageIndex: 1 }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
"bucket": [
{ "name": "invoices", "creationdate": "2023-02-27T11:46:24.000Z" },
{ "name": "www", "creationdate": "2023-02-27T11:46:24.000Z" }
]
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}))
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
done();
})
})
})
});
describe('listFiles', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should fetch a list of objects', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '1887',
'x-amz-id-2': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-amz-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-trans-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-openstack-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
date: 'Mon, 20 Mar 2023 07:14:31 GMT',
connection: 'close'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket')
.query({ 'list-type' : 2 })
.reply(200, () => {
return _listObjectsResponseXML;
});
storage.listFiles('bucket', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it('should fetch a list of objects from a bucket as ALIAS', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '1887',
'x-amz-id-2': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-amz-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-trans-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-openstack-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
date: 'Mon, 20 Mar 2023 07:14:31 GMT',
connection: 'close'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/invoices-gra-1234')
.query({ 'list-type' : 2 })
.reply(200, () => {
return _listObjectsResponseXML;
});
storage.listFiles('invoices', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it('should fetch a list of objects with query parameters (prefix & limit)', function (done) {
const _header = {
'content-type': 'application/xml',
'content-length': '1887',
'x-amz-id-2': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-amz-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-trans-id': 'txf0b438dfd25b444ba3f60-00641807d7',
'x-openstack-request-id': 'txf0b438dfd25b444ba3f60-00641807d7',
date: 'Mon, 20 Mar 2023 07:14:31 GMT',
connection: 'close'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket')
.query({
"list-type" : 2,
"prefix" : "document",
"max-keys" : 2
})
.reply(200, () => {
return _listObjectsResponseXML;
});
storage.listFiles('bucket', { queries: { "prefix": "document", "max-keys": 2 } }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it("should return an error if the bucket does not exist", function (done) {
const _headers = {
'content-type': 'application/xml',
'x-amz-id-2': 'tx8fa5f00b19af4756b9ef3-0064184d77',
'x-amz-request-id': 'tx8fa5f00b19af4756b9ef3-0064184d77',
'x-trans-id': 'tx8fa5f00b19af4756b9ef3-0064184d77',
'x-openstack-request-id': 'tx8fa5f00b19af4756b9ef3-0064184d77',
date: 'Mon, 20 Mar 2023 12:11:35 GMT',
'transfer-encoding': 'chunked',
connection: 'close'
}
const _expectedBody = {
error: {
code: 'NoSuchBucket',
message: 'The specified bucket does not exist.',
requestid: 'txe285e692106542e88a2f5-0064184e80',
bucketname: 'buckeeeet'
}
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_headers)
.get('/buckeeeet')
.query({
"list-type" : 2
})
.reply(404, () => {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>NoSuchBucket</Code><Message>The specified bucket does not exist.</Message><RequestId>txe285e692106542e88a2f5-0064184e80</RequestId><BucketName>buckeeeet</BucketName></Error>";
});
storage.listFiles('buckeeeet', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_expectedBody));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
})
describe("SWITCH TO CHILD STORAGE", function () {
it('should fetch a list of objects', function (done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket')
.query({ 'list-type' : 2 })
.reply(500, '');
const nockRequestS2 = nock(url2S3)
.get('/bucket')
.query({ 'list-type' : 2 })
.reply(200, () => {
return _listObjectsResponseXML;
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.listFiles('bucket', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify(_listObjectsResponseJSON));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify({}))
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
});
});
describe('downloadFile', function() {
describe("REQUEST MAIN STORAGE", function () {
it('should download a file', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it('should download a file as STREAM', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt;
});
storage.downloadFile('bucket', 'file.docx', { stream: true }, function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
let data = '';
resp.on('data', chunk => data += chunk);
resp.on('end', function () {
assert.strictEqual(data,fileTxt)
assert.strictEqual(nockRequest.pendingMocks().length, 0);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
done();
});
})
})
it('should download a file from an alias', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
}
const nockRequest = nock(url1S3, {
reqheaders: {
'x-amz-content-sha256': () => true,
'x-amz-date': () => true,
'authorization': () => true,
'host': () => true
}
})
.defaultReplyHeaders(_header)
.get('/invoices-gra-1234/file.docx')
.reply(200, (uri, body) => {
console.log(uri);
return fileTxt;
});
storage.downloadFile('invoices', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it('should download a file with options', function(done) {
const _header = {
'content-length': '1492',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
}
const _options = {
headers: {
"x-amz-server-side-encryption-customer-key-MD5": "SSECustomerKeyMD5",
"x-amz-checksum-mode": "ChecksumMode"
},
queries: {
test : "2",
partNumber : "PartNumber"
}
}
const nockRequest = nock(url1S3, {
reqheaders: {
'x-amz-server-side-encryption-customer-key-MD5': () => true,
'x-amz-checksum-mode': () => true
}
})
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.query(_options.queries)
.reply(200, () => {
return fileTxt;
});
storage.downloadFile('bucket', 'file.docx', _options, function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it('should return code 404 if the file does not exist', function(done) {
const _header = {'content-type': 'application/xml'}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(404, "<?xml version='1.0' encoding='UTF-8'?><Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><RequestId>txc03d49a36c324653854de-006408d963</RequestId><Key>template222.odt</Key></Error>");
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
error: {
code: 'NoSuchKey',
message: 'The specified key does not exist.',
requestid: 'txc03d49a36c324653854de-006408d963',
key: 'template222.odt'
}
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
it('[STREAM] should return code 404 if the file does not exist and should convert the XML as JSON', function(done) {
const _header = {'content-type': 'application/xml'}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/bucket/file.docx')
.reply(404, "<?xml version='1.0' encoding='UTF-8'?><Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><RequestId>txc03d49a36c324653854de-006408d963</RequestId><Key>template222.odt</Key></Error>");
storage.downloadFile('bucket', 'file.docx', { stream: true }, function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
let data = '';
resp.on('data', chunk => data += chunk);
resp.on('end', function () {
assert.strictEqual(JSON.stringify(storage.xmlToJson(data)), JSON.stringify({
error: {
code: 'NoSuchKey',
message: 'The specified key does not exist.',
requestid: 'txc03d49a36c324653854de-006408d963',
key: 'template222.odt'
}
}))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
done();
});
})
})
it("should return an error if the bucket does not exist", function (done) {
const _header = {
'content-type': 'application/xml',
'x-amz-id-2': 'txfa644d038be848a9938e3-00641850f0',
'x-amz-request-id': 'txfa644d038be848a9938e3-00641850f0',
'x-trans-id': 'txfa644d038be848a9938e3-00641850f0',
'x-openstack-request-id': 'txfa644d038be848a9938e3-00641850f0',
date: 'Mon, 20 Mar 2023 12:26:24 GMT',
'transfer-encoding': 'chunked',
connection: 'close'
}
const nockRequest = nock(url1S3)
.defaultReplyHeaders(_header)
.get('/buckeeeet/file.docx')
.reply(404, () => {
return "<?xml version='1.0' encoding='UTF-8'?><Error><Code>NoSuchBucket</Code><Message>The specified bucket does not exist.</Message><RequestId>txfa644d038be848a9938e3-00641850f0</RequestId><BucketName>buckeeeet</BucketName></Error>";
});
storage.downloadFile('buckeeeet', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(JSON.stringify(resp.body), JSON.stringify({
error: {
code: 'NoSuchBucket',
message: 'The specified bucket does not exist.',
requestid: 'txfa644d038be848a9938e3-00641850f0',
bucketname: 'buckeeeet'
}
}));
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header))
assert.strictEqual(nockRequest.pendingMocks().length, 0);
done();
})
})
});
describe("SWITCH TO CHILD STORAGE", function () {
it('should download a file from the second storage if the main storage returns a 500 error', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
it('should download a file from the second storage if the main storage returns a 500 error and the container is an ALIAS', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/invoices-gra-1234/file.docx')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/invoices-de-8888/file.docx')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.downloadFile('invoices', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
it('should download a file from the second storage if the main storage returns a 500 error, then should RECONNECT to the main storage', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(500)
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(200);
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
it('should download a file from the second storage if the authentication on the main storage is not allowed', function(done) {
let nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(401, 'Unauthorized')
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(401, 'Unauthorized')
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
it('should download a file from the second storage if the main storage timeout', function(done) {
storage.setTimeout(200);
let nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.delayConnection(500)
.reply(200, {});
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.delayConnection(500)
.reply(200, {});
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
it('should download a file from the second storage if the main storage returns any kind of error', function(done) {
let nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.replyWithError('Error Message 1234');
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.replyWithError('Error Message 1234');
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(resp.body.toString(), fileTxt);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 1);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
it('should return an error if all storage are not available, and reset the active storage to the main', function(done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.docx')
.reply(500)
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.docx')
.reply(500, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
storage.downloadFile('bucket', 'file.docx', function (err, resp) {
assert.strictEqual(err.toString(), 'Error: All S3 storages are not available');
assert.strictEqual(resp, undefined);
const _config = storage.getConfig();
assert.strictEqual(_config.activeStorage, 0);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
done();
})
})
});
describe("PARALLEL REQUESTS", function () {
function getDownloadFilePromise() {
return new Promise((resolve, reject) => {
try {
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
if (err) {
return reject(err);
}
return resolve(resp);
});
} catch(err) {
return reject(err);
}
});
}
it('should fallback to a child if the main storage return any kind of errors, then should reconnect to the main storage automatically', function (done) {
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.odt')
.reply(500)
.get('/bucket/file.odt')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
})
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(200);
const nockRequestS4 = nock(url1S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
})
let promise1 = getDownloadFilePromise()
let promise2 = getDownloadFilePromise()
Promise.all([promise1, promise2]).then(results => {
assert.strictEqual(results.length, 2)
assert.strictEqual(results[0].body.toString(), fileTxt);
assert.strictEqual(results[0].statusCode, 200);
assert.strictEqual(results[1].body.toString(), fileTxt);
assert.strictEqual(results[1].statusCode, 200);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
/** Last batch requesting the main storage, everything is ok */
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(nockRequestS4.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
done();
});
}).catch(err => {
assert.strictEqual(err, null);
done();
});
})
it('should fallback to a child if the main storage return any kind of errors, then should reconnect to the main storage after multiple try', function (done) {
/** First Batch */
const nockRequestS1 = nock(url1S3)
.get('/bucket/file.odt')
.reply(500)
.get('/bucket/file.odt')
.reply(500);
const nockRequestS2 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
})
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
});
const nockRequestS3 = nock(url1S3)
.get('/')
.reply(500);
/** Second Batch */
const nockRequestS4 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
})
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
});
const nockRequestS5 = nock(url1S3)
.get('/')
.reply(500);
/** Third Batch */
const nockRequestS6 = nock(url2S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
})
const nockRequestS7 = nock(url1S3)
.get('/')
.reply(200);
/** Fourth Batch */
const nockRequestS8 = nock(url1S3)
.get('/bucket/file.odt')
.reply(200, () => {
return fileTxt
})
/** First batch of requests > S3 main return error > Child storage response ok */
let promise1 = getDownloadFilePromise()
let promise2 = getDownloadFilePromise()
Promise.all([promise1, promise2]).then(function (results) {
assert.strictEqual(results.length, 2)
assert.strictEqual(results[0].body.toString(), fileTxt);
assert.strictEqual(results[0].statusCode, 200);
assert.strictEqual(results[1].body.toString(), fileTxt);
assert.strictEqual(results[1].statusCode, 200);
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
assert.strictEqual(nockRequestS2.pendingMocks().length, 0);
assert.strictEqual(nockRequestS3.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
/** Second batch of requests > Still requesting the child storage, the main storage is still not available */
let promise3 = getDownloadFilePromise()
let promise4 = getDownloadFilePromise()
Promise.all([promise3, promise4]).then(function (results) {
assert.strictEqual(results.length, 2)
assert.strictEqual(results[0].body.toString(), fileTxt);
assert.strictEqual(results[0].statusCode, 200);
assert.strictEqual(results[1].body.toString(), fileTxt);
assert.strictEqual(results[1].statusCode, 200);
assert.strictEqual(nockRequestS4.pendingMocks().length, 0);
assert.strictEqual(nockRequestS5.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
/** Third batch of requests > Still requesting the child storage, the main storage is now Available! Active storage is reset to the main storage */
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(nockRequestS6.pendingMocks().length, 0);
assert.strictEqual(nockRequestS7.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
/** Fourth batch requesting the main storage, everything is ok */
storage.downloadFile('bucket', 'file.odt', function (err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.body.toString(), fileTxt);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(nockRequestS8.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
done();
});
});
}).catch(function (err) {
assert.strictEqual(err, null);
done();
});
}).catch(function (err) {
assert.strictEqual(err, null);
done();
});
})
});
});
describe('uploadFile', function() {
describe("REQUEST MAIN STORAGE", function () {
const _header = {
'content-length': '0',
'last-modified': 'Wed, 03 Nov 2021 13:02:39 GMT',
date: 'Wed, 03 Nov 2021 14:28:48 GMT',
etag: 'a30776a059eaf26eebf27756a849097d',
'x-amz-request-id': '318BC8BC148832E5',
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2mkiDFu8yU9AS1ed4OpIszj7UDNEHGran'
}
it("should upload a file provided as buffer", function() {
const nockRequestS1 = nock(url1S3)
.defaultReplyHeaders(_header)
.put('/bucket/file.pdf')
.reply(200, (uri, requestBody) => {
assert.strictEqual(requestBody, fileXml);
return '';
});
storage.uploadFile('bucket', 'file.pdf', Buffer.from(fileXml), function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
})
})
it("should upload a file provided as buffer to a bucket alias", function() {
const nockRequestS1 = nock(url1S3)
.defaultReplyHeaders(_header)
.put('/invoices-gra-1234/file.pdf')
.reply(200, (uri, requestBody) => {
assert.strictEqual(requestBody, fileXml);
return '';
});
storage.uploadFile('invoices', 'file.pdf', Buffer.from(fileXml), function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
})
})
it("should upload a file provided as local path", function() {
const nockRequestS1 = nock(url1S3)
.defaultReplyHeaders(_header)
.put('/bucket/file.pdf')
.reply(200, (uri, requestBody) => {
assert.strictEqual(requestBody, fileXml);
return '';
});
storage.uploadFile('bucket', 'file.pdf', fileXmlPath, function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_header));
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(nockRequestS1.pendingMocks().length, 0);
})
})
it("should return an error if the file provided as local path does not exist", function() {
storage.uploadFile('bucket', 'file.pdf', '/var/random/path/file.pdf', function(err, resp) {
assert.strictEqual(err.toString(), "Error: ENOENT: no such file or directory, open '/var/random/path/file.pdf'");
assert.strictEqual(resp, undefined);
})
})
it("should upload a file provided as buffer with OPTIONS (like metadata)", function(done) {
const _options = {
headers: {
"x-amz-meta-name": "carbone",
"x-amz-checksum-sha256": "0ea4be78f6c3948588172edc6d