tiny-storage-client
Version:
Tiny node client to request distributed AWS S3 or the OpenStack Swift Object Storage.
1,170 lines (1,047 loc) • 322 kB
JavaScript
/** Init retry delay for "Rock-req" tests */
const _rock = require('rock-req');
const storageSDK = require('../index.js');
const nock = require('nock');
const assert = require('assert');
const fs = require('fs');
const path = require('path');
var stream = require('stream');
const authURL = 'https://auth.cloud.ovh.net/v3';
const publicUrlGRA = 'https://storage.gra.cloud.ovh.net/v1/AUTH_ce3e510224d740a685cb0ae7bdb8ebc3';
const publicUrlSBG = 'https://storage.sbg.cloud.ovh.net/v1/AUTH_ce3e510224d740a685cb0ae7bdb8ebc3';
const tokenAuth = 'gAAAAABe8JlEGYPUwwOyjqgUBl11gSjDOw5VTtUZ5n8SWxghRGwakDkP_lelLfRctzyhbIFUXjsdPaGmV2xicL-9333lJUnL3M4JYlYCYMWsX3IhnLPYboyti835VdhAHQ7K_d0OC4OYvM04bvL3w_uSbkxPmL27uO0ISUgQdB_mHxoYlol8xYI';
const fileTxt = fs.readFileSync(path.join(__dirname, './assets/file.txt'));
let _dataStreams = ['', '', '', ''];
function resetDataStreams() {
_dataStreams = ['', '', '', ''];
}
function getOutputStreamFunction (index) {
return function (_opt, _res) {
_dataStreams[index] = '';
const outputStream = new stream.Writable();
outputStream._write = function (chunk, _encoding, done) {
_dataStreams[index] += chunk;
done();
};
outputStream.on('error', (err) => {
console.log('Error Stream:', err.toString());
_dataStreams[index] = '';
});
return outputStream;
};
}
let dataStream = '';
const outputStreamFunction = function (_opt, _res) {
dataStream = '';
const outputStream = new stream.Writable();
outputStream._write = function (chunk, _encoding, done) {
dataStream += chunk;
done();
};
outputStream.on('error', (err) => {
console.log('Error Stream:', err.toString());
dataStream = '';
});
return outputStream;
};
describe('Ovh Object Storage Swift', function () {
const storage = storageSDK([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
}]);
beforeEach(function (done) {
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.setTimeout(5000);
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA',
buckets : {
invoices: 'invoices-gra-1234'
}
}]);
storage.connection((err) => {
assert.strictEqual(err, null);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
describe('New instance', function() {
it('should create 2 new instances', function () {
const _swift1 = storageSDK({
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : 'swift.gra.ovh.io',
tenantName : 'storage-1-tenant',
region : 'GRA'
});
const _swift2 = storageSDK({
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : 'swift.sbg.ovh.io',
tenantName : 'storage-2-tenant',
region : 'SBG'
});
assert.strictEqual(_swift1.getConfig().storages.length, 1);
assert.strictEqual(_swift1.getConfig().storages[0].username, 'storage-1-user');
assert.strictEqual(_swift1.getConfig().storages[0].password, 'storage-1-password');
assert.strictEqual(_swift1.getConfig().storages[0].tenantName, 'storage-1-tenant');
assert.strictEqual(_swift1.getConfig().storages[0].authUrl, 'swift.gra.ovh.io');
assert.strictEqual(_swift1.getConfig().storages[0].region, 'GRA');
assert.strictEqual(_swift2.getConfig().storages.length, 1);
assert.strictEqual(_swift2.getConfig().storages[0].username, 'storage-2-user');
assert.strictEqual(_swift2.getConfig().storages[0].password, 'storage-2-password');
assert.strictEqual(_swift2.getConfig().storages[0].tenantName, 'storage-2-tenant');
assert.strictEqual(_swift2.getConfig().storages[0].authUrl, 'swift.sbg.ovh.io');
assert.strictEqual(_swift2.getConfig().storages[0].region, 'SBG');
});
it('should not throw an error without tenantName', function (done) {
const _swift1 = storageSDK({
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : 'swift.gra.ovh.io',
region : 'GRA'
});
assert.strictEqual(_swift1.getConfig().storages.length, 1);
assert.strictEqual(_swift1.getConfig().storages[0].username, 'storage-1-user');
assert.strictEqual(_swift1.getConfig().storages[0].password, 'storage-1-password');
assert.strictEqual(_swift1.getConfig().storages[0].authUrl, 'swift.gra.ovh.io');
assert.strictEqual(_swift1.getConfig().storages[0].region, 'GRA');
assert.strictEqual(_swift1.getConfig().storages[0]?.tenantName, undefined);
done();
});
});
describe('Connection', function () {
it('should connect to object storage swift (without tenantName and the region as lowercase)', function(done) {
const storageTest = storageSDK([{
username : 'storage-X-user',
password : 'storage-X-password',
authUrl : authURL,
region : 'gra'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, function (_uri, body) {
assert.strictEqual(!!body?.auth?.scope, false);
assert.strictEqual(JSON.stringify(body), '{"auth":{"identity":{"methods":["password"],"password":{"user":{"name":"storage-X-user","domain":{"id":"default"},"password":"storage-X-password"}}}}}');
return connectionResultSuccessV3;
}, { "X-Subject-Token": tokenAuth });
storageTest.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storageTest.getConfig().token, tokenAuth);
assert.deepStrictEqual(storageTest.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[20].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to object storage swift with a tenantName', function (done) {
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, function (_uri, body) {
assert.strictEqual(!!body?.auth?.scope, true);
assert.strictEqual(JSON.stringify(body), '{"auth":{"identity":{"methods":["password"],"password":{"user":{"name":"storage-1-user","domain":{"id":"default"},"password":"storage-1-password"}}},"scope":{"project":{"domain":{"id":"default"},"name":"storage-1-tenant"}}}}');
return connectionResultSuccessV3;
}, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[20].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to object storage swift with the interface (ADMIN)', function(done) {
/** ADMIN INTERFACE */
const storageTest = storageSDK([{
username : 'storage-X-user',
password : 'storage-X-password',
authUrl : authURL,
region : 'DE',
interface : 'ADMIN'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, function (_uri, body) {
assert.strictEqual(!!body?.auth?.scope, false);
assert.strictEqual(JSON.stringify(body), '{"auth":{"identity":{"methods":["password"],"password":{"user":{"name":"storage-X-user","domain":{"id":"default"},"password":"storage-X-password"}}}}}');
return connectionResultSuccessV3;
}, { "X-Subject-Token": tokenAuth });
storageTest.connection((err) => {
assert.strictEqual(err, null);
console.log(storageTest.getConfig());
assert.deepStrictEqual(storageTest.getConfig().token, tokenAuth);
assert.deepStrictEqual(storageTest.getConfig().endpoints.region, 'DE');
assert.deepStrictEqual(storageTest.getConfig().endpoints.interface, 'admin');
assert.deepStrictEqual(storageTest.getConfig().endpoints.url.includes("_admin"), true);
assert.deepStrictEqual(storageTest.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[3].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to object storage swift with the interface (PUBLIC)', function(done) {
/** PUBLIC INTERFACE */
const storageTest = storageSDK([{
username : 'storage-X-user',
password : 'storage-X-password',
authUrl : authURL,
region : 'DE',
interface : 'PUBLIC'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, function (_uri, body) {
assert.strictEqual(!!body?.auth?.scope, false);
assert.strictEqual(JSON.stringify(body), '{"auth":{"identity":{"methods":["password"],"password":{"user":{"name":"storage-X-user","domain":{"id":"default"},"password":"storage-X-password"}}}}}');
return connectionResultSuccessV3;
}, { "X-Subject-Token": tokenAuth });
storageTest.connection((err) => {
assert.strictEqual(err, null);
console.log(storageTest.getConfig());
assert.deepStrictEqual(storageTest.getConfig().token, tokenAuth);
assert.deepStrictEqual(storageTest.getConfig().endpoints.region, 'DE');
assert.deepStrictEqual(storageTest.getConfig().endpoints.interface, 'public');
assert.deepStrictEqual(storageTest.getConfig().endpoints.url.includes("_public"), true);
assert.deepStrictEqual(storageTest.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[10].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return an error if status code is not a 200', function (done) {
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(300, {});
storage.connection((err) => {
assert.notStrictEqual(err, null);
assert.strictEqual(err.message, 'Object Storages are not available');
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return an error if endpoints cannot be found', function (done) {
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3WithoutObjectStore);
storage.connection((err) => {
assert.notStrictEqual(err, null);
assert.strictEqual(err.message, 'Object Storages are not available');
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return an error if endpoints cannot be found because region is invalid', function (done) {
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3WithoutRegion);
storage.connection((err) => {
assert.notStrictEqual(err, null);
assert.strictEqual(err.message, 'Object Storages are not available');
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to the second object storage if the first one is not available (error 500)', function (done) {
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
tenantName : 'storage-2-tenant',
region : 'SBG'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(500, {})
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[4].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to the second object storage if any request is return', function (done) {
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
tenantName : 'storage-2-tenant',
region : 'SBG'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.replyWithError("This is an error")
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[4].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to the second object storage if the main storage timeout', function (done) {
storage.setTimeout(200);
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
tenantName : 'storage-2-tenant',
region : 'SBG'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.delayConnection(500)
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth })
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[4].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to the second object storage if endpoints of the first object storage cannot be found', function (done) {
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
tenantName : 'storage-2-tenant',
region : 'SBG'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3WithoutObjectStore)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[4].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should connect to the second object storage if endpoints of the first object storage cannot be found because region is invalid', function (done) {
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
tenantName : 'storage-2-tenant',
region : 'SBG'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3WithoutRegion)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().activeStorage, 1);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[4].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should reconnect to the first object storage if the first and second one are not available (error 500) but the first storage revive', function (done) {
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
tenantName : 'storage-2-tenant',
region : 'SBG'
}]);
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(500, {})
.post('/auth/tokens')
.reply(500, {})
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.notStrictEqual(err, null);
assert.strictEqual(err.message, 'Object Storages are not available');
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[20].url);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
});
});
describe('deleteFiles', function() {
it('should deletes files', function(done){
const _filesToDelete = [{ 'filenameCustom1234': 'contract-2024.pdf' }, '1685696359848.jpg', { key: 'invoice.docx' }, { name: 'test file |1234.odt' }];
const _headers = {
'content-type': 'application/json',
'x-trans-id': 'tx34d586803a5e4acbb9ac5-0064c7dfbc',
'x-openstack-request-id': 'tx34d586803a5e4acbb9ac5-0064c7dfbc',
date: 'Mon, 31 Jul 2023 16:22:21 GMT',
'transfer-encoding': 'chunked',
'x-iplb-request-id': '53C629C3:E4CA_5762BBC9:01BB_64C7DFBC_B48B74E:1342B',
'x-iplb-instance': '42087'
};
const _returnedBody = '{"Response Status":"200 OK","Response Body":"","Number Deleted":4,"Number Not Found":0,"Errors":[]}';
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.post(/\/.*bulk-delete.*/g)
.reply(200, (_url, body) => {
assert.strictEqual(body.includes('container1/contract-2024.pdf'), true);
assert.strictEqual(body.includes('container1/test%20file%20%7C1234.odt'), true);
assert.strictEqual(body.includes('container1/invoice.docx'), true);
assert.strictEqual(body.includes('container1/1685696359848.jpg'), true);
return _returnedBody;
});
storage.deleteFiles('container1', _filesToDelete, {"fileNameKey": 'filenameCustom1234'}, function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(JSON.stringify(resp.body), _returnedBody);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should deletes files (ALIAS)', function(done){
const _filesToDelete = [ { name: '1685696359848.jpg' }, { name: 'invoice.docx' }, { name: 'test file |1234.odt' }];
const _headers = { 'content-type': 'application/json' };
const _returnedBody = '{"Response Status":"200 OK","Response Body":"","Number Deleted":3,"Number Not Found":0,"Errors":[]}';
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.post(/\/.*bulk-delete.*/g)
.reply(200, (_url, body) => {
assert.strictEqual(body.includes('invoices-gra-1234/test%20file%20%7C1234.odt'), true);
assert.strictEqual(body.includes('invoices-gra-1234/invoice.docx'), true);
assert.strictEqual(body.includes('invoices-gra-1234/1685696359848.jpg'), true);
return _returnedBody;
});
storage.deleteFiles('invoices', _filesToDelete, function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(JSON.stringify(resp.body), _returnedBody);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return the raw result as XML (Buffer)', function(done) {
const _returnedBodyXML = '<delete><number_deleted>0</number_deleted><number_not_found>3</number_not_found><response_body></response_body><response_status>200 OK</response_status><errors></errors></delete>';
const _filesToDelete = [ { name: '1685696359848.jpg' }, { name: 'invoice.docx' }, { name: 'test file |1234.odt' }];
const _headers = {
'content-type': 'application/xml',
'x-trans-id': 'tx76823c2b380f47bab5908-0064c7e60d',
'x-openstack-request-id': 'tx76823c2b380f47bab5908-0064c7e60d',
date: 'Mon, 31 Jul 2023 16:49:18 GMT',
'transfer-encoding': 'chunked',
'x-iplb-request-id': '53C629C3:E61B_3626E64B:01BB_64C7E60D_151EBF17:1C316',
'x-iplb-instance': '33618'
};
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.post(/\/.*bulk-delete.*/g)
.reply(200, (_url, body) => {
assert.strictEqual(body.includes('container1/test%20file%20%7C1234.odt'), true);
assert.strictEqual(body.includes('container1/invoice.docx'), true);
assert.strictEqual(body.includes('container1/1685696359848.jpg'), true);
return _returnedBodyXML;
});
storage.deleteFiles('container1', _filesToDelete, { headers: { 'Accept': "application/xml" } }, function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(resp.body.toString(), _returnedBodyXML);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
describe("MUTLIPLE STORAGES", function () {
beforeEach(function (done) {
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.setTimeout(5000);
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
region : 'GRA',
buckets : {
invoices: 'invoices-ovh-gra'
}
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
region : 'SBG',
buckets : {
invoices: 'invoices-aws-paris'
}
}]);
storage.connection((err) => {
assert.strictEqual(err, null);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should reconnect automatically to the second object storage if the first storage authentication fail and should retry the request (alias)', function (done) {
const _filesToDelete = [ { name: '1685696359848.jpg' }, { name: 'invoice.docx' }, { name: 'test file |1234.odt' }];
const _headers = { 'content-type': 'application/json' };
const _returnedBody = '{"Response Status":"200 OK","Response Body":"","Number Deleted":3,"Number Not Found":0,"Errors":[]}';
const firstNock = nock(publicUrlGRA)
/** 1 */
.post(/\/.*bulk-delete.*/g)
.reply(401);
const secondNock = nock(authURL)
/** 2 */
.post('/auth/tokens')
.reply(500, {})
/** 3 */
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
const thirdNock = nock(publicUrlSBG)
.defaultReplyHeaders(_headers)
/** 4 */
.post(/\/.*bulk-delete.*/g)
.reply(200, (_url, body) => {
assert.strictEqual(body.includes('invoices-aws-paris/test%20file%20%7C1234.odt'), true);
assert.strictEqual(body.includes('invoices-aws-paris/invoice.docx'), true);
assert.strictEqual(body.includes('invoices-aws-paris/1685696359848.jpg'), true);
return _returnedBody;
});
storage.deleteFiles('invoices', _filesToDelete, function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(JSON.stringify(resp.body), _returnedBody);
assert.strictEqual(firstNock.pendingMocks().length, 0);
assert.strictEqual(secondNock.pendingMocks().length, 0);
assert.strictEqual(thirdNock.pendingMocks().length, 0);
done();
});
});
});
});
describe('headBucket', function() {
it('should return metadatas about a container', function(done){
const _statusCode = 204;
const _headers = {
'content-type': 'application/json; charset=utf-8',
'x-container-object-count': '4',
'x-container-bytes-used': '431631',
'x-timestamp': '1641995016.74740',
'x-container-sync-key': 'ZAIrQQjDzWM+APD5Fz6stQ/pBUe+Nck2UnPPswJw1cU=',
'x-container-sync-to': '//OVH_PUBLIC_CLOUD/GRA/AUTH_XXXX/container1',
'last-modified': 'Wed, 12 Jan 2022 13:58:51 GMT',
'accept-ranges': 'bytes',
'x-storage-policy': 'PCS',
vary: 'Accept',
'x-trans-id': 'tx3bd64d99f87147f18ffd9-0064c7d400',
'x-openstack-request-id': 'tx3bd64d99f87147f18ffd9-0064c7d400',
date: 'Mon, 31 Jul 2023 15:32:17 GMT',
'transfer-encoding': 'chunked',
'x-iplb-request-id': '53C629C3:E1FB_3626E64B:01BB_64C7D3FF_150CAC23:25615',
'x-iplb-instance': '12309'
};
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.intercept("/container1", "HEAD")
.reply(_statusCode);
storage.headBucket('container1', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, _statusCode);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return metadatas about a container (ALIAS)', function(done){
const _statusCode = 204;
const _headers = {
'content-type': 'application/json; charset=utf-8',
'x-container-object-count': '4',
'x-container-bytes-used': '431631',
'x-timestamp': '1641995016.74740',
'x-container-sync-key': 'ZAIrQQjDzWM+APD5Fz6stQ/pBUe+Nck2UnPPswJw1cU=',
'x-container-sync-to': '//OVH_PUBLIC_CLOUD/GRA/AUTH_XXXX/container1',
'last-modified': 'Wed, 12 Jan 2022 13:58:51 GMT',
'accept-ranges': 'bytes',
'x-storage-policy': 'PCS',
vary: 'Accept',
'x-trans-id': 'tx3bd64d99f87147f18ffd9-0064c7d400',
'x-openstack-request-id': 'tx3bd64d99f87147f18ffd9-0064c7d400',
date: 'Mon, 31 Jul 2023 15:32:17 GMT',
'transfer-encoding': 'chunked',
'x-iplb-request-id': '53C629C3:E1FB_3626E64B:01BB_64C7D3FF_150CAC23:25615',
'x-iplb-instance': '12309'
};
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.intercept("/invoices-gra-1234", "HEAD")
.reply(_statusCode);
storage.headBucket('invoices', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, _statusCode);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return an error if the container does not exist', function (done) {
const _statusCode = 404;
const _headers = {
'content-type': 'text/html; charset=UTF-8',
'content-length': '0',
'x-trans-id': 'txe8450fb1fc8847dbb43bd-0064c7d4db',
'x-openstack-request-id': 'txe8450fb1fc8847dbb43bd-0064c7d4db',
date: 'Mon, 31 Jul 2023 15:35:56 GMT',
'x-iplb-request-id': '53C629C3:E222_3626E64B:01BB_64C7D4DB_169B0C5F:9CAC',
'x-iplb-instance': '12308'
};
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.intercept("/container1", "HEAD")
.reply(_statusCode);
storage.headBucket('container1', function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, _statusCode);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(resp.body.toString(), '');
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
});
describe('listBuckets', function() {
it('should return the list of containers (as Object)', function (done) {
const _headers = {
'content-type': 'application/json; charset=utf-8',
'x-account-container-count': '3',
'x-account-object-count': '67',
'x-account-bytes-used': '471229983',
'x-timestamp': '1641994731.69438',
'accept-ranges': 'bytes',
'content-length': '310',
'x-account-project-domain-id': 'default',
vary: 'Accept',
'x-trans-id': 'tx80980ebf27f64de29c8a5-0064c7d100',
'x-openstack-request-id': 'tx80980ebf27f64de29c8a5-0064c7d100',
date: 'Mon, 31 Jul 2023 15:19:28 GMT',
'x-iplb-request-id': '53C629C3:E186_5762BBC9:01BB_64C7D0FF_C1131CF:1263E',
'x-iplb-instance': '48126'
};
const _body = [
{
name: 'container1',
count: 55,
bytes: 106522,
last_modified: '2022-01-12T14:02:33.672010'
},
{
name: 'container2',
count: 8,
bytes: 470691830,
last_modified: '2022-10-24T10:59:13.250920'
},
{
name: 'container3',
count: 4,
bytes: 431631,
last_modified: '2022-01-12T13:58:50.868090'
}
];
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.intercept("/", "GET")
.reply(200, JSON.stringify(_body));
storage.listBuckets(function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(resp.body[0].name, _body[0].name);
assert.strictEqual(resp.body[0].count, _body[0].count);
assert.strictEqual(resp.body[0].bytes, _body[0].bytes);
assert.strictEqual(resp.body[0].last_modified, _body[0].last_modified);
assert.strictEqual(resp.body[1].name, _body[1].name);
assert.strictEqual(resp.body[2].name, _body[2].name);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return an error if the body returned is not a valid JSON format (should never happen)', function (done) {
const _expectedBody = "BODY NOT VALID";
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders({ 'content-type': 'application/json; charset=utf-8' })
.intercept("/", "GET")
.reply(200, _expectedBody);
storage.listBuckets(function(err, resp) {
assert.strictEqual(err.toString(), 'Error: Listing bucket JSON parse: SyntaxError: Unexpected token \'B\', "BODY NOT VALID" is not valid JSON');
assert.strictEqual(resp.body.toString(), _expectedBody);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return the body as Buffer if the content-type is `application/xml`', function(done) {
const _expectedBody = '<?xml version="1.0" encoding="UTF-8"?>\n<container name="templates"><object><name>Screenshot 2023-06-16 at 13.26.49.png</name><hash>184e81896337c81836419419455b9954</hash><bytes>166963</bytes><content_type>image/png</content_type><last_modified>2023-06-16T15:30:34.093290</last_modified></object></container>';
const _headers = {
'content-type': 'application/xml',
'x-account-container-count': '3',
'x-account-object-count': '67',
'x-account-bytes-used': '471229983',
'x-timestamp': '1641994731.69438',
'accept-ranges': 'bytes',
'content-length': '310',
'x-account-project-domain-id': 'default',
vary: 'Accept',
'x-trans-id': 'tx80980ebf27f64de29c8a5-0064c7d100',
'x-openstack-request-id': 'tx80980ebf27f64de29c8a5-0064c7d100',
date: 'Mon, 31 Jul 2023 15:19:28 GMT',
'x-iplb-request-id': '53C629C3:E186_5762BBC9:01BB_64C7D0FF_C1131CF:1263E',
'x-iplb-instance': '48126'
};
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders(_headers)
.intercept("/", "GET")
.reply(200, _expectedBody);
storage.listBuckets({ headers: { "Accept": "application/xml" } }, function(err, resp) {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), JSON.stringify(_headers));
assert.strictEqual(resp.body.toString(), _expectedBody);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
});
describe('log', function () {
it('should overload the log function', function (done) {
let i = 0;
storage.setLogFunction(function (message) {
assert.strictEqual(message.length > 0, true);
i++;
});
const firstMock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.connection((err) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(storage.getConfig().token, tokenAuth);
assert.deepStrictEqual(storage.getConfig().endpoints.url, connectionResultSuccessV3.token.catalog[9].endpoints[20].url);
assert.strictEqual(i > 0, true);
assert.strictEqual(firstMock.pendingMocks().length, 0);
done();
});
});
});
describe('setStorage/getStorages/setTimeout/getConfig', function () {
it('should update the initial configuration', function (done) {
const _expectedConfig = {
authUrl : 'https://carbone.io',
username : 'John',
password : 'Wick',
tenantName : 'toto',
region : 'GRA22'
};
storage.setStorages(_expectedConfig);
const _storages = storage.getStorages();
assert.strictEqual(_storages[0].authUrl, _expectedConfig.authUrl);
assert.strictEqual(_storages[0].username, _expectedConfig.username);
assert.strictEqual(_storages[0].password, _expectedConfig.password);
assert.strictEqual(_storages[0].tenantName, _expectedConfig.tenantName);
assert.strictEqual(_storages[0].region, _expectedConfig.region);
done();
});
it('should set the request timeout', function (done) {
storage.setTimeout(200);
assert.strictEqual(storage.getConfig().timeout, 200);
done();
});
});
describe('listFiles', function() {
describe("SINGLE STORAGE", function () {
it('should return a list of files as a JSON', function (done) {
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders({
'content-type': 'application/json',
})
.get('/templates')
.reply(200, () => {
return fs.createReadStream(path.join(__dirname, 'assets', 'files.json'));
});
storage.listFiles('templates', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), '{"content-type":"application/json"}');
const _files = resp.body;
assert.strictEqual(_files.length > 0, true);
assert.strictEqual(_files[0].bytes > 0, true);
assert.strictEqual(_files[0].last_modified.length > 0, true);
assert.strictEqual(_files[0].hash.length > 0, true);
assert.strictEqual(_files[0].name.length > 0, true);
assert.strictEqual(_files[0].content_type.length > 0, true);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return a list of files as a JSON (ALIAS)', function (done) {
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders({
'content-type': 'application/json',
})
.get('/invoices-gra-1234')
.reply(200, () => {
return fs.createReadStream(path.join(__dirname, 'assets', 'files.json'));
});
storage.listFiles('invoices', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), '{"content-type":"application/json"}');
const _files = resp.body;
assert.strictEqual(_files.length > 0, true);
assert.strictEqual(_files[0].bytes > 0, true);
assert.strictEqual(_files[0].last_modified.length > 0, true);
assert.strictEqual(_files[0].hash.length > 0, true);
assert.strictEqual(_files[0].name.length > 0, true);
assert.strictEqual(_files[0].content_type.length > 0, true);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return a list of files as a XML and the header is overwritted', function (done) {
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders({
'content-type': 'application/xml',
})
.get('/templates')
.reply(200, () => {
return fs.createReadStream(path.join(__dirname, 'assets', 'files.xml'));
});
storage.listFiles('templates', { headers : { Accept: 'application/xml' } }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), '{"content-type":"application/xml"}');
const _files = resp.body.toString();
assert.strictEqual(_files.includes('<?xml'), true);
assert.strictEqual(_files.includes('<container name="templates">'), true);
assert.strictEqual(_files.includes('<bytes>47560</bytes>'), true);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return a list of files with a prefix', function (done) {
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders({
'content-type': 'application/json',
})
.get('/templates')
.query({ prefix : 'keys' })
.reply(200, () => {
const _file = fs.readFileSync(path.join(__dirname, 'assets', 'files.json'));
return Buffer.from(JSON.stringify(JSON.parse(_file.toString()).filter((el => el.name.includes('keys')))));
});
storage.listFiles('templates', { queries: { prefix: 'keys' } }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), '{"content-type":"application/json"}');
const _files = resp.body;
_files.forEach(el => {
assert.strictEqual(el.name.includes('keys'), true);
});
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should reconnect automatically to the storage', function (done) {
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders({
'content-type': 'application/json',
})
.get('/templates')
.reply(401, 'Unauthorized')
.get('/templates')
.reply(200, () => {
return fs.createReadStream(path.join(__dirname, 'assets', 'files.json'));
});
const secondNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.listFiles('templates', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), '{"content-type":"application/json"}');
const _files = resp.body;
assert.strictEqual(_files.length > 0, true);
assert.strictEqual(_files[0].bytes > 0, true);
assert.strictEqual(_files[0].last_modified.length > 0, true);
assert.strictEqual(_files[0].hash.length > 0, true);
assert.strictEqual(_files[0].name.length > 0, true);
assert.strictEqual(_files[0].content_type.length > 0, true);
assert.strictEqual(firstNock.pendingMocks().length, 0);
assert.strictEqual(secondNock.pendingMocks().length, 0);
done();
});
});
it('should reconnect automatically to the storage with a prefix and delimiter as option/query parameters', function (done) {
const firstNock = nock(publicUrlGRA)
.defaultReplyHeaders({
'content-type': 'application/json',
})
.get('/templates')
.query({ prefix : 'keys', delimiter : '/' })
.reply(401, 'Unauthorized')
.get('/templates')
.query({ prefix : 'keys', delimiter : '/' })
.reply(200, () => {
return Buffer.from(JSON.stringify([
{
"subdir": "keys/"
}]));
});
const secondNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.listFiles('templates', { queries: { prefix: 'keys', delimiter : '/' } }, (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 200);
assert.strictEqual(JSON.stringify(resp.headers), '{"content-type":"application/json"}');
const _files = resp.body;
assert.strictEqual(_files[0].subdir, 'keys/');
assert.strictEqual(firstNock.pendingMocks().length, 0);
assert.strictEqual(secondNock.pendingMocks().length, 0);
done();
});
});
it('should return an error if the single storage timeout', function (done) {
storage.setTimeout(200);
const firstNock = nock(publicUrlGRA)
.get('/templates')
.delayConnection(500)
.reply(200, {});
storage.listFiles('templates', (err, resp) => {
assert.strictEqual(err.toString(), 'Error: Object Storages are not available');
assert.strictEqual(resp, undefined);
assert.strictEqual(storage.getConfig().activeStorage, 0);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should return an error if the single storage return any kind of errors', function (done) {
const firstNock = nock(publicUrlGRA)
.get('/templates')
.replyWithError('Error Message 1234');
storage.listFiles('templates', (err, resp) => {
assert.strictEqual(err.toString(), 'Error: Object Storages are not available');
assert.strictEqual(resp, undefined);
assert.strictEqual(firstNock.pendingMocks().length, 0);
assert.deepStrictEqual(storage.getConfig().activeStorage, 0);
done();
});
});
it('should return an error if the container does not exist', function (done) {
const _expectedContent = '<html><h1>Not Found</h1><p>The resource could not be found.</p></html>';
const firstNock = nock(publicUrlGRA)
.get('/templates')
.reply(404, _expectedContent);
storage.listFiles('templates', (err, resp) => {
assert.strictEqual(err, null);
assert.strictEqual(resp.statusCode, 404);
assert.strictEqual(resp.body.toString(), _expectedContent);
assert.strictEqual(JSON.stringify(resp.headers), '{}');
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
});
describe("MUTLIPLE STORAGES", function () {
beforeEach(function (done) {
const firstNock = nock(authURL)
.post('/auth/tokens')
.reply(200, connectionResultSuccessV3, { "X-Subject-Token": tokenAuth });
storage.setTimeout(5000);
storage.setStorages([{
username : 'storage-1-user',
password : 'storage-1-password',
authUrl : authURL,
tenantName : 'storage-1-tenant',
region : 'GRA'
},
{
username : 'storage-2-user',
password : 'storage-2-password',
authUrl : authURL,
tenantName : 'storage-2-tenant',
region : 'SBG'
}]);
storage.connection((err) => {
assert.strictEqual(err, null);
assert.strictEqual(firstNock.pendingMocks().length, 0);
done();
});
});
it('should reconnect automatically to the second object storage if the first storage authentication fail and should retry the request', function (done) {
const firstNock = nock(publicUrlGRA)
/** 1 */
.get('/templates')
.reply(401, 'Unauthorized');
const secondNock = nock(aut