@bitblit/ratchet-aws
Version:
Common tools for use with AWS browser and node
140 lines • 6.17 kB
JavaScript
import { CopyObjectCommand, GetObjectCommand, ListObjectsV2Command, } from '@aws-sdk/client-s3';
import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
import { Logger } from '@bitblit/ratchet-common/logger/logger';
import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
import { Upload } from '@aws-sdk/lib-storage';
export class S3LocationSyncRatchet {
config;
constructor(config) {
RequireRatchet.notNullOrUndefined(config, 'config');
this.config = config;
if (!this.config.maxNumThreads) {
this.config.maxNumThreads = 15;
}
if (!this.config.maxRetries) {
this.config.maxRetries = 5;
}
}
updateSrcPrefix(prefix) {
this.config.srcPrefix = prefix;
}
updateDstPrefix(prefix) {
this.config.dstPrefix = prefix;
}
async copyObject(key, size, express = false) {
const dstKey = key.replace(this.config.srcPrefix, this.config.dstPrefix);
let completedCopying = false;
let retries = 0;
while (!completedCopying && retries < this.config.maxRetries) {
Logger.debug(`${retries > 0 ? `Retry ${retries} ` : ''}${express ? 'Express' : 'Slow'} copying
[${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
try {
if (express) {
const params = {
CopySource: encodeURIComponent([this.config.srcBucket, key].join('/')),
Bucket: this.config.dstBucket,
Key: dstKey,
MetadataDirective: 'COPY',
};
await this.config.dstS3.send(new CopyObjectCommand(params));
}
else {
const fetched = await this.config.srcS3.send(new GetObjectCommand({ Bucket: this.config.srcBucket, Key: key }));
const params = {
Bucket: this.config.dstBucket,
Key: dstKey,
Body: fetched.Body,
ContentLength: size,
};
const upload = new Upload({
client: this.config.dstS3,
params: params,
tags: [],
queueSize: 4,
partSize: 1024 * 1024 * 5,
leavePartsOnError: false,
});
upload.on('httpUploadProgress', (progress) => {
Logger.debug('Uploading : %s', progress);
});
await upload.done();
}
completedCopying = true;
}
catch (err) {
Logger.warn(`Can't ${express ? 'express' : 'slow'} copy
[${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]: %j`, err);
retries++;
}
}
Logger.debug(`Finished ${express ? 'express' : 'slow'} copying
[${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
}
async listObjects(bucket, prefix, s3) {
Logger.info(`Scanning bucket [${[bucket, prefix].join('/')}]`);
const params = {
Bucket: bucket,
Prefix: prefix,
};
let more = true;
const rval = {};
while (more) {
const response = await s3.send(new ListObjectsV2Command(params));
more = response.IsTruncated;
response.Contents.forEach((obj) => {
rval[obj.Key] = { Key: obj.Key, LastModified: obj.LastModified, ETag: obj.ETag, Size: obj.Size };
});
if (more) {
params.ContinuationToken = response.NextContinuationToken;
}
}
return rval;
}
async startSyncing() {
Logger.info(`Syncing [${this.config.srcBucket}/${this.config.srcPrefix}
---> ${this.config.dstBucket}/${this.config.dstPrefix}]`);
const cp = async (obj) => {
await this.copyObject(obj.Key, obj.Size);
};
let cmpResult = await this.compareSrcAndDst();
if (cmpResult.needCopy.length > 0 || cmpResult.diff.length > 0) {
await PromiseRatchet.runBoundedParallelSingleParam(cp, cmpResult.needCopy, this, this.config.maxNumThreads);
await PromiseRatchet.runBoundedParallelSingleParam(cp, cmpResult.diff, this, this.config.maxNumThreads);
Logger.info('Verifying...');
cmpResult = await this.compareSrcAndDst();
Logger.debug('Compare result %j', cmpResult);
}
return cmpResult.needCopy.length === 0 && cmpResult.diff.length === 0;
}
async compareSrcAndDst() {
const getSrc = this.listObjects(this.config.srcBucket, this.config.srcPrefix, this.config.srcS3);
const getDst = this.listObjects(this.config.dstBucket, this.config.dstPrefix, this.config.dstS3);
const srcObjs = await getSrc;
const dstObjs = await getDst;
const rval = {
needCopy: [],
existed: [],
diff: [],
};
await PromiseRatchet.runBoundedParallelSingleParam(async (key) => {
const sObj = srcObjs[key];
const dstKey = key.replace(this.config.srcPrefix, this.config.dstPrefix);
const dObj = dstKey in dstObjs ? dstObjs[dstKey] : undefined;
if (!dObj) {
rval.needCopy.push(sObj);
return;
}
if (sObj.Size !== dObj.Size) {
rval.diff.push(sObj);
return;
}
if (sObj.LastModified.getTime() <= dObj.LastModified.getTime()) {
rval.existed.push(sObj);
return;
}
rval.diff.push(sObj);
}, Object.keys(srcObjs), this, this.config.maxNumThreads);
return rval;
}
}
//# sourceMappingURL=s3-location-sync-ratchet.js.map