mwoffliner/lib/S3.js

MediaWiki ZIM scraper

import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadBucketCommand } from '@aws-sdk/client-s3';
import * as logger from './Logger.js';
import { publicIpv4 } from 'public-ip';
import { NodeHttpHandler } from '@smithy/node-http-handler';
import { Agent } from 'https';
class S3 {
    url;
    params;
    s3Handler;
    bucketName;
    region;
    reqTimeout;
    insecure;
    constructor(s3Url, s3Params, reqTimeout, insecure) {
        this.url = s3Url;
        this.params = s3Params;
        this.bucketName = s3Params.get('bucketName');
        this.reqTimeout = reqTimeout;
        this.insecure = insecure;
        this.setRegion();
    }
    setRegion() {
        const url = new URL(this.url);
        const regionRegex = /^s3\.([^.]+)/;
        const match = url.hostname.match(regionRegex);
        if (match && match[1]) {
            this.region = match[1];
        }
        else {
            throw new Error('Unknown S3 region set');
        }
    }
    async initialise() {
        const s3UrlBase = new URL(this.url);
        this.s3Handler = new S3Client({
            credentials: {
                accessKeyId: this.params.get('keyId'),
                secretAccessKey: this.params.get('secretAccessKey'),
            },
            endpoint: s3UrlBase.href,
            forcePathStyle: s3UrlBase.protocol === 'http:',
            region: this.region,
            requestHandler: new NodeHttpHandler({
                connectionTimeout: this.reqTimeout,
                requestTimeout: this.reqTimeout,
                httpAgent: new Agent({ keepAlive: true }),
                httpsAgent: new Agent({ keepAlive: true, rejectUnauthorized: !this.insecure }), // rejectUnauthorized: false disables TLS
            }),
        });
        return this.bucketExists(this.bucketName)
            .then(() => true)
            .catch(async () => {
            throw new Error(`
        Unable to connect to S3, either S3 login credentials are wrong or bucket cannot be found
                            Bucket used: ${this.bucketName}
                            End point used: ${s3UrlBase.href}
                            Public IP used: ${await publicIpv4()}
        `);
        });
    }
    bucketExists(bucket) {
        const command = new HeadBucketCommand({ Bucket: bucket });
        return new Promise((resolve, reject) => {
            this.s3Handler.send(command, (err) => {
                return err ? reject(err) : resolve(true);
            });
        });
    }
    uploadBlob(key, data, eTag, version) {
        const command = new PutObjectCommand({
            Bucket: this.bucketName,
            Key: key,
            Metadata: { etag: eTag, version },
            Body: data,
        });
        return new Promise((resolve, reject) => {
            this.s3Handler
                .send(command)
                .then((response) => {
                resolve(response);
            })
                .catch((err) => {
                logger.error('Cache error while uploading object:', err);
                reject(err);
            });
        });
    }
    downloadBlob(key, version = '1') {
        const command = new GetObjectCommand({ Bucket: this.bucketName, Key: key });
        return new Promise((resolve, reject) => {
            this.s3Handler
                .send(command)
                .then((response) => {
                if (response) {
                    const { Metadata } = response;
                    if (Metadata?.version !== version) {
                        Metadata.etag = undefined;
                    }
                    resolve(response);
                }
                else
                    reject();
            })
                .catch((err) => {
                // For 404 error handle AWS service-specific exception
                if (err && err.name === 'NoSuchKey') {
                    logger.info(`The specified key '${key}' does not exist in the cache.`);
                    resolve(null);
                }
                else {
                    logger.error(`Error (${err}) while downloading the object '${key}' from the cache.`);
                    reject(err);
                }
            });
        });
    }
    // Only for testing purpose
    deleteBlob(params) {
        const command = new DeleteObjectCommand(params);
        return new Promise((resolve, reject) => {
            this.s3Handler
                .send(command)
                .then((val) => resolve(val))
                .catch((err) => {
                logger.error('Error while deleting object in the cache', err);
                reject(err);
            });
        });
    }
}
export default S3;
//# sourceMappingURL=S3.js.map