@cumulus/ingest/SftpProviderClient.js

Ingest utilities

//@ts-check
'use strict';
const get = require('lodash/get');
const KMS = require('@cumulus/aws-client/KMS');
const log = require('@cumulus/common/log');
const omit = require('lodash/omit');
const S3 = require('@cumulus/aws-client/S3');
const { SftpClient } = require('@cumulus/sftp-client');
const isNil = require('lodash/isNil');
const { getRequiredEnvVar } = require('@cumulus/common/env');
const { recursion } = require('./recursion');
const { decrypt } = require('./util');
/**
 * @typedef {import('./recursion').RecursionFile} RecursionFile
 */
class SftpProviderClient {
    /**
     *
     * @param {Object} providerConfig
     * @param {string} providerConfig.username
     * @param {string} providerConfig.password
     * @param {string} providerConfig.plaintextProviderKey
     * @param {string} providerConfig.host
     * @param {string} providerConfig.privateKey
     * @param {string} providerConfig.cmKeyId
     */
    constructor(providerConfig) {
        this.providerConfig = providerConfig;
        if (get(providerConfig, 'encrypted', false) === false) {
            this.plaintextUsername = providerConfig.username;
            this.plaintextPassword = providerConfig.password;
            this.plaintextPrivateKey = providerConfig.plaintextProviderKey;
        }
        this.connected = false;
        this.privateSftpClient = undefined;
    }
    async connect() {
        if (this.connected)
            return;
        this.privateSftpClient = new SftpClient({
            host: this.host,
            port: get(this.providerConfig, 'port', 22),
            username: await this.getUsername(),
            password: await this.getPassword(),
            privateKey: await this.getPrivateKey(),
        });
        await this.privateSftpClient.connect();
        this.connected = true;
    }
    async end() {
        if (this.connected) {
            await this.getSftpClient().end();
            this.connected = false;
        }
    }
    /* @private */
    get host() {
        return this.providerConfig.host;
    }
    /* @private */
    async getUsername() {
        if (isNil(this.providerConfig.username))
            return undefined;
        if (isNil(this.plaintextUsername)) {
            this.plaintextUsername = await decrypt(this.providerConfig.username);
        }
        return this.plaintextUsername;
    }
    /* @private */
    async getPassword() {
        if (isNil(this.providerConfig.password))
            return undefined;
        if (isNil(this.plaintextPassword)) {
            this.plaintextPassword = await decrypt(this.providerConfig.password);
        }
        return this.plaintextPassword;
    }
    /* @private */
    async getPrivateKey() {
        if (isNil(this.providerConfig.privateKey))
            return undefined;
        if (isNil(this.plaintextProviderKey)) {
            // we are assuming that the specified private key is in the S3 crypto
            // directory
            const fetchedKey = await S3.getTextObject(getRequiredEnvVar('system_bucket', process.env), `${process.env.stackName}/crypto/${this.providerConfig.privateKey}`);
            if (this.providerConfig.cmKeyId) {
                // we are using AWS KMS and the privateKey is encrypted
                this.plaintextProviderKey = await KMS.decryptBase64String(fetchedKey);
            }
            else {
                this.plaintextProviderKey = fetchedKey;
            }
        }
        return this.plaintextProviderKey;
    }
    /** @private
     * @returns {SftpClient}
     */
    getSftpClient() {
        if (!this.connected || !this.privateSftpClient) {
            throw new Error('Client not connected');
        }
        return this.privateSftpClient;
    }
    /**
     * Download a remote file to disk
     * @param {Object} params             - parameter object
     * @param {string} params.remotePath  - the full path to the remote file to be fetched
     * @param {string} params.localPath   - the full local destination file path
     * @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
     * @returns {Promise<void>}        - the path that the file was saved to
     */
    async download(params) {
        const { remotePath, localPath, fastDownload } = params;
        return await this.getSftpClient().download(remotePath, localPath, fastDownload);
    }
    /**
     * List all files from a given endpoint
     *
     * @param {string} path - the path to list
     * @returns {Promise<Array<Pick<RecursionFile, "name">>>}
     */
    async list(path) {
        const sftpClient = this.getSftpClient();
        const listFn = sftpClient.list.bind(sftpClient);
        const files = await recursion(listFn, path);
        log.info({ host: this.host }, `${files.length} files were found on ${this.host}`);
        // Type 'type' field is required to support recursive file listing, but
        // should not be part of the returned result.
        return files.map((file) => omit(file, 'type'));
    }
    /**
     * Transfer the remote file to a given s3 location
     *
     * @param {Object} params
     * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
     * @param {string} params.destinationBucket - destination s3 bucket of the file
     * @param {string} params.destinationKey - destination s3 key of the file
     * @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
     * @returns {Promise<{ s3uri: string, etag?: string }>} an object containing
     *    the S3 URI and ETag of the destination file
     */
    async sync(params) {
        const remotePath = params.fileRemotePath;
        const bucket = params.destinationBucket;
        const key = params.destinationKey;
        const syncMethod = params.fastDownload ? 'syncToS3Fast' : 'syncToS3';
        return await this.getSftpClient()[syncMethod](remotePath, bucket, key);
    }
}
module.exports = SftpProviderClient;
//# sourceMappingURL=SftpProviderClient.js.map