@zowe/zos-files-for-zowe-sdk
Version:
Zowe SDK to interact with files and data sets on z/OS
776 lines • 43.7 kB
JavaScript
;
/*
* This program and the accompanying materials are made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v20.html
*
* SPDX-License-Identifier: EPL-2.0
*
* Copyright Contributors to the Zowe Project.
*
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Download = void 0;
const imperative_1 = require("@zowe/imperative");
const path_1 = require("path");
const fs = require("fs");
const util = require("util");
const core_for_zowe_sdk_1 = require("@zowe/core-for-zowe-sdk");
const ZosFiles_constants_1 = require("../../constants/ZosFiles.constants");
const ZosFiles_messages_1 = require("../../constants/ZosFiles.messages");
const ZosFilesUtils_1 = require("../../utils/ZosFilesUtils");
const List_1 = require("../list/List");
const ZosmfRestClientProperties_1 = require("../../doc/types/ZosmfRestClientProperties");
const utilities_1 = require("../utilities");
const ZosFilesAttributes_1 = require("../../utils/ZosFilesAttributes");
/**
* This class holds helper functions that are used to download data sets, members and more through the z/OS MF APIs
*/
class Download {
/**
* Retrieve data sets and/or members contents and save them in your local workspace
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} dataSetName - contains the data set name
* @param {IDownloadSingleOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} data set name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*
* @example
* ```typescript
*
* // Download "USER.DATA.SET.PS" to "user/data/set/ps.txt"
* await Download.dataSet(session, "USER.DATA.SET.PS");
*
* // Download "USER.DATA.SET.PDS(MEMBER)" to "user/data/set/pds/member.txt"
* await Download.dataSet(session, "USER.DATA.SET.PDS(MEMBER)");
*
* // Download "USER.DATA.SET" to "./path/to/file.txt"
* await Download.dataSet(session, "USER.DATA.SET", {file: "./path/to/file.txt"});
* ```
*
* @see https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.izua700/IZUHPINFO_API_GetReadDataSet.htm
*/
static dataSet(session_1, dataSetName_1) {
return __awaiter(this, arguments, void 0, function* (session, dataSetName, options = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(dataSetName, ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
imperative_1.ImperativeExpect.toNotBeEqual(dataSetName, "", ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
let destination;
try {
// Format the endpoint to send the request to
let endpoint = path_1.posix.join(ZosFiles_constants_1.ZosFilesConstants.RESOURCE, ZosFiles_constants_1.ZosFilesConstants.RES_DS_FILES);
if (options.volume) {
endpoint = path_1.posix.join(endpoint, `-(${encodeURIComponent(options.volume)})`);
}
endpoint = path_1.posix.join(endpoint, encodeURIComponent(dataSetName));
if (options.queryParams) {
endpoint += options.queryParams;
}
imperative_1.Logger.getAppLogger().debug(`Endpoint: ${endpoint}`);
const reqHeaders = this.generateHeadersBasedOnOptions(options);
// Get contents of the data set
let extension = ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION;
if (options.extension != null) {
extension = options.extension;
}
if (options.stream == null) {
// Get a proper destination for the file to be downloaded
// If the "file" is not provided, we create a folder structure similar to the data set name
// Note that the "extension" options do not affect the destination if the "file" options were provided
destination = (() => {
if (options.file) {
return options.file;
}
let generatedFilePath = ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetName);
// Method above lowercased characters.
// In case of preserving original letter case, uppercase all characters.
if (options.preserveOriginalLetterCase) {
generatedFilePath = generatedFilePath.toUpperCase();
}
return generatedFilePath + imperative_1.IO.normalizeExtension(extension);
})();
imperative_1.IO.createDirsSyncFromFilePath(destination);
}
const writeStream = (_a = options.stream) !== null && _a !== void 0 ? _a : imperative_1.IO.createWriteStream(destination);
// Use specific options to mimic ZosmfRestClient.getStreamed()
const requestOptions = {
resource: endpoint,
reqHeaders,
responseStream: writeStream,
normalizeResponseNewLines: !(options.binary || options.record),
task: options.task
};
if (options.range) {
reqHeaders.push({ [core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RECORD_RANGE]: options.range });
}
// If requestor needs etag, add header + get "response" back
if (options.returnEtag) {
requestOptions.reqHeaders.push(core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RETURN_ETAG);
requestOptions.dataToReturn = [ZosmfRestClientProperties_1.CLIENT_PROPERTY.response];
}
const request = yield core_for_zowe_sdk_1.ZosmfRestClient.getExpectFullResponse(session, requestOptions);
// By default, apiResponse is empty when downloading
const apiResponse = {};
// Return Etag in apiResponse, if requested
if (options.returnEtag) {
apiResponse.etag = request.response.headers.etag;
}
return {
success: true,
commandResponse: destination != null ? util.format(ZosFiles_messages_1.ZosFilesMessages.datasetDownloadedWithDestination.message, destination) :
ZosFiles_messages_1.ZosFilesMessages.datasetDownloadedSuccessfully.message,
apiResponse
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
if (destination != null) {
imperative_1.IO.deleteFile(destination);
}
throw error;
}
});
}
/**
* Retrieve all members from a PDS and save them in your local workspace
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} dataSetName - contains the data set name
* @param {IDownloadOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} data set name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*
* @example
* ```typescript
*
* // Download all members of "USER.DATA.SET.PDS" to "user/data/set/pds/"
* await Download.allMembers(session, "USER.DATA.SET.PDS");
*
* // Download all members of "USER.DATA.SET.PDS" to "./path/to/dir/"
* await Download.allMembers(session, "USER.DATA.SET.PDS", {directory: "./path/to/dir/"});
* ```
*
* @see https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.izua700/IZUHPINFO_API_GetReadDataSet.htm
*/
static allMembers(session_1, dataSetName_1) {
return __awaiter(this, arguments, void 0, function* (session, dataSetName, options = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(dataSetName, ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
imperative_1.ImperativeExpect.toNotBeEqual(dataSetName, "", ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
try {
const response = yield List_1.List.allMembers(session, dataSetName, {
volume: options.volume,
responseTimeout: options.responseTimeout
});
const memberList = (_a = options.memberPatternResponse) !== null && _a !== void 0 ? _a : response.apiResponse.items;
if (memberList.length === 0) {
return {
success: false,
commandResponse: ZosFiles_messages_1.ZosFilesMessages.noMembersFound.message,
apiResponse: response.apiResponse
};
}
// If the "directory" option is provided, use it.
// Otherwise use default directory generated from data set name.
const baseDir = (() => {
if (options.directory) {
return options.directory;
}
let generatedDirectory = ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetName);
// Method above lowercased characters.
// In case of preserving original letter case, uppercase all characters.
if (options.preserveOriginalLetterCase) {
generatedDirectory = generatedDirectory.toUpperCase();
}
return generatedDirectory;
})();
const downloadErrors = [];
const failedMembers = [];
let downloadsInitiated = 0;
let defaultExtension = ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION;
if (options.extension != null) {
defaultExtension = options.extension;
}
/**
* Function that takes a member and turns it into a promise to download said member
* @param mem - an object with a "member" field containing the name of the data set member
*/
const createDownloadPromise = (mem) => {
var _a;
// update the progress bar if any
if (options.task != null) {
options.task.statusMessage = "Downloading " + mem.member;
options.task.percentComplete = Math.floor(imperative_1.TaskProgress.ONE_HUNDRED_PERCENT *
(downloadsInitiated / memberList.length));
downloadsInitiated++;
}
const fileName = options.preserveOriginalLetterCase ? mem.member : mem.member.toLowerCase();
// Determine extension using extensionMap if provided
let extension = defaultExtension;
if (options.extensionMap != null) {
// Use the member name as the key, lowercased if preserveOriginalLetterCase is false
extension = (_a = options.extensionMap[fileName]) !== null && _a !== void 0 ? _a : extension;
}
// Normalize the extension, remove leading periods
if (extension && extension.startsWith(".")) {
extension = extension.replace(/^\.+/g, "");
}
return this.dataSet(session, `${dataSetName}(${mem.member})`, {
volume: options.volume,
file: path_1.posix.join(baseDir, fileName + imperative_1.IO.normalizeExtension(extension)),
binary: options.binary,
record: options.record,
encoding: options.encoding,
responseTimeout: options.responseTimeout
}).catch((err) => {
downloadErrors.push(err);
failedMembers.push(fileName);
// Delete the file that could not be downloaded
imperative_1.IO.deleteFile((0, path_1.join)(baseDir, fileName + imperative_1.IO.normalizeExtension(extension)));
// If we should fail fast, rethrow error
if (options.failFast || options.failFast === undefined) {
throw err;
}
});
};
const maxConcurrentRequests = options.maxConcurrentRequests == null ? 1 : options.maxConcurrentRequests;
if (maxConcurrentRequests === 0) {
yield Promise.all(memberList.map(createDownloadPromise));
}
else {
yield (0, core_for_zowe_sdk_1.asyncPool)(maxConcurrentRequests, memberList, createDownloadPromise);
}
// Handle failed downloads if no errors were thrown yet
if (downloadErrors.length > 0) {
throw new imperative_1.ImperativeError({
msg: ZosFiles_messages_1.ZosFilesMessages.memberDownloadFailed.message + failedMembers.join("\n") + "\n\n" +
downloadErrors.map((err) => err.message).join("\n"),
causeErrors: downloadErrors,
additionalDetails: failedMembers.join("\n")
});
}
return {
success: true,
commandResponse: util.format(ZosFiles_messages_1.ZosFilesMessages.memberDownloadedWithDestination.message, baseDir),
apiResponse: response.apiResponse
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
});
}
/**
* Download a list of data sets to local files
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {IZosmfListResponse[]} dataSetObjs - contains data set objects returned by z/OSMF List API
* @param {IDownloadOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} data set name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*
* @example
* ```typescript
*
* // Download a list of "PS" and "PO" datasets to the directory "./path/to/dir/"
* await Download.allDataSets(session, [
* { dsname: "USER.DATA.SET.PS", dsorg: "PS" },
* { dsname: "USER.DATA.SET.PDS", dsorg: "PO" }
* ], {directory: "./path/to/dir/"});
* ```
*
* @see https://www.ibm.com/support/knowledgecenter/SSLTBW_2.2.0/com.ibm.zos.v2r2.izua700/IZUHPINFO_API_GetReadDataSet.htm
*/
static allDataSets(session_1, dataSetObjs_1) {
return __awaiter(this, arguments, void 0, function* (session, dataSetObjs, options = {}) {
var _a, _b, _c, _d, _e;
imperative_1.ImperativeExpect.toNotBeEqual(dataSetObjs.length, 0, ZosFiles_messages_1.ZosFilesMessages.missingDataSets.message);
const result = this.emptyDownloadDsmResult();
const zosmfResponses = [...dataSetObjs];
try {
// Download data sets
const poDownloadTasks = [];
const psDownloadTasks = [];
const mutableOptions = Object.assign(Object.assign({}, options), { task: undefined });
for (const dataSetObj of zosmfResponses) {
let llq = dataSetObj.dsname.substring(dataSetObj.dsname.lastIndexOf(".") + 1, dataSetObj.dsname.length);
if (!options.preserveOriginalLetterCase) {
llq = llq.toLowerCase();
}
if (options.extensionMap != null) {
mutableOptions.extension = (_a = options.extensionMap[llq]) !== null && _a !== void 0 ? _a : options.extension;
}
// Normalize the extension, remove leading periods
if (mutableOptions.extension && mutableOptions.extension.startsWith(".")) {
mutableOptions.extension = mutableOptions.extension.replace(/^\.+/g, "");
}
if (options.directory == null) {
if ((_b = dataSetObj.dsorg) === null || _b === void 0 ? void 0 : _b.startsWith("PO")) {
mutableOptions.directory = ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetObj.dsname);
}
else {
mutableOptions.file = `${dataSetObj.dsname}.` +
`${(_c = mutableOptions.extension) !== null && _c !== void 0 ? _c : ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION}`;
if (!options.preserveOriginalLetterCase) {
mutableOptions.file = mutableOptions.file.toLowerCase();
}
mutableOptions.directory = undefined;
mutableOptions.extension = undefined;
}
}
else if ((_d = dataSetObj.dsorg) === null || _d === void 0 ? void 0 : _d.startsWith("PO")) {
mutableOptions.directory = `${mutableOptions.directory}/${ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetObj.dsname)}`;
}
else {
mutableOptions.file = `${dataSetObj.dsname}.${(_e = mutableOptions.extension) !== null && _e !== void 0 ? _e : ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION}`;
if (!options.preserveOriginalLetterCase) {
mutableOptions.file = mutableOptions.file.toLowerCase();
}
mutableOptions.file = `${mutableOptions.directory}/${mutableOptions.file}`;
mutableOptions.directory = undefined;
mutableOptions.extension = undefined;
}
if (dataSetObj.error != null) {
result.failedWithErrors[dataSetObj.dsname] = dataSetObj.error;
}
else if (dataSetObj.dsorg == null) {
dataSetObj.status = `Skipped: Archived data set or alias - type ${dataSetObj.vol}.`;
result.failedArchived.push(dataSetObj.dsname);
}
else if (dataSetObj.dsorg === "PS") {
psDownloadTasks.push({
handler: Download.dataSet.bind(this),
dsname: dataSetObj.dsname,
options: Object.assign({}, mutableOptions),
onSuccess: (downloadResponse) => {
dataSetObj.status = downloadResponse.commandResponse;
}
});
}
else if (dataSetObj.dsorg.startsWith("PO")) {
poDownloadTasks.push({
handler: Download.allMembers.bind(this),
dsname: dataSetObj.dsname,
options: Object.assign({}, mutableOptions),
onSuccess: (downloadResponse, options) => {
dataSetObj.status = downloadResponse.commandResponse;
const listMembers = downloadResponse.apiResponse.items.map((item) => ` ${item.member}`);
if (listMembers.length === 0) { // Create directory for empty PO data set
imperative_1.IO.createDirsSyncFromFilePath(options.directory);
}
else {
dataSetObj.status += `\nMembers: ${listMembers};`;
}
}
});
}
else {
dataSetObj.status = `Skipped: Unsupported data set - type ${dataSetObj.dsorg}.`;
result.failedUnsupported.push(dataSetObj.dsname);
}
mutableOptions.directory = options.directory;
}
// If we should fail fast, throw error
if ((result.failedArchived.length > 0 || result.failedUnsupported.length > 0 ||
Object.keys(result.failedWithErrors).length > 0) && options.failFast !== false) {
throw new imperative_1.ImperativeError({
msg: ZosFiles_messages_1.ZosFilesMessages.failedToDownloadDataSets.message,
additionalDetails: this.buildDownloadDsmResponse(result, options)
});
}
let downloadsInitiated = 0;
const createDownloadPromise = (task) => {
if (options.task != null) {
options.task.statusMessage = "Downloading data set " + task.dsname;
options.task.percentComplete = Math.floor(imperative_1.TaskProgress.ONE_HUNDRED_PERCENT *
(downloadsInitiated / (poDownloadTasks.length + psDownloadTasks.length)));
downloadsInitiated++;
}
return task.handler(session, task.dsname, task.options).then((downloadResponse) => {
result.downloaded.push(task.dsname);
task.onSuccess(downloadResponse, task.options);
}, (err) => {
result.failedWithErrors[task.dsname] = err;
// If we should fail fast, rethrow error
if (options.failFast || options.failFast === undefined) {
throw new imperative_1.ImperativeError({
msg: `Failed to download ${task.dsname}`,
causeErrors: err,
additionalDetails: this.buildDownloadDsmResponse(result, options)
});
}
});
};
// First download the partitioned data sets
// We execute the promises sequentially to make sure that
// we do not exceed `--mcr` when downloading multiple members
for (const task of poDownloadTasks) {
yield createDownloadPromise(task);
}
// Next download the sequential data sets in a pool
const maxConcurrentRequests = options.maxConcurrentRequests == null ? 1 : options.maxConcurrentRequests;
if (maxConcurrentRequests === 0) {
yield Promise.all(psDownloadTasks.map(createDownloadPromise));
}
else {
yield (0, core_for_zowe_sdk_1.asyncPool)(maxConcurrentRequests, psDownloadTasks, createDownloadPromise);
}
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
// Handle failed downloads if no errors were thrown yet
if (Object.keys(result.failedWithErrors).length > 0) {
throw new imperative_1.ImperativeError({
msg: ZosFiles_messages_1.ZosFilesMessages.datasetDownloadFailed.message + Object.keys(result.failedWithErrors).join("\n"),
causeErrors: Object.values(result.failedWithErrors),
additionalDetails: this.buildDownloadDsmResponse(result, options)
});
}
const numFailed = result.failedArchived.length + result.failedUnsupported.length + Object.keys(result.failedWithErrors).length;
return {
success: numFailed === 0,
commandResponse: this.buildDownloadDsmResponse(result, options),
apiResponse: zosmfResponses,
errorMessage: numFailed > 0 ? ZosFiles_messages_1.ZosFilesMessages.someDownloadsFailed.message : undefined
};
});
}
/**
* Retrieve USS file content and save it in your local workspace.
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} ussFileName - contains the USS file name
* @param {IDownloadSingleOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} USS file name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*/
static ussFile(session_1, ussFileName_1) {
return __awaiter(this, arguments, void 0, function* (session, ussFileName, options = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(ussFileName, ZosFiles_messages_1.ZosFilesMessages.missingUSSFileName.message);
imperative_1.ImperativeExpect.toNotBeEqual(ussFileName, "", ZosFiles_messages_1.ZosFilesMessages.missingUSSFileName.message);
imperative_1.ImperativeExpect.toNotBeEqual(options.record, true, ZosFiles_messages_1.ZosFilesMessages.unsupportedDataType.message);
try {
let destination;
if (options.stream == null) {
destination = options.file || path_1.posix.normalize(path_1.posix.basename(ussFileName));
imperative_1.IO.createDirsSyncFromFilePath(destination);
}
const writeStream = (_a = options.stream) !== null && _a !== void 0 ? _a : imperative_1.IO.createWriteStream(destination);
if (options.attributes) {
options = Object.assign(Object.assign({}, options), this.parseAttributeOptions(ussFileName, options));
if (options.binary)
options.encoding = undefined;
}
// If data type is not defined by user via encoding flag or attributes file, check for USS tags
if (options.binary == null && options.encoding == null) {
yield utilities_1.Utilities.applyTaggedEncoding(session, ussFileName, options);
}
// Get a proper destination for the file to be downloaded
// If the "file" is not provided, we create a folder structure similar to the uss file structure
ussFileName = ZosFilesUtils_1.ZosFilesUtils.sanitizeUssPathForRestCall(ussFileName);
const endpoint = path_1.posix.join(ZosFiles_constants_1.ZosFilesConstants.RESOURCE, ZosFiles_constants_1.ZosFilesConstants.RES_USS_FILES, ussFileName);
const reqHeaders = this.generateHeadersBasedOnOptions(options);
// Use specific options to mimic ZosmfRestClient.getStreamed()
const requestOptions = {
resource: endpoint,
reqHeaders,
responseStream: writeStream,
normalizeResponseNewLines: !options.binary,
task: options.task
};
if (options.range) {
reqHeaders.push({ [core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RECORD_RANGE]: options.range });
}
// If requestor needs etag, add header + get "response" back
if (options.returnEtag) {
requestOptions.reqHeaders.push(core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RETURN_ETAG);
requestOptions.dataToReturn = [ZosmfRestClientProperties_1.CLIENT_PROPERTY.response];
}
const request = yield core_for_zowe_sdk_1.ZosmfRestClient.getExpectFullResponse(session, requestOptions);
// By default, apiResponse is empty when downloading
const apiResponse = {};
// Return Etag in apiResponse, if requested
if (options.returnEtag) {
apiResponse.etag = request.response.headers.etag;
}
return {
success: true,
commandResponse: destination != null ? util.format(ZosFiles_messages_1.ZosFilesMessages.ussFileDownloadedWithDestination.message, destination) :
ZosFiles_messages_1.ZosFilesMessages.ussFileDownloadedSuccessfully.message,
apiResponse
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
});
}
/**
* Retrieve USS file content and save it in your local workspace.
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} ussDirName - contains the USS file name
* @param {IDownloadOptions} [fileOptions={}] - contains the file options to be sent
* @param {IUSSListOptions} [listOptions={}] - contains the uss list options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} USS file name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*/
static ussDir(session_1, ussDirName_1) {
return __awaiter(this, arguments, void 0, function* (session, ussDirName, fileOptions = {}, listOptions = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(ussDirName, ZosFiles_messages_1.ZosFilesMessages.missingUSSDirName.message);
imperative_1.ImperativeExpect.toNotBeEqual(ussDirName.trim(), "", ZosFiles_messages_1.ZosFilesMessages.missingUSSDirName.message);
imperative_1.ImperativeExpect.toNotBeEqual(fileOptions.record, true, ZosFiles_messages_1.ZosFilesMessages.unsupportedDataType.message);
const result = this.emptyDownloadUssDirResult();
const workingDirectory = fileOptions.directory ? fileOptions.directory : process.cwd();
const responses = [];
const downloadTasks = [];
let downloadsInitiated = 0;
let downloadsTotal = 0;
const createPromise = (task) => {
if (task.file) {
return createFilePromise(task);
}
else {
return createDirPromise(task);
}
};
const createFilePromise = (task) => {
var _a, _b;
if (fileOptions.task != null) {
fileOptions.task.statusMessage = "Downloading file: " + task.file;
fileOptions.task.percentComplete = Math.floor(imperative_1.TaskProgress.ONE_HUNDRED_PERCENT * (downloadsInitiated / downloadsTotal));
downloadsInitiated++;
}
// task.options.file is only null for directories, but we may want to fall back to the filename itself (just in case)
if (fs.existsSync((_b = (_a = task.options) === null || _a === void 0 ? void 0 : _a.file) !== null && _b !== void 0 ? _b : task.file) && !fileOptions.overwrite) {
result.skippedExisting.push(task.file);
}
else {
return this.ussFile(session, path_1.posix.join(ussDirName, task.file), task.options).then((downloadResponse) => {
responses.push(downloadResponse);
result.downloaded.push(task.file);
}, (err) => {
result.failedWithErrors[task.file] = err;
if (fileOptions.failFast || fileOptions.failFast === undefined) {
throw new imperative_1.ImperativeError({
msg: `Failed to download ${task.file}`,
causeErrors: err,
additionalDetails: this.buildDownloadUssDirResponse(result, fileOptions)
});
}
});
}
};
const createDirPromise = (task) => {
return fs.promises.mkdir(task.dirName, { recursive: true }).then(() => {
result.downloaded.push(task.dirName);
}, (err) => {
const relDirName = (0, path_1.relative)(workingDirectory, task.dirName);
result.failedWithErrors[relDirName] = err;
if (fileOptions.failFast || fileOptions.failFast === undefined) {
throw new imperative_1.ImperativeError({
msg: `Failed to create directory ${relDirName}`,
causeErrors: err,
additionalDetails: this.buildDownloadUssDirResponse(result, fileOptions)
});
}
});
};
try {
const mutableOptions = Object.assign(Object.assign({}, fileOptions), { task: undefined });
// Populate list options
listOptions = Object.assign({ name: "*" }, listOptions);
// Get the directory listing from z/OSMF
const list = (yield List_1.List.fileList(session, ussDirName, listOptions)).apiResponse.items;
// For each item in the listing...
for (const item of list) {
if (item.name === "." || item.name === ".." || item.name === "..." || !fileOptions.includeHidden && /(^|\/)\./.test(item.name)) {
// If the name is ., .., ..., or a hidden file, ignore it.
continue;
}
else if (item.mode.startsWith("-")) {
// If mode starts with -, the item is a file, download it
if ((_a = fileOptions.attributes) === null || _a === void 0 ? void 0 : _a.fileShouldBeIgnored(item.name)) {
// If .zosattributes says to ignore the file, skip it
continue;
}
mutableOptions.file = (0, path_1.join)(workingDirectory, item.name);
downloadTasks.push({
file: item.name,
options: Object.assign(Object.assign({}, mutableOptions), this.parseAttributeOptions(item.name, fileOptions)),
});
downloadsTotal++;
}
else if (item.mode.startsWith("d")) {
// If mode starts with d, the item is a directory, create it
downloadTasks.push({
dirName: (0, path_1.join)(workingDirectory, item.name),
});
}
// Otherwise, skip it entirely.
}
// Next download the USS files in a pool
const maxConcurrentRequests = fileOptions.maxConcurrentRequests == null ? 1 : fileOptions.maxConcurrentRequests;
if (maxConcurrentRequests === 0) {
yield Promise.all(downloadTasks.map(createPromise));
}
else {
yield (0, core_for_zowe_sdk_1.asyncPool)(maxConcurrentRequests, downloadTasks, createPromise);
}
return {
success: Object.keys(result.failedWithErrors).length === 0,
commandResponse: this.buildDownloadUssDirResponse(result, fileOptions),
apiResponse: responses
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
});
}
/**
* Create an empty download data sets matching result.
* @returns Results object with all lists initialized as empty
*/
static emptyDownloadDsmResult() {
return {
downloaded: [],
failedArchived: [],
failedUnsupported: [],
failedWithErrors: {}
};
}
/**
* Create an empty download data sets matching result.
* @returns Results object with all lists initialized as empty
*/
static emptyDownloadUssDirResult() {
return {
downloaded: [],
skippedExisting: [],
failedWithErrors: {}
};
}
/**
* Build a response string from a download data sets matching result.
* @param result Result object from the download API
* @param options Options passed to the download API
* @returns Response string to print to console
*/
static buildDownloadDsmResponse(result, options = {}) {
var _a;
const failedDsnames = Object.keys(result.failedWithErrors);
const numFailed = result.failedArchived.length + result.failedUnsupported.length + failedDsnames.length;
const responseLines = [];
if (result.downloaded.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.green(`${result.downloaded.length} data set(s) downloaded successfully to `) +
((_a = options.directory) !== null && _a !== void 0 ? _a : "./"));
}
if (numFailed > 0) {
responseLines.push(imperative_1.TextUtils.chalk.red(`${numFailed} data set(s) failed to download:`));
if (result.failedArchived.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.yellow(`${result.failedArchived.length} failed because they are archived`), ...result.failedArchived.map(dsname => ` ${dsname}`));
}
if (result.failedUnsupported.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.yellow(`${result.failedUnsupported.length} failed because they are an unsupported type`), ...result.failedUnsupported.map(dsname => ` ${dsname}`));
}
if (failedDsnames.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.yellow(`${failedDsnames.length} failed because of an uncaught error`), ...failedDsnames.map(dsname => ` ${dsname}`), "", ...Object.values(result.failedWithErrors).map((err) => err.message));
}
if (options.failFast !== false) {
responseLines.push("\nSome data sets may have been skipped because --fail-fast is true.", "To ignore errors and continue downloading, rerun the command with --fail-fast set to false.");
}
}
return responseLines.join("\n") + "\n";
}
/**
* Build a response string from a download ussDir result.
* @param result Result object from the download API
* @param options Options passed to the download API
* @returns Response string to print to console
*/
static buildDownloadUssDirResponse(result, options = {}) {
var _a;
const failedFiles = Object.keys(result.failedWithErrors);
const numFailed = failedFiles.length;
const responseLines = [];
if (result.downloaded.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.green(`${result.downloaded.length} file(s) downloaded successfully to `) +
((_a = options.directory) !== null && _a !== void 0 ? _a : "./"));
}
if (result.skippedExisting.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.yellow(`${result.skippedExisting.length} file(s) skipped because they already exist.`), ...result.skippedExisting.map(filename => ` ${filename}`), "\nRerun the command with --overwrite to download the files listed above.");
}
if (numFailed > 0) {
responseLines.push(imperative_1.TextUtils.chalk.red(`${numFailed} file(s) failed to download:`));
if (failedFiles.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.yellow(`${failedFiles.length} failed because of an uncaught error`), ...failedFiles.map(filename => ` ${filename}`), "", ...Object.values(result.failedWithErrors).map((err) => err.message));
}
if (options.failFast !== false) {
responseLines.push("\nSome files may have been skipped because --fail-fast is true.", "To ignore errors and continue downloading, rerun the command with --fail-fast set to false.");
}
}
return responseLines.join("\n") + "\n";
}
static generateHeadersBasedOnOptions(options) {
const reqHeaders = ZosFilesUtils_1.ZosFilesUtils.generateHeadersBasedOnOptions(options);
if (!options.binary && !options.record) {
if (options.localEncoding) {
reqHeaders.push({ [imperative_1.Headers.CONTENT_TYPE]: options.localEncoding });
}
else {
reqHeaders.push(core_for_zowe_sdk_1.ZosmfHeaders.TEXT_PLAIN);
}
}
return reqHeaders;
}
static parseAttributeOptions(filename, options) {
const newOptions = {};
if (options.attributes != null) {
newOptions.binary = options.attributes.getFileTransferMode(filename, options.binary) === ZosFilesAttributes_1.TransferMode.BINARY;
if (!newOptions.binary) {
newOptions.encoding = options.attributes.getRemoteEncoding(filename);
newOptions.localEncoding = options.attributes.getLocalEncoding(filename);
}
}
return newOptions;
}
}
exports.Download = Download;
//# sourceMappingURL=Download.js.map