@zowe/zos-files-for-zowe-sdk
Version:
Zowe SDK to interact with files and data sets on z/OS
872 lines • 54.5 kB
JavaScript
"use strict";
/*
* This program and the accompanying materials are made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v20.html
*
* SPDX-License-Identifier: EPL-2.0
*
* Copyright Contributors to the Zowe Project.
*
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Download = void 0;
const imperative_1 = require("@zowe/imperative");
const path_1 = require("path");
const fs = require("fs");
const util = require("util");
const core_for_zowe_sdk_1 = require("@zowe/core-for-zowe-sdk");
const ZosFiles_constants_1 = require("../../constants/ZosFiles.constants");
const ZosFiles_messages_1 = require("../../constants/ZosFiles.messages");
const ZosFilesUtils_1 = require("../../utils/ZosFilesUtils");
const List_1 = require("../list/List");
const ZosmfRestClientProperties_1 = require("../../doc/types/ZosmfRestClientProperties");
const utilities_1 = require("../utilities");
const ZosFilesAttributes_1 = require("../../utils/ZosFilesAttributes");
/**
* This class holds helper functions that are used to download data sets, members and more through the z/OS MF APIs
*/
class Download {
/**
* Retrieve data sets and/or members contents and save them in your local workspace
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} dataSetName - contains the data set name
* @param {IDownloadSingleOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} data set name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*
* @example
* ```typescript
*
* // Download "USER.DATA.SET.PS" to "user/data/set/ps.txt"
* await Download.dataSet(session, "USER.DATA.SET.PS");
*
* // Download "USER.DATA.SET.PDS(MEMBER)" to "user/data/set/pds/member.txt"
* await Download.dataSet(session, "USER.DATA.SET.PDS(MEMBER)");
*
* // Download "USER.DATA.SET" to "./path/to/file.txt"
* await Download.dataSet(session, "USER.DATA.SET", {file: "./path/to/file.txt"});
* ```
*
* @see https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.izua700/IZUHPINFO_API_GetReadDataSet.htm
*/
static dataSet(session_1, dataSetName_1) {
return __awaiter(this, arguments, void 0, function* (session, dataSetName, options = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(dataSetName, ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
imperative_1.ImperativeExpect.toNotBeEqual(dataSetName, "", ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(session);
let destination;
let existedBefore = false;
let downloadStarted = false;
try {
// Format the endpoint to send the request to
let endpoint = ZosFiles_constants_1.ZosFilesConstants.RESOURCE + ZosFiles_constants_1.ZosFilesConstants.RES_DS_FILES;
if (options.volume) {
endpoint += `/-(${options.volume})`;
}
endpoint = imperative_1.EncodeUri.encUriPathForZos(session, endpoint + "/" + dataSetName);
if (options.queryParams) {
endpoint += options.queryParams;
}
imperative_1.Logger.getAppLogger().debug(`Endpoint: ${endpoint}`);
const reqHeaders = this.generateHeadersBasedOnOptions(options);
// Get contents of the data set
let extension = ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION;
if (options.extension != null) {
extension = options.extension;
}
// By default, apiResponse is empty when downloading
const apiResponse = {};
if (options.stream == null) {
// Get a proper destination for the file to be downloaded
// If the "file" is not provided, we create a folder structure similar to the data set name
// Note that the "extension" options do not affect the destination if the "file" options were provided
destination = (() => {
if (options.file) {
return options.file;
}
let generatedFilePath = ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetName);
// Method above lowercased characters.
// In case of preserving original letter case, uppercase all characters.
if (options.preserveOriginalLetterCase) {
generatedFilePath = generatedFilePath.toUpperCase();
}
return generatedFilePath + imperative_1.IO.normalizeExtension(extension);
})();
apiResponse.destination = destination;
// Track if a file at destination already exists before staring the download
existedBefore = imperative_1.IO.existsSync(destination);
// If file exists and we should not overwrite, skip downloading with message
if (existedBefore && !options.overwrite) {
return {
success: true,
commandResponse: util.format(ZosFiles_messages_1.ZosFilesMessages.datasetDownloadSkipped.message, destination),
apiResponse: { destination }
};
}
imperative_1.IO.createDirsSyncFromFilePath(destination);
}
const writeStream = (_a = options.stream) !== null && _a !== void 0 ? _a : imperative_1.IO.createWriteStream(destination);
// Use specific options to mimic ZosmfRestClient.getStreamed()
const requestOptions = {
resource: endpoint,
reqHeaders,
responseStream: writeStream,
normalizeResponseNewLines: !(options.binary || options.record),
task: options.task
};
if (options.range) {
reqHeaders.push({ [core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RECORD_RANGE]: options.range });
}
// If requestor needs etag, add header + get "response" back
if (options.returnEtag) {
requestOptions.reqHeaders.push(core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RETURN_ETAG);
requestOptions.dataToReturn = [ZosmfRestClientProperties_1.CLIENT_PROPERTY.response];
}
downloadStarted = true;
const request = yield core_for_zowe_sdk_1.ZosmfRestClient.getExpectFullResponse(session, requestOptions);
// Return Etag in apiResponse, if requested
if (options.returnEtag) {
apiResponse.etag = request.response.headers.etag;
}
return {
success: true,
commandResponse: destination != null ? util.format(ZosFiles_messages_1.ZosFilesMessages.datasetDownloadedWithDestination.message, destination) :
ZosFiles_messages_1.ZosFilesMessages.datasetDownloadedSuccessfully.message,
apiResponse
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
// Only delete the file if it did not exist before and error occured during download
if (destination != null && downloadStarted && !existedBefore) {
try {
imperative_1.IO.deleteFile(destination);
}
catch (deleteError) {
imperative_1.Logger.getAppLogger().warn(`Failed to clean up partially download file ${destination}: ${deleteError.message}`);
}
}
throw error;
}
});
}
/**
* Retrieve all members from a PDS and save them in your local workspace
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} dataSetName - contains the data set name
* @param {IDownloadOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} data set name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*
* @example
* ```typescript
*
* // Download all members of "USER.DATA.SET.PDS" to "user/data/set/pds/"
* await Download.allMembers(session, "USER.DATA.SET.PDS");
*
* // Download all members of "USER.DATA.SET.PDS" to "./path/to/dir/"
* await Download.allMembers(session, "USER.DATA.SET.PDS", {directory: "./path/to/dir/"});
* ```
*
* @see https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.izua700/IZUHPINFO_API_GetReadDataSet.htm
*/
static allMembers(session_1, dataSetName_1) {
return __awaiter(this, arguments, void 0, function* (session, dataSetName, options = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(dataSetName, ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
imperative_1.ImperativeExpect.toNotBeEqual(dataSetName, "", ZosFiles_messages_1.ZosFilesMessages.missingDatasetName.message);
try {
const response = yield List_1.List.allMembers(session, dataSetName, {
volume: options.volume,
responseTimeout: options.responseTimeout
});
const memberList = (_a = options.memberPatternResponse) !== null && _a !== void 0 ? _a : response.apiResponse.items;
if (memberList.length === 0) {
return {
success: false,
commandResponse: ZosFiles_messages_1.ZosFilesMessages.noMembersFound.message,
apiResponse: response.apiResponse
};
}
// If the "directory" option is provided, use it.
// Otherwise use default directory generated from data set name.
const baseDir = (() => {
if (options.directory) {
// Prevent double slashes
if ((options.directory.endsWith(path_1.posix.sep) || options.directory.endsWith(path_1.win32.sep)) && !imperative_1.IO.isRootDir(options.directory)) {
return options.directory.slice(0, -1);
}
return options.directory;
}
let generatedDirectory = ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetName);
// Method above lowercased characters.
// In case of preserving original letter case, uppercase all characters.
if (options.preserveOriginalLetterCase) {
generatedDirectory = generatedDirectory.toUpperCase();
}
return generatedDirectory;
})();
const downloadErrors = [];
const failedMembers = [];
const skippedMembers = [];
let cancelledCount = 0;
let downloadCancelled = false;
let downloadsInitiated = 0;
let defaultExtension = ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION;
if (options.extension != null) {
defaultExtension = options.extension;
}
/**
* Function that takes a member and turns it into a promise to download said member
* @param mem - an object with a "member" field containing the name of the data set member
*/
const createDownloadPromise = (mem) => {
var _a, _b;
if (downloadCancelled || ((_a = options.abortDownload) === null || _a === void 0 ? void 0 : _a.call(options))) {
downloadCancelled = true;
cancelledCount++;
return Promise.resolve();
}
// update the progress bar if any
if (options.task != null) {
options.task.statusMessage = "Downloading " + mem.member;
options.task.percentComplete = Math.floor(imperative_1.TaskProgress.ONE_HUNDRED_PERCENT *
(downloadsInitiated / memberList.length));
downloadsInitiated++;
}
const fileName = options.preserveOriginalLetterCase ? mem.member : mem.member.toLowerCase();
// Determine extension using extensionMap if provided
let extension = defaultExtension;
if (options.extensionMap != null) {
// Use the member name as the key, lowercased if preserveOriginalLetterCase is false
extension = (_b = options.extensionMap[fileName]) !== null && _b !== void 0 ? _b : extension;
}
// Normalize the extension, remove leading periods
if (extension && extension.startsWith(".")) {
extension = extension.replace(/^\.+/g, "");
}
const memberFilePath = path_1.posix.join(baseDir, fileName + imperative_1.IO.normalizeExtension(extension));
if (!imperative_1.IO.isSubPath(baseDir, memberFilePath) || imperative_1.IO.fileEvaluatesToDir(mem.member)) {
throw new imperative_1.ImperativeError({ msg: "The generated data set file path contains illegal characters." });
}
const memberExistedBefore = imperative_1.IO.existsSync(memberFilePath);
// Check if file exists and should not be overwritten
if (memberExistedBefore && !options.overwrite) {
skippedMembers.push(fileName);
return Promise.resolve();
}
return this.dataSet(session, `${dataSetName}(${mem.member})`, {
volume: options.volume,
file: memberFilePath,
binary: options.binary,
record: options.record,
encoding: options.encoding,
responseTimeout: options.responseTimeout,
overwrite: options.overwrite,
}).catch((err) => {
downloadErrors.push(err);
failedMembers.push(fileName);
// Only delete the file if it didn't exist before the download attempt
if (!memberExistedBefore) {
try {
imperative_1.IO.deleteFile(memberFilePath);
}
catch (deleteError) {
imperative_1.Logger.getAppLogger().warn(`Failed to clean up partially download file ${memberFilePath}: ${deleteError.message}`);
}
}
// If we should fail fast, rethrow error
if (options.failFast || options.failFast === undefined) {
throw err;
}
});
};
const maxConcurrentRequests = options.maxConcurrentRequests == null ? 1 : options.maxConcurrentRequests;
if (maxConcurrentRequests === 0) {
yield Promise.all(memberList.map(createDownloadPromise));
}
else {
yield (0, core_for_zowe_sdk_1.asyncPool)(maxConcurrentRequests, memberList, createDownloadPromise);
}
if (downloadCancelled && options.task != null) {
options.task.statusMessage = "Operation cancelled";
}
// Handle failed downloads if no errors were thrown yet
if (downloadErrors.length > 0) {
throw new imperative_1.ImperativeError({
msg: ZosFiles_messages_1.ZosFilesMessages.memberDownloadFailed.message + failedMembers.join("\n") + "\n\n" +
downloadErrors.map((err) => err.message).join("\n"),
causeErrors: downloadErrors,
additionalDetails: failedMembers.join("\n")
});
}
const downloadedCount = memberList.length - skippedMembers.length - failedMembers.length - cancelledCount;
let responseMessage = "";
if (downloadedCount > 0) {
responseMessage = util.format(ZosFiles_messages_1.ZosFilesMessages.memberCountDownloadedWithDestination.message, downloadedCount, baseDir);
}
if (skippedMembers.length > 0) {
if (downloadedCount > 0) {
responseMessage += "\n";
}
responseMessage += util.format(ZosFiles_messages_1.ZosFilesMessages.memberDownloadSkipped.message, skippedMembers.length);
}
if (downloadCancelled) {
responseMessage = "The download was cancelled.\n" + responseMessage;
}
const downloadResult = {
downloaded: downloadedCount,
skipped: skippedMembers.length,
failed: failedMembers.length,
total: memberList.length,
skippedMembers,
failedMembers
};
return {
success: !downloadCancelled,
commandResponse: responseMessage,
apiResponse: Object.assign(Object.assign({}, response.apiResponse), { downloadResult, destination: baseDir })
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
});
}
/**
* Download a list of data sets to local files
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {IZosmfListResponse[]} dataSetObjs - contains data set objects returned by z/OSMF List API
* @param {IDownloadOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} data set name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*
* @example
* ```typescript
*
* // Download a list of "PS" and "PO" datasets to the directory "./path/to/dir/"
* await Download.allDataSets(session, [
* { dsname: "USER.DATA.SET.PS", dsorg: "PS" },
* { dsname: "USER.DATA.SET.PDS", dsorg: "PO" }
* ], {directory: "./path/to/dir/"});
* ```
*
* @see https://www.ibm.com/support/knowledgecenter/SSLTBW_2.2.0/com.ibm.zos.v2r2.izua700/IZUHPINFO_API_GetReadDataSet.htm
*/
static allDataSets(session_1, dataSetObjs_1) {
return __awaiter(this, arguments, void 0, function* (session, dataSetObjs, options = {}) {
var _a, _b, _c, _d, _e, _f;
imperative_1.ImperativeExpect.toNotBeEqual(dataSetObjs.length, 0, ZosFiles_messages_1.ZosFilesMessages.missingDataSets.message);
const result = this.emptyDownloadDsmResult();
const zosmfResponses = [...dataSetObjs];
let downloadCancelled = false;
try {
// Download data sets
const poDownloadTasks = [];
const psDownloadTasks = [];
const mutableOptions = Object.assign(Object.assign({}, options), { task: undefined });
for (const dataSetObj of zosmfResponses) {
if (imperative_1.IO.fileEvaluatesToDir(dataSetObj.dsname)) {
throw new imperative_1.ImperativeError({ msg: "The data set name contains illegal characters." });
}
let llq = dataSetObj.dsname.substring(dataSetObj.dsname.lastIndexOf(".") + 1, dataSetObj.dsname.length);
if (!options.preserveOriginalLetterCase) {
llq = llq.toLowerCase();
}
if (options.extensionMap != null) {
mutableOptions.extension = (_a = options.extensionMap[llq]) !== null && _a !== void 0 ? _a : options.extension;
}
// Normalize the extension, remove leading periods
if (mutableOptions.extension && mutableOptions.extension.startsWith(".")) {
mutableOptions.extension = mutableOptions.extension.replace(/^\.+/g, "");
}
if (options.directory == null) {
if ((_b = dataSetObj.dsorg) === null || _b === void 0 ? void 0 : _b.startsWith("PO")) {
mutableOptions.directory = ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetObj.dsname);
if (options.preserveOriginalLetterCase) {
mutableOptions.directory = mutableOptions.directory.toUpperCase();
}
}
else {
mutableOptions.file = `${dataSetObj.dsname}.` +
`${(_c = mutableOptions.extension) !== null && _c !== void 0 ? _c : ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION}`;
if (!options.preserveOriginalLetterCase) {
mutableOptions.file = mutableOptions.file.toLowerCase();
}
mutableOptions.directory = undefined;
mutableOptions.extension = undefined;
}
}
else if ((_d = dataSetObj.dsorg) === null || _d === void 0 ? void 0 : _d.startsWith("PO")) {
let generatedDir = ZosFilesUtils_1.ZosFilesUtils.getDirsFromDataSet(dataSetObj.dsname);
if (options.preserveOriginalLetterCase) {
generatedDir = generatedDir.toUpperCase();
}
mutableOptions.directory = `${mutableOptions.directory}/${generatedDir}`;
}
else {
mutableOptions.file = `${dataSetObj.dsname}.${(_e = mutableOptions.extension) !== null && _e !== void 0 ? _e : ZosFilesUtils_1.ZosFilesUtils.DEFAULT_FILE_EXTENSION}`;
if (!options.preserveOriginalLetterCase) {
mutableOptions.file = mutableOptions.file.toLowerCase();
}
mutableOptions.file = `${mutableOptions.directory}/${mutableOptions.file}`;
mutableOptions.directory = undefined;
mutableOptions.extension = undefined;
}
if (dataSetObj.error != null) {
result.failedWithErrors[dataSetObj.dsname] = dataSetObj.error;
}
else if (dataSetObj.dsorg == null) {
dataSetObj.status = `Skipped: Archived data set or alias - type ${dataSetObj.vol}.`;
result.failedArchived.push(dataSetObj.dsname);
}
else if (dataSetObj.dsorg === "PS") {
const targetFile = mutableOptions.file;
if (targetFile && imperative_1.IO.existsSync(targetFile) && !options.overwrite) {
dataSetObj.status = `Skipped: File already exists - ${targetFile}`;
result.skippedExisting = result.skippedExisting || [];
result.skippedExisting.push(dataSetObj.dsname);
}
else {
psDownloadTasks.push({
handler: Download.dataSet.bind(this),
dsname: dataSetObj.dsname,
options: Object.assign(Object.assign({}, mutableOptions), { overwrite: true }),
onSuccess: (downloadResponse) => {
dataSetObj.status = downloadResponse.commandResponse;
}
});
}
}
else if (dataSetObj.dsorg.startsWith("PO")) {
poDownloadTasks.push({
handler: Download.allMembers.bind(this),
dsname: dataSetObj.dsname,
options: Object.assign(Object.assign({}, mutableOptions), { overwrite: options.overwrite }),
onSuccess: (downloadResponse, options) => {
dataSetObj.status = downloadResponse.commandResponse;
const listMembers = downloadResponse.apiResponse.items.map((item) => ` ${item.member}`);
if (listMembers.length === 0) { // Create directory for empty PO data set
imperative_1.IO.createDirsSyncFromFilePath(options.directory);
}
else {
dataSetObj.status += `\nMembers:${listMembers}`;
}
}
});
}
else {
dataSetObj.status = `Skipped: Unsupported data set - type ${dataSetObj.dsorg}.`;
result.failedUnsupported.push(dataSetObj.dsname);
}
mutableOptions.directory = options.directory;
}
// If we should fail fast, throw error
if ((result.failedArchived.length > 0 || result.failedUnsupported.length > 0 ||
Object.keys(result.failedWithErrors).length > 0) && options.failFast !== false) {
throw new imperative_1.ImperativeError({
msg: ZosFiles_messages_1.ZosFilesMessages.failedToDownloadDataSets.message,
additionalDetails: this.buildDownloadDsmResponse(result, options)
});
}
let downloadsInitiated = 0;
const createDownloadPromise = (task) => {
var _a;
if (downloadCancelled || ((_a = options.abortDownload) === null || _a === void 0 ? void 0 : _a.call(options))) {
downloadCancelled = true;
return Promise.resolve();
}
if (options.task != null) {
options.task.statusMessage = "Downloading data set " + task.dsname;
options.task.percentComplete = Math.floor(imperative_1.TaskProgress.ONE_HUNDRED_PERCENT *
(downloadsInitiated / (poDownloadTasks.length + psDownloadTasks.length)));
downloadsInitiated++;
}
return task.handler(session, task.dsname, task.options).then((downloadResponse) => {
var _a;
const downloadResult = (_a = downloadResponse.apiResponse) === null || _a === void 0 ? void 0 : _a.downloadResult;
if (downloadResult) {
// This is a PO dataset as it has a download result
if (downloadResult.downloaded > 0) {
result.downloaded.push(task.dsname);
}
else if (downloadResult.skipped > 0 && downloadResult.downloaded === 0) {
result.skippedExisting = result.skippedExisting || [];
result.skippedExisting.push(task.dsname);
}
else {
result.downloaded.push(task.dsname);
}
}
else {
// This is a PS dataset has there is no download result
result.downloaded.push(task.dsname);
}
task.onSuccess(downloadResponse, task.options);
}, (err) => {
result.failedWithErrors[task.dsname] = err;
// If we should fail fast, rethrow error
if (options.failFast || options.failFast === undefined) {
throw new imperative_1.ImperativeError({
msg: `Failed to download ${task.dsname}`,
causeErrors: err,
additionalDetails: this.buildDownloadDsmResponse(result, options)
});
}
});
};
// First download the partitioned data sets
// We execute the promises sequentially to make sure that
// we do not exceed `--mcr` when downloading multiple members
for (const task of poDownloadTasks) {
if (downloadCancelled || ((_f = options.abortDownload) === null || _f === void 0 ? void 0 : _f.call(options))) {
downloadCancelled = true;
break;
}
yield createDownloadPromise(task);
}
// Next download the sequential data sets in a pool
const maxConcurrentRequests = options.maxConcurrentRequests == null ? 1 : options.maxConcurrentRequests;
if (maxConcurrentRequests === 0) {
yield Promise.all(psDownloadTasks.map(createDownloadPromise));
}
else {
yield (0, core_for_zowe_sdk_1.asyncPool)(maxConcurrentRequests, psDownloadTasks, createDownloadPromise);
}
if (downloadCancelled && options.task != null) {
options.task.statusMessage = "Operation cancelled";
}
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
// Handle failed downloads if no errors were thrown yet
if (!downloadCancelled && Object.keys(result.failedWithErrors).length > 0) {
throw new imperative_1.ImperativeError({
msg: ZosFiles_messages_1.ZosFilesMessages.datasetDownloadFailed.message + Object.keys(result.failedWithErrors).join("\n"),
causeErrors: Object.values(result.failedWithErrors),
additionalDetails: this.buildDownloadDsmResponse(result, options)
});
}
const numFailed = result.failedArchived.length + result.failedUnsupported.length + Object.keys(result.failedWithErrors).length;
const commandResponse = downloadCancelled
? "The download was cancelled.\n" + this.buildDownloadDsmResponse(result, options)
: this.buildDownloadDsmResponse(result, options);
return {
success: !downloadCancelled && numFailed === 0,
commandResponse,
apiResponse: zosmfResponses,
errorMessage: numFailed > 0 ? ZosFiles_messages_1.ZosFilesMessages.someDownloadsFailed.message : undefined
};
});
}
/**
* Retrieve USS file content and save it in your local workspace.
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} ussFileName - contains the USS file name
* @param {IDownloadSingleOptions} [options={}] - contains the options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} USS file name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*/
static ussFile(session_1, ussFileName_1) {
return __awaiter(this, arguments, void 0, function* (session, ussFileName, options = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(ussFileName, ZosFiles_messages_1.ZosFilesMessages.missingUSSFileName.message);
imperative_1.ImperativeExpect.toNotBeEqual(ussFileName, "", ZosFiles_messages_1.ZosFilesMessages.missingUSSFileName.message);
imperative_1.ImperativeExpect.toNotBeEqual(options.record, true, ZosFiles_messages_1.ZosFilesMessages.unsupportedDataType.message);
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(session);
try {
let destination;
// By default, apiResponse is empty when downloading
const apiResponse = {};
if (options.stream == null) {
destination = options.file || path_1.posix.normalize(path_1.posix.basename(ussFileName));
if (imperative_1.IO.existsSync(destination) && !options.overwrite) {
return {
success: true,
commandResponse: util.format(ZosFiles_messages_1.ZosFilesMessages.ussFileDownloadSkipped.message, destination),
apiResponse: {}
};
}
imperative_1.IO.createDirsSyncFromFilePath(destination);
apiResponse.destination = destination;
}
const writeStream = (_a = options.stream) !== null && _a !== void 0 ? _a : imperative_1.IO.createWriteStream(destination);
if (options.attributes) {
options = Object.assign(Object.assign({}, options), this.parseAttributeOptions(ussFileName, options));
if (options.binary)
options.encoding = undefined;
}
// If data type is not defined by user via encoding flag or attributes file, check for USS tags
if (options.binary == null && options.encoding == null) {
yield utilities_1.Utilities.applyTaggedEncoding(session, ussFileName, options);
}
// Get a proper destination for the file to be downloaded
// If the "file" is not provided, we create a folder structure similar to the uss file structure
const endpoint = imperative_1.EncodeUri.encUriPathForUss(session, ZosFiles_constants_1.ZosFilesConstants.RESOURCE + ZosFiles_constants_1.ZosFilesConstants.RES_USS_FILES + "/" + ussFileName);
const reqHeaders = this.generateHeadersBasedOnOptions(options);
// Use specific options to mimic ZosmfRestClient.getStreamed()
const requestOptions = {
resource: endpoint,
reqHeaders,
responseStream: writeStream,
normalizeResponseNewLines: !options.binary,
task: options.task
};
if (options.range) {
reqHeaders.push({ [core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RECORD_RANGE]: options.range });
}
// If requestor needs etag, add header + get "response" back
if (options.returnEtag) {
requestOptions.reqHeaders.push(core_for_zowe_sdk_1.ZosmfHeaders.X_IBM_RETURN_ETAG);
requestOptions.dataToReturn = [ZosmfRestClientProperties_1.CLIENT_PROPERTY.response];
}
const request = yield core_for_zowe_sdk_1.ZosmfRestClient.getExpectFullResponse(session, requestOptions);
// Return Etag in apiResponse, if requested
if (options.returnEtag) {
apiResponse.etag = request.response.headers.etag;
}
return {
success: true,
commandResponse: destination != null ? util.format(ZosFiles_messages_1.ZosFilesMessages.ussFileDownloadedWithDestination.message, destination) :
ZosFiles_messages_1.ZosFilesMessages.ussFileDownloadedSuccessfully.message,
apiResponse
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
});
}
/**
* Retrieve USS file content and save it in your local workspace.
*
* @param {AbstractSession} session - z/OS MF connection info
* @param {string} ussDirName - contains the USS file name
* @param {IDownloadOptions} [fileOptions={}] - contains the file options to be sent
* @param {IUSSListOptions} [listOptions={}] - contains the uss list options to be sent
*
* @returns {Promise<IZosFilesResponse>} A response indicating the outcome of the API
*
* @throws {ImperativeError} USS file name must be set
* @throws {Error} When the {@link ZosmfRestClient} throws an error
*/
static ussDir(session_1, ussDirName_1) {
return __awaiter(this, arguments, void 0, function* (session, ussDirName, fileOptions = {}, listOptions = {}) {
var _a;
// required
imperative_1.ImperativeExpect.toNotBeNullOrUndefined(ussDirName, ZosFiles_messages_1.ZosFilesMessages.missingUSSDirName.message);
imperative_1.ImperativeExpect.toNotBeEqual(ussDirName.trim(), "", ZosFiles_messages_1.ZosFilesMessages.missingUSSDirName.message);
imperative_1.ImperativeExpect.toNotBeEqual(fileOptions.record, true, ZosFiles_messages_1.ZosFilesMessages.unsupportedDataType.message);
const result = this.emptyDownloadUssDirResult();
const workingDirectory = fileOptions.directory ? fileOptions.directory : process.cwd();
const responses = [];
const downloadTasks = [];
let downloadCancelled = false;
let downloadsInitiated = 0;
let downloadsTotal = 0;
const createPromise = (task) => {
if (task.file) {
return createFilePromise(task);
}
else {
return createDirPromise(task);
}
};
const createFilePromise = (task) => {
var _a, _b, _c;
if (downloadCancelled || ((_a = fileOptions.abortDownload) === null || _a === void 0 ? void 0 : _a.call(fileOptions))) {
downloadCancelled = true;
return Promise.resolve();
}
if (fileOptions.task != null) {
fileOptions.task.statusMessage = "Downloading file: " + task.file;
fileOptions.task.percentComplete = Math.floor(imperative_1.TaskProgress.ONE_HUNDRED_PERCENT * (downloadsInitiated / downloadsTotal));
downloadsInitiated++;
}
// task.options.file is only null for directories, but we may want to fall back to the filename itself (just in case)
if (imperative_1.IO.existsSync((_c = (_b = task.options) === null || _b === void 0 ? void 0 : _b.file) !== null && _c !== void 0 ? _c : task.file) && !fileOptions.overwrite) {
result.skippedExisting.push(task.file);
}
else {
return this.ussFile(session, path_1.posix.join(ussDirName, task.file), task.options).then((downloadResponse) => {
responses.push(downloadResponse);
result.downloaded.push(task.file);
}, (err) => {
result.failedWithErrors[task.file] = err;
if (fileOptions.failFast || fileOptions.failFast === undefined) {
throw new imperative_1.ImperativeError({
msg: `Failed to download ${task.file}`,
causeErrors: err,
additionalDetails: this.buildDownloadUssDirResponse(result, fileOptions)
});
}
});
}
};
const createDirPromise = (task) => {
var _a;
if (downloadCancelled || ((_a = fileOptions.abortDownload) === null || _a === void 0 ? void 0 : _a.call(fileOptions))) {
downloadCancelled = true;
return Promise.resolve();
}
return fs.promises.mkdir(task.dirName, { recursive: true }).then(() => {
result.downloaded.push(task.dirName);
}, (err) => {
const relDirName = (0, path_1.relative)(workingDirectory, task.dirName);
result.failedWithErrors[relDirName] = err;
if (fileOptions.failFast || fileOptions.failFast === undefined) {
throw new imperative_1.ImperativeError({
msg: `Failed to create directory ${relDirName}`,
causeErrors: err,
additionalDetails: this.buildDownloadUssDirResponse(result, fileOptions)
});
}
});
};
try {
const mutableOptions = Object.assign(Object.assign({}, fileOptions), { task: undefined });
// Populate list options
listOptions = Object.assign({ name: "*" }, listOptions);
// Get the directory listing from z/OSMF
const list = (yield List_1.List.fileList(session, ussDirName, listOptions)).apiResponse.items;
// For each item in the listing...
for (const item of list) {
if (item.name === "." || item.name === ".." || item.name === "..." || !fileOptions.includeHidden && /(^|\/)\./.test(item.name)) {
// If the name is ., .., ..., or a hidden file, ignore it.
continue;
}
else if (item.mode.startsWith("-")) {
// If mode starts with -, the item is a file, download it
if ((_a = fileOptions.attributes) === null || _a === void 0 ? void 0 : _a.fileShouldBeIgnored(item.name)) {
// If .zosattributes says to ignore the file, skip it
continue;
}
if (imperative_1.IO.containsBacktrack(item.name) ||
!imperative_1.IO.isSubPath(workingDirectory, (0, path_1.join)(workingDirectory, item.name))) {
throw new imperative_1.ImperativeError({ msg: "The generated file path contains illegal backtracking." });
}
mutableOptions.file = (0, path_1.join)(workingDirectory, item.name);
downloadTasks.push({
file: item.name,
options: Object.assign(Object.assign({}, mutableOptions), this.parseAttributeOptions(item.name, fileOptions)),
});
downloadsTotal++;
}
else if (item.mode.startsWith("d")) {
if (imperative_1.IO.containsBacktrack(item.name) ||
!imperative_1.IO.isSubPath(workingDirectory, (0, path_1.join)(workingDirectory, item.name))) {
throw new imperative_1.ImperativeError({ msg: "The generated file path contains illegal backtracking." });
}
// If mode starts with d, the item is a directory, create it
downloadTasks.push({
dirName: (0, path_1.join)(workingDirectory, item.name),
});
}
// Otherwise, skip it entirely.
}
// Next download the USS files in a pool
const maxConcurrentRequests = fileOptions.maxConcurrentRequests == null ? 1 : fileOptions.maxConcurrentRequests;
if (maxConcurrentRequests === 0) {
yield Promise.all(downloadTasks.map(createPromise));
}
else {
yield (0, core_for_zowe_sdk_1.asyncPool)(maxConcurrentRequests, downloadTasks, createPromise);
}
if (downloadCancelled && fileOptions.task != null) {
fileOptions.task.statusMessage = "Operation cancelled";
}
const commandResponse = downloadCancelled
? "The download was cancelled.\n" + this.buildDownloadUssDirResponse(result, fileOptions)
: this.buildDownloadUssDirResponse(result, fileOptions);
result.totalCount = downloadsTotal;
// Because apiResponse was never originally being returned as an object,
// this is required to add the downloadResult to the apiResponse without breaking changes to consumers
// TODO: In next major release, make apiResponse consistently an object in the future and remove this workaround
const apiResponse = responses;
apiResponse.downloadResult = result;
return {
success: !downloadCancelled && Object.keys(result.failedWithErrors).length === 0,
commandResponse,
apiResponse,
};
}
catch (error) {
imperative_1.Logger.getAppLogger().error(error);
throw error;
}
});
}
/**
* Create an empty download data sets matching result.
* @returns Results object with all lists initialized as empty
*/
static emptyDownloadDsmResult() {
return {
downloaded: [],
failedArchived: [],
failedUnsupported: [],
failedWithErrors: {}
};
}
/**
* Create an empty download data sets matching result.
* @returns Results object with all lists initialized as empty
*/
static emptyDownloadUssDirResult() {
return {
downloaded: [],
skippedExisting: [],
failedWithErrors: {}
};
}
/**
* Build a response string from a download data sets matching result.
* @param result Result object from the download API
* @param options Options passed to the download API
* @returns Response string to print to console
*/
static buildDownloadDsmResponse(result, options = {}) {
var _a;
const failedDsnames = Object.keys(result.failedWithErrors);
const numFailed = result.failedArchived.length + result.failedUnsupported.length + failedDsnames.length;
const responseLines = [];
if (result.downloaded.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.green(`${result.downloaded.length} data set(s) downloaded successfully to `) +
((_a = options.directory) !== null && _a !== void 0 ? _a : "./"), ...result.downloaded.map(dsname => ` ${dsname}`));
}
if (result.skippedExisting && result.skippedExisting.length > 0) {
responseLines.push(imperative_1.TextUtils.chalk.yellow(`${re