@esri/solution-common
Version:
Provides general helper functions for @esri/solution.js.
348 lines • 19.8 kB
JavaScript
;
/** @license
* Copyright 2021 Esri
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports._sendZipsSeriallyToItem = exports._detemplatizeResources = exports._copyAssociatedFileZips = exports.copyAssociatedFilesByType = exports.copyFilesAsResources = void 0;
const tslib_1 = require("tslib");
const interfaces_1 = require("../interfaces");
const hub_common_1 = require("@esri/hub-common");
const copyDataIntoItem_1 = require("./copyDataIntoItem");
const copyMetadataIntoItem_1 = require("./copyMetadataIntoItem");
const copyResourceIntoZip_1 = require("./copyResourceIntoZip");
const copyZipIntoItem_1 = require("./copyZipIntoItem");
const createCopyResults_1 = require("./createCopyResults");
const generalHelpers_1 = require("../generalHelpers");
const restHelpersGet_1 = require("../restHelpersGet");
const jszip_1 = tslib_1.__importDefault(require("jszip"));
// ------------------------------------------------------------------------------------------------------------------ //
/**
* Copies the files for storing the resources, metadata, and thumbnail of an item or group to a storage item
* with a specified path by collecting files into zip files.
*
* @param files List of item files' URLs and folder/filenames for storing the files
* @param destinationItemId Id of item to receive copy of resource/metadata/thumbnail
* @param destinationAuthentication Credentials for the request to the storage
* @param maxFilesPerZip Number of files to include per zip file; AGO limits zips to 50 files
* @returns A promise which resolves to a list of the result of the copies
*/
function copyFilesAsResources(files, destinationItemId, destinationAuthentication, maxFilesPerZip = 40) {
return new Promise((resolve) => {
const awaitAllItems = [];
const zipInfos = [];
if (files.length > 0) {
// Bundle the resources into chunked zip updates because AGO tends to have problems with
// many updates in a row to the same item: it claims success despite randomly failing.
// Note that AGO imposes a limit of 50 files per zip and a maximum upload file size under
// 50MB, so we break the list of resource file info into chunks below this threshold and
// start a zip for each.
// https://developers.arcgis.com/rest/users-groups-and-items/add-resources.htm
const maxZipSize = 49999000; // bytes
let zipIndex = 0;
let currentZipInfo = _createEmptyZipInfo(zipIndex);
zipInfos.push(currentZipInfo);
let currentZipSize = 0;
files.forEach((file) => {
if (currentZipInfo.filelist.length >= maxFilesPerZip || currentZipSize + file.file.size >= maxZipSize) {
// Create a new zip for the next chunk
zipIndex++;
currentZipInfo = _createEmptyZipInfo(zipIndex);
zipInfos.push(currentZipInfo);
currentZipSize = 0;
}
awaitAllItems.push((0, copyResourceIntoZip_1.copyResourceIntoZip)(file, currentZipInfo));
currentZipSize += file.file.size;
});
}
if (awaitAllItems.length > 0) {
// Wait until the Resource zip file(s) are prepared
void Promise.all(awaitAllItems).then((results) => {
// We have three types of results:
// | fetchedFromSource | copiedToDestination | interpretation |
// +-------------------+---------------------+------------------------------------------------+
// | false | * | could not fetch file from source |
// | true | true | file has been fetched and sent to AGO |
// | true | undefined | file has been fetched and will be sent via zip |
// Filter out copiedToDestination===undefined; we'll get their status when we send their zip
results = results.filter((result) => !(result.fetchedFromSource && typeof result.copiedToDestination === "undefined"));
// Now send the resources to AGO
// eslint-disable-next-line @typescript-eslint/no-floating-promises
_copyAssociatedFileZips(zipInfos, destinationItemId, destinationAuthentication).then((zipResults) => {
resolve(results.concat(zipResults));
});
});
}
else {
// No data, metadata, or resources to send; we're done
resolve([]);
}
});
}
exports.copyFilesAsResources = copyFilesAsResources;
/**
* Copies the files described by a list of full URLs and storage folder/filename combinations for storing
* the resources, metadata, and thumbnail of an item or group to a storage item with different
* handling based on the type of file.
*
* @param fileInfos List of item files' URLs and folder/filenames for storing the files
* @param sourceAuthentication Credentials for the request to the source
* @param sourceItemId Id of item supplying resource/metadata
* @param destinationItemId Id of item to receive copy of resource/metadata/thumbnail
* @param destinationAuthentication Credentials for the request to the storage
* @param template Description of item that will receive files
* @param templateDictionary Hash of facts: org URL, adlib replacements, deferreds for dependencies
* @returns A promise which resolves to a list of the result of the copies
*/
async function copyAssociatedFilesByType(fileInfos, sourceAuthentication, sourceItemId, destinationItemId, destinationAuthentication, template = {}, templateDictionary = {}) {
return new Promise((resolve) => {
let awaitAllItems = [];
let resourceFileInfos = fileInfos;
// eslint-disable-next-line @typescript-eslint/no-floating-promises
awaitAllItems = fileInfos
.filter((fileInfo) => fileInfo.type === interfaces_1.EFileType.Data || fileInfo.type === interfaces_1.EFileType.Metadata)
.map((fileInfo) => {
// Handle Data and Metadata first
switch (fileInfo.type) {
case interfaces_1.EFileType.Data:
// We are updating an item with a zip file, which is written to AGO. If the updated
// item is in a folder, the zip file is moved to the item's folder after being written.
// Without the folder information in the URL, AGO writes the zip to the root folder,
// which causes a conflict if an item with the same data is already in that root folder.
return (0, copyDataIntoItem_1.copyDataIntoItem)(fileInfo, sourceAuthentication, destinationItemId, destinationAuthentication);
case interfaces_1.EFileType.Metadata:
return (0, copyMetadataIntoItem_1.copyMetadataIntoItem)(fileInfo, sourceAuthentication, destinationItemId, destinationAuthentication);
}
});
// Now add in the Resources
resourceFileInfos = fileInfos.filter((fileInfo) => fileInfo.type === interfaces_1.EFileType.Info || fileInfo.type === interfaces_1.EFileType.Resource);
const zipInfos = [];
// eslint-disable-next-line @typescript-eslint/no-floating-promises
const awaitAllResources = new Promise((resolve2) => {
if (resourceFileInfos.length > 0) {
// De-templatize as needed in files before adding them to the zip
// eslint-disable-next-line @typescript-eslint/no-floating-promises
_detemplatizeResources(sourceAuthentication, sourceItemId, template, resourceFileInfos, destinationAuthentication, templateDictionary).then(() => {
// Bundle the resources into chunked zip updates because AGO tends to have problems with
// many updates in a row to the same item: it claims success despite randomly failing.
// Note that AGO imposes a limit of 50 files per zip, so we break the list of resource
// file info into chunks below this threshold and start a zip for each
// https://developers.arcgis.com/rest/users-groups-and-items/add-resources.htm
const chunkedResourceFileInfo = (0, hub_common_1.chunkArray)(resourceFileInfos, 40); // leave a bit of room below threshold
chunkedResourceFileInfo.forEach((chunk, index) => {
// Create a zip for this chunk
const zipInfo = {
filename: `resources${index}.zip`,
zip: new jszip_1.default(),
filelist: [],
};
// eslint-disable-next-line @typescript-eslint/no-floating-promises
awaitAllItems = awaitAllItems.concat(chunk.map((fileInfo) => {
return (0, copyResourceIntoZip_1.copyResourceIntoZipFromInfo)(fileInfo, sourceAuthentication, zipInfo);
}));
zipInfos.push(zipInfo);
});
resolve2(null);
});
}
else {
resolve2(null);
}
});
// Wait until the Resource zip file(s) have been prepared
void awaitAllResources.then(() => {
if (awaitAllItems.length > 0) {
// Wait until all Data and Metadata files have been copied
void Promise.all(awaitAllItems).then((results) => {
// We have three types of results:
// | fetchedFromSource | copiedToDestination | interpretation | |
// +-------------------+---------------------+------------------------------------------------+
// | false | * | could not fetch file from source |
// | true | true | file has been fetched and sent to AGO |
// | true | undefined | file has been fetched and will be sent via zip |
// Filter out copiedToDestination===undefined; we'll get their status when we send their zip
results = results.filter((result) => !(result.fetchedFromSource && typeof result.copiedToDestination === "undefined"));
// Now send the resources to AGO
// eslint-disable-next-line @typescript-eslint/no-floating-promises
_copyAssociatedFileZips(zipInfos, destinationItemId, destinationAuthentication).then((zipResults) => {
resolve(results.concat(zipResults));
});
});
}
else {
// No data, metadata, or resources to send; we're done
resolve([]);
}
});
});
}
exports.copyAssociatedFilesByType = copyAssociatedFilesByType;
/**
* Copies one or more zipfiles to a storage item.
*
* @param zipInfos List of zip files containing files to store
* @param destinationItemId Id of item to receive copy of resource/metadata/thumbnail
* @param destinationAuthentication Credentials for the request to the storage
* @returns A promise which resolves to a list of the result of the copies
* @private
*/
function _copyAssociatedFileZips(zipInfos, destinationItemId, destinationAuthentication) {
return new Promise((resolve) => {
const results = [];
// Filter out empty zips, which can happen when none of the files in the chunk going into a zip
// can be fetched; e.g., the only file is metadata.xml, and the source item doesn't have metadata
const nonEmptyZipInfos = zipInfos.filter((zipInfo) => Object.keys(zipInfo.zip.files).length > 0);
if (nonEmptyZipInfos.length > 0) {
// Send the zip(s) to AGO
void _sendZipsSeriallyToItem(nonEmptyZipInfos, destinationItemId, destinationAuthentication).then((zipResults) => {
resolve(zipResults);
});
}
else {
// No resources to send; we're done
resolve(results);
}
});
}
exports._copyAssociatedFileZips = _copyAssociatedFileZips;
/**
* Creates an empty zip info object.
*
* @param index Index of the zip info object, used as a suffix for the filename
* @returns An empty zip info object
*/
function _createEmptyZipInfo(index) {
return {
filename: `resources${index}.zip`,
zip: new jszip_1.default(),
filelist: [],
};
}
/**
* Replace templatizations in an item's resources
*
* @param sourceAuthentication Credentials for the request to the source
* @param sourceItemId Id of item supplying resource/metadata
* @param itemTemplate Item being created
* @param fileInfos Resources for the item; these resources are modified as needed
* by removing the templatization: the `url` property is replaced by the `file` property
* @param destinationAuthentication Credentials for the request to the storage
* @param templateDictionary Hash of facts: org URL, adlib replacements, deferreds for dependencies
*
* @returns A promise that resolves when all de-templatization has completed
*/
function _detemplatizeResources(sourceAuthentication, sourceItemId, itemTemplate, fileInfos, destinationAuthentication, templateDictionary = {}) {
const synchronizePromises = [];
if (itemTemplate.type === "Vector Tile Service") {
// Get the root.json files
const rootJsonResources = fileInfos.filter((file) => file.filename === "root.json");
const templatizedResourcePath = "{{" + sourceItemId + ".url}}";
const resourcePath = destinationAuthentication.portal + "/content/items/" + itemTemplate.itemId;
const replacer = new RegExp(templatizedResourcePath, "g");
// Templatize the paths in the files that reference the source item id
rootJsonResources.forEach((rootFileResource) => {
// eslint-disable-next-line @typescript-eslint/no-floating-promises
synchronizePromises.push(new Promise((resolve) => {
// Fetch the file
// eslint-disable-next-line @typescript-eslint/no-floating-promises
(0, restHelpersGet_1.getBlobAsFile)(rootFileResource.url, rootFileResource.filename, sourceAuthentication).then((file) => {
// Read the file
// eslint-disable-next-line @typescript-eslint/no-floating-promises
(0, generalHelpers_1.blobToJson)(file).then((fileJson) => {
// Templatize by turning JSON into string, replacing paths with template, and re-JSONing
const updatedFileJson = JSON.parse(JSON.stringify(fileJson).replace(replacer, resourcePath));
// Write the changes back into the file
rootFileResource.file = (0, generalHelpers_1.jsonToFile)(updatedFileJson, rootFileResource.filename);
rootFileResource.url = "";
resolve(null);
});
});
}));
});
}
else if (itemTemplate.type === "Geoprocessing Service") {
const paths = {};
paths[`{{${sourceItemId}.itemId}}`] = itemTemplate.itemId;
itemTemplate.dependencies.forEach((id) => {
paths[`{{${id}.itemId}}`] = templateDictionary[id].itemId;
});
fileInfos.forEach((fileResource) => {
// eslint-disable-next-line @typescript-eslint/no-floating-promises
synchronizePromises.push(new Promise((resolve) => {
// Fetch the file
// eslint-disable-next-line @typescript-eslint/no-floating-promises
(0, restHelpersGet_1.getBlobAsFile)(fileResource.url, fileResource.filename, sourceAuthentication).then((file) => {
// Read the file
// eslint-disable-next-line @typescript-eslint/no-floating-promises
(0, generalHelpers_1.blobToJson)(file).then((fileJson) => {
// DeTemplatize by turning JSON into string, replacing paths with new value, and re-JSONing
let fileString = JSON.stringify(fileJson);
Object.keys(paths).forEach((k) => {
const replacer = new RegExp(k, "g");
fileString = fileString.replace(replacer, paths[k]);
});
const updatedFileJson = JSON.parse(fileString);
// Write the changes back into the file
fileResource.file = (0, generalHelpers_1.jsonToFile)(updatedFileJson, fileResource.filename);
fileResource.url = "";
resolve(null);
});
});
}));
});
}
return Promise.all(synchronizePromises);
}
exports._detemplatizeResources = _detemplatizeResources;
/**
* Copies one or more zipfiles to a storage item in a serial fashion, waiting a bit between sends.
*
* @param zipInfos List of zip files containing files to store
* @param destinationItemId Id of item to receive copy of resource/metadata/thumbnail
* @param destinationAuthentication Credentials for the request to the storage
* @returns A promise which resolves to a list of the result of the copies
*/
function _sendZipsSeriallyToItem(zipInfos, destinationItemId, destinationAuthentication) {
return new Promise((resolve) => {
let allResults = [];
// Remove zip from bottom of list
const zipInfoToSend = zipInfos.pop();
// Send predecessors in list
let sendOthersPromise = Promise.resolve([]);
if (zipInfos.length > 0) {
sendOthersPromise = _sendZipsSeriallyToItem(zipInfos, destinationItemId, destinationAuthentication);
}
void sendOthersPromise
.then((response) => {
allResults = response;
// Stall a little to give AGO time to catch up
return new Promise((resolveSleep) => {
setTimeout(() => resolveSleep(), 1000);
});
})
.then(() => {
// Now send the zip removed from bottom of the input list
return (0, copyZipIntoItem_1.copyZipIntoItem)(zipInfoToSend, destinationItemId, destinationAuthentication);
})
.then((zipResult) => {
// Save the result of copying this zip as a status for each of the files that it contains
zipResult.filelist.forEach((fileInfo) => {
allResults.push((0, createCopyResults_1.createCopyResults)(fileInfo, true, zipResult.copiedToDestination));
});
resolve(allResults);
});
});
}
exports._sendZipsSeriallyToItem = _sendZipsSeriallyToItem;
//# sourceMappingURL=copyAssociatedFiles.js.map