@cumulus/ingest
Version:
Ingest utilities
106 lines • 4.93 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.recursion = void 0;
const path_1 = __importDefault(require("path"));
const log = __importStar(require("@cumulus/common/log"));
/**
* Recur on directory, list all files, and recur into any further directories,
* as specified regex segments allow.
*
* @param {Function} fn - list function
* @param {string} currentPath - current path to list
* @param {Array<string>} segments - path segments
* @param {number} position - current position in the segment list
* @returns {Array<Object>} - filtered contents of directory
*/
async function recurOnDirectory(fn, currentPath, segments, position) {
// interpret the next path segment as a regex for filtering, and
// recursively list everything when we've run out of segments
const filterExpr = segments[position + 1] || '.*';
const filterRegex = new RegExp(filterExpr);
const contents = await fn(currentPath);
let files = [];
for (let ctr = 0; ctr < contents.length; ctr += 1) {
const item = contents[ctr];
// check if item passes filter
if (filterRegex.test(item.name)) {
if (['-', 0].includes(item.type)) {
files.push(item);
}
else if (['d', 1].includes(item.type)) {
const nextDir = path_1.default.normalize(`${currentPath}/${item.name}`);
files = files.concat(
// eslint-disable-next-line no-await-in-loop
await recurOnDirectory(fn, nextDir, segments, position + 1));
}
}
}
return files;
}
/**
* Handles recursion of a FTP/SFTP list operation
* It requests a promisified list function that returns contents of
* a directory on a server, filtering on provided regex segments.
*
* Note that calls to the list function will use either a relative or absolute path, corresponding
* to the `configuredPath` passed into this function. The list function will initially be called
* with '.' for a relative path or '/' for an absolute path. List functions will need to be able to
* normalize or correct these paths as appropriate for their protocol.
*
* Further calls to the list functions will append the current path to that starting path, such
* that all calls will start with either '.' or '/', regardless of additional characters, e.g.
* `fn('./path')` vs. `fn('path')`.
*
* In the case of failure during the recursive list, this function will only apply `path.normalize`
* to the `configuredPath` and then call the list function with the entire normalizedPath.
*
* @param {function} fn - the promisified function for listing a directory
* @param {string} configuredPath - path string configured by operator, which may contain
* regexes for filtering
* @returns {Promise} the promise of an object that has the path is the key and
* list of files as values
*/
async function recursion(fn, configuredPath) {
const normalizedPath = path_1.default.normalize(configuredPath);
const isAbsolutePath = path_1.default.isAbsolute(normalizedPath);
try {
const segments = normalizedPath
.split('/') // split on divider
.filter((segment) => segment.trim() !== ''); // filter out empty strings from split
const startingPath = isAbsolutePath ? '/' : '.';
return await recurOnDirectory(fn, startingPath, segments, -1);
}
catch (error) {
log.error(`Encountered error during recursive list filtering: ${error}`);
log.info('Falling back to unfiltered directory listing...');
return recurOnDirectory(fn, normalizedPath, [], 0);
}
}
exports.recursion = recursion;
//# sourceMappingURL=recursion.js.map