@ngageoint/mage.sftp.service
Version:
The SFTP service package is a MAGE server plugin that sends observations to and SFTP location on create and update.
253 lines • 12.6 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SftpController = void 0;
const entities_events_1 = require("@ngageoint/mage.service/lib/entities/events/entities.events");
const entities_observations_1 = require("@ngageoint/mage.service/lib/entities/observations/entities.observations");
const ssh2_sftp_client_1 = __importDefault(require("ssh2-sftp-client"));
const stream_1 = require("stream");
const SFTPPluginConfig_1 = require("../configuration/SFTPPluginConfig");
const entities_format_1 = require("../format/entities.format");
const fs_1 = __importDefault(require("fs"));
const adapters_sftp_mongoose_1 = require("../adapters/adapters.sftp.mongoose");
const adapters_sftp_teams_1 = require("../adapters/adapters.sftp.teams");
const { name: packageName } = require('../../package.json');
/**
* Class used to process observations for SFTP
*/
class SftpController {
/**
* Constructor.
* @param stateRepository The plugins configuration.
* @param eventRepository Used to get all the active events.
* @param observationRepository Used to get new observations.
* @param userRepository Used to get user information.
* @param console Used to log to the console.
*/
constructor(console, { stateRepository, eventRepository, observationRepository, userRepository, attachmentStore }, dbConnection) {
/**
* True if the processor is currently active, false otherwise.
*/
this.isRunning = false;
/**
* SFTP client configuration
*/
this.sftpClient = new ssh2_sftp_client_1.default();
/**
* SFTP plugin configuration
*/
this.configuration = null;
const sftpObservationModel = (0, adapters_sftp_mongoose_1.SftpObservationModel)(dbConnection, `${packageName}/observations`);
const sftpObservationRepository = new adapters_sftp_mongoose_1.MongooseSftpObservationRepository(sftpObservationModel);
const teamRepo = new adapters_sftp_teams_1.MongooseTeamsRepository(dbConnection);
const archiverFactory = new entities_format_1.ArchiverFactory(userRepository, attachmentStore);
this.stateRepository = stateRepository;
this.eventRepository = eventRepository;
this.sftpObservationRepository = sftpObservationRepository;
this.observationRepository = observationRepository;
this.archiveFactory = archiverFactory;
this.console = console;
this.teamRepository = teamRepo;
this.userRepository = userRepository;
}
/**
* Gets the current configuration from the database.
* @returns The current configuration from the database.
*/
getConfiguration() {
return __awaiter(this, void 0, void 0, function* () {
if (this.configuration === null) {
return yield this.stateRepository.get().then((x) => !!x ? x : this.stateRepository.put(SFTPPluginConfig_1.defaultSFTPPluginConfig));
}
else {
return this.configuration;
}
});
}
/**
* Updates new configuration in the state repository.
* @param configuration The new config to put into the state repo.
*/
updateConfiguration(configuration) {
return __awaiter(this, void 0, void 0, function* () {
try {
yield this.stateRepository.put(configuration);
}
catch (err) {
this.console.log(`ERROR: updateConfiguration: ${err}`);
}
});
}
/**
* Starts the processor.
*/
start() {
return __awaiter(this, void 0, void 0, function* () {
this.configuration = yield this.getConfiguration();
if (!this.configuration.enabled) {
return;
}
try {
const sftpKeyFilename = process.env['MAGE_SFTP_KEY_FILE'];
const sftpKeyFile = fs_1.default.readFileSync(sftpKeyFilename);
yield this.sftpClient.connect({
host: this.configuration.sftpClient.host,
username: this.configuration.sftpClient.username,
privateKey: sftpKeyFile
});
this.isRunning = true;
yield this.processAndScheduleNext();
}
catch (e) {
this.console.error("error connecting to sftp endpoint", e);
}
});
}
/**
* Stops the processor.
*/
stop() {
return __awaiter(this, void 0, void 0, function* () {
this.configuration = null;
this.isRunning = false;
yield this.sftpClient.end();
clearTimeout(this.nextTimeout);
});
}
/**
* Processes any new observations and then schedules its next run if it hasn't been stopped.
*/
processAndScheduleNext() {
return __awaiter(this, void 0, void 0, function* () {
const configuration = yield this.getConfiguration();
if (this.isRunning) {
try {
this.console.info('processing new observations');
const events = yield this.eventRepository.findActiveEvents();
for (const attrs of events) {
const event = new entities_events_1.MageEvent(attrs);
yield this.processEvent(event, configuration);
}
}
catch (e) {
this.console.error('sftp error', e);
}
this.scheduleNext(configuration.interval);
}
});
}
/**
* Schedule next run.
* @param interval interval in seconds in which to schedule the next run from now
*/
scheduleNext(interval) {
if (this.isRunning) {
this.nextTimeout = setTimeout(() => { this.processAndScheduleNext(); }, interval * 1000);
}
}
processEvent(event, configuration) {
return __awaiter(this, void 0, void 0, function* () {
const observationRepository = yield this.observationRepository(event.id);
this.console.debug('fetching pending observations for event ' + event.name);
const pending = yield this.sftpObservationRepository.findAllByStatus(event.id, [adapters_sftp_mongoose_1.SftpStatus.PENDING]);
for (const sftpAttrs of pending) {
const observation = yield observationRepository.findById(sftpAttrs.observationId);
if (observation !== null) {
yield this.sftpObservation(observation, event, configuration.archiveFormat, configuration.sftpClient.path, configuration.initiation.timeout);
}
}
const latest = yield this.sftpObservationRepository.findLatest(event.id);
let queryTime = 0;
if (latest !== null) {
const observation = yield observationRepository.findById(latest.observationId);
if (observation !== null) {
queryTime = observation.lastModified.getTime() + 1;
}
}
const page = {
pageSize: configuration.pageSize,
pageIndex: 0
};
this.console.debug('fetching new observations for event ' + event.name);
let { items: observations } = yield observationRepository.findLastModifiedAfter(queryTime, page);
observations = yield this.applyTriggerRule(event, observations, configuration.initiation.rule);
if (observations.length) {
for (const observationAttrs of observations) {
const observation = entities_observations_1.Observation.evaluate(observationAttrs, event);
yield this.sftpObservation(observation, event, configuration.archiveFormat, configuration.sftpClient.path, configuration.initiation.timeout);
}
page.pageIndex = ++page.pageIndex;
}
else {
this.console.debug('no new observations');
}
});
}
applyTriggerRule(event, observations, rule) {
return __awaiter(this, void 0, void 0, function* () {
if (rule === entities_format_1.TriggerRule.Create) {
const filtered = [];
for (const observation of observations) {
const isProcessed = yield this.sftpObservationRepository.isProcessed(event.id, observation.id);
if (!isProcessed) {
filtered.push(observation);
}
}
return filtered;
}
else {
return observations;
}
});
}
sftpObservation(observation, event, format, sftpPath, timeout) {
return __awaiter(this, void 0, void 0, function* () {
const archiver = this.archiveFactory.createArchiver(format);
const result = yield archiver.createArchive(observation, event);
if (result instanceof entities_format_1.ArchiveResult) {
if (result.status === entities_format_1.ArchiveStatus.Complete || (result.status === entities_format_1.ArchiveStatus.Incomplete && (observation.lastModified.getTime() + timeout) > Date.now())) {
this.console.log(`posting status of success`);
try {
const stream = new stream_1.PassThrough();
result.archive.pipe(stream);
// TODO: This will fail if the observation has a .mov file
yield result.archive.finalize();
const teams = yield this.teamRepository.findTeamsByUserId(observation.userId);
// Filter out events from the teams response (bug) and teams that are not in the event
const newTeams = teams.filter((team) => { var _a; return team.teamEventId == null && ((_a = event.teamIds) === null || _a === void 0 ? void 0 : _a.map((teamId) => teamId.toString()).includes(team._id.toString())); });
const teamNames = newTeams.length > 0 ? `${newTeams.map(team => team.name).join('_')}_` : '';
const user = yield this.userRepository.findById(observation.userId || '');
const filename = (`${event.name}_${teamNames}${(user === null || user === void 0 ? void 0 : user.username) || observation.userId}_${observation.id}`);
this.console.info(`Adding sftp observation ${observation.id} to ${sftpPath}/${filename}.zip`);
yield this.sftpClient.put(stream, `${sftpPath}/${filename}.zip`);
yield this.sftpObservationRepository.postStatus(event.id, observation.id, adapters_sftp_mongoose_1.SftpStatus.SUCCESS);
}
catch (error) {
this.console.error(`error uploading observation ${observation.id}`, error);
}
}
else {
this.console.info(`pending observation ${observation.id}`);
yield this.sftpObservationRepository.postStatus(event.id, observation.id, adapters_sftp_mongoose_1.SftpStatus.PENDING);
}
}
else {
this.console.info(`error observation ${observation.id}`, result);
yield this.sftpObservationRepository.postStatus(event.id, observation.id, adapters_sftp_mongoose_1.SftpStatus.FAILED);
}
});
}
}
exports.SftpController = SftpController;
//# sourceMappingURL=controller.js.map