@ngageoint/mage.image.service
Version:
Orient images attached to MAGE observations according to EXIF meta-data and generate configurable size thumbnails.
911 lines • 53 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __rest = (this && this.__rest) || function (s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
};
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var g = generator.apply(thisArg, _arguments || []), i, q = [];
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
function fulfill(value) { resume("next", value); }
function reject(value) { resume("throw", value); }
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const processor_1 = require("./processor");
const entities_events_1 = require("@ngageoint/mage.service/lib/entities/events/entities.events");
const entities_observations_1 = require("@ngageoint/mage.service/lib/entities/observations/entities.observations");
const stream_1 = __importDefault(require("stream"));
const util_1 = __importDefault(require("util"));
const util_spec_1 = require("./util.spec");
const entities_events_forms_1 = require("@ngageoint/mage.service/lib/entities/events/entities.events.forms");
const lodash_1 = __importDefault(require("lodash"));
function minutes(x) {
return 1000 * 60 * x;
}
function minutesAgo(x) {
return Date.now() - minutes(x);
}
function makeEvent(id) {
return {
id,
acl: {},
feedIds: [],
forms: [
{
id: 1,
archived: false,
color: 'lavender',
fields: [
{
id: 1,
name: 'field1',
required: false,
title: 'Attachments',
type: entities_events_forms_1.FormFieldType.Attachment,
}
],
name: 'Image Plugin Test',
userFields: []
}
],
layerIds: [],
name: `Event ${id}`,
style: {},
};
}
function makeObservation(id, eventId, ageInMinutes) {
const createdAt = typeof ageInMinutes === 'number' ? new Date(Date.now() - ageInMinutes * 60 * 1000) : new Date();
return {
id,
eventId,
createdAt,
lastModified: createdAt,
type: 'Feature',
geometry: { type: 'Point', coordinates: [100, 40] },
properties: {
timestamp: new Date(),
forms: []
},
attachments: [],
states: [],
favoriteUserIds: [],
};
}
function sameAsObservationWithoutDates(expected) {
const _a = (0, entities_observations_1.copyObservationAttrs)(expected), { createdAt, lastModified } = _a, expectedAttrs = __rest(_a, ["createdAt", "lastModified"]);
expectedAttrs.attachments.forEach(x => delete x.lastModified);
return {
asymmetricMatch(actual, matchersUtil) {
const _a = (0, entities_observations_1.copyObservationAttrs)(actual), { createdAt, lastModified } = _a, actualAttrs = __rest(_a, ["createdAt", "lastModified"]);
actualAttrs.attachments.forEach(x => delete x.lastModified);
return actual instanceof entities_observations_1.Observation && matchersUtil.equals(expectedAttrs, actualAttrs);
},
jasmineToString(prettyPrint) {
return `<an Observation instance equivalent to ${prettyPrint(expectedAttrs)}>`;
}
};
}
/**
* Return `ObservationAttrs` with the given attachments array, and populated
* with the form entries that the attachments reference. The `lastModified`
* timestamp on the observation will be the latest of the given attachments.
* @param id
* @param eventId
* @param attachments
* @returns
*/
function observationWithAttachments(id, event, attachments) {
return entities_observations_1.Observation.evaluate(Object.assign(Object.assign({}, makeObservation(id, event.id)), { lastModified: attachments.reduce((latestTimestamp, attachment) => {
var _a;
return (((_a = attachment.lastModified) === null || _a === void 0 ? void 0 : _a.getTime()) || 0) > latestTimestamp.getTime() ? new Date(attachment.lastModified) : latestTimestamp;
}, new Date(0)), properties: {
timestamp: new Date(),
forms: attachments.reduce((formEntries, attachment) => {
const attachmentFormEntry = formEntries.find(x => x.id === attachment.observationFormId);
if (!attachmentFormEntry) {
return [
...formEntries,
{
id: attachment.observationFormId,
formId: 1
}
];
}
return formEntries;
}, [])
}, attachments }), event);
}
const asyncIterableOf = (items) => {
return {
[Symbol.asyncIterator]() {
return __asyncGenerator(this, arguments, function* _a() {
for (const item of items) {
yield yield __await(Promise.resolve(item));
}
});
}
};
};
function closeTo(target, delta) {
return {
asymmetricMatch(other) {
return Math.abs(target - other) <= delta;
},
jasmineToString(prettyPrint) {
return `a number within ${delta} of ${target}`;
}
};
}
/**
* Within 100 milliseconds of `Date.now()`
*/
const closeToNow = () => closeTo(Date.now(), 100);
describe('processing interval', () => {
let event1;
let event2;
let allEvents;
let eventRepo;
let observationRepos;
let attachmentStore;
let imageService;
const observationRepoForEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
const repo = observationRepos.get(event);
if (repo) {
return repo;
}
throw new Error(`no observation repository for event ${event}`);
});
beforeEach(() => {
event1 = new entities_events_1.MageEvent(makeEvent(1));
event2 = new entities_events_1.MageEvent(makeEvent(2));
allEvents = new Map()
.set(event1.id, event1)
.set(event2.id, event2);
eventRepo = jasmine.createSpyObj('eventRepo', ['findActiveEvents']);
eventRepo.findActiveEvents.and.resolveTo(Array.from(allEvents.values()).map(entities_events_1.copyMageEventAttrs));
observationRepos = Array.from(allEvents.values()).reduce((repos, event) => {
return repos.set(event.id, jasmine.createSpyObj(`observationRepo-${event.id}`, ['findById', 'patchAttachment', 'save']));
}, new Map());
attachmentStore = jasmine.createSpyObj('attachmentStore', ['readContent', 'saveContent', 'readThumbnailContent', 'saveThumbnailContent', 'stagePendingContent']);
imageService = jasmine.createSpyObj('imageService', ['autoOrient', 'scaleToDimension']);
});
describe('orient phase', () => {
it('orients the attachment image and produces an attachment patch', () => __awaiter(void 0, void 0, void 0, function* () {
const att = Object.freeze({
id: '1.123.1',
observationFormId: 'form1',
fieldName: 'field1',
oriented: false,
thumbnails: [],
contentType: 'image/jpeg',
name: 'test1.jpeg',
size: 320000,
contentLocator: String(Date.now())
});
const originalContent = Buffer.from('sniugnep fo otohp');
const originalConstentStream = stream_1.default.Readable.from(originalContent);
const stagedContent = new entities_observations_1.StagedAttachmentContent('stage1', new util_spec_1.BufferWriteable());
const obsBefore = observationWithAttachments('1.123', event1, [att]);
const obsRepo = observationRepos.get(event1.id);
attachmentStore.readContent.and.resolveTo(originalConstentStream);
attachmentStore.stagePendingContent.and.resolveTo(stagedContent);
attachmentStore.saveContent.and.resolveTo({ size: 321321, contentLocator: att.contentLocator });
imageService.autoOrient.and.callFake((source, dest) => __awaiter(void 0, void 0, void 0, function* () {
const oriented = new util_spec_1.BufferWriteable();
yield util_1.default.promisify(stream_1.default.pipeline)(source.bytes, oriented);
yield util_1.default.promisify(stream_1.default.pipeline)(stream_1.default.Readable.from(oriented.content.reverse()), dest);
return {
mediaType: 'image/jpeg',
sizeInBytes: 321321,
dimensions: { width: 1000, height: 1200 },
};
}));
const oriented = yield (0, processor_1.orientAttachmentImage)(obsBefore, att.id, imageService, attachmentStore, console);
const orientedContent = stagedContent.tempLocation;
expect(oriented.patch).toEqual({ contentType: 'image/jpeg', size: 321321, width: 1000, height: 1200, oriented: true, contentLocator: att.contentLocator });
expect(orientedContent.content).toEqual(Buffer.from('photo of penguins'));
expect(imageService.autoOrient).toHaveBeenCalledOnceWith(jasmine.objectContaining({ bytes: originalConstentStream }), stagedContent.tempLocation);
expect(attachmentStore.saveContent).toHaveBeenCalledOnceWith(stagedContent, att.id, obsBefore);
expect(obsRepo.patchAttachment).not.toHaveBeenCalled();
expect(obsRepo.save).not.toHaveBeenCalled();
}));
it('does not produce an attachment patch if the content does not exist', () => __awaiter(void 0, void 0, void 0, function* () {
const att = Object.freeze({
id: '1.123.1',
observationFormId: 'form1',
fieldName: 'field1',
oriented: false,
thumbnails: [],
contentType: 'image/jpeg',
name: 'test1.jpeg',
size: 320000,
contentLocator: String(Date.now())
});
const obsBefore = observationWithAttachments('1.123', event1, [att]);
const obsRepo = observationRepos.get(event1.id);
attachmentStore.readContent.and.resolveTo(null);
const oriented = yield (0, processor_1.orientAttachmentImage)(obsBefore, att.id, imageService, attachmentStore, console);
expect(oriented.patch).toBeUndefined();
expect(imageService.autoOrient).not.toHaveBeenCalled();
expect(attachmentStore.saveContent).not.toHaveBeenCalled();
expect(obsRepo.patchAttachment).not.toHaveBeenCalled();
expect(obsRepo.save).not.toHaveBeenCalled();
}));
it('does not produce an attachment patch if reading content fails', () => __awaiter(void 0, void 0, void 0, function* () {
const att = Object.freeze({
id: '1.123.1',
observationFormId: 'form1',
fieldName: 'field1',
oriented: false,
thumbnails: [],
contentType: 'image/jpeg',
name: 'test1.jpeg',
size: 320000,
contentLocator: String(Date.now())
});
const obsBefore = observationWithAttachments('1.123', event1, [att]);
const obsRepo = observationRepos.get(event1.id);
attachmentStore.readContent.and.resolveTo(new entities_observations_1.AttachmentStoreError(entities_observations_1.AttachmentStoreErrorCode.ContentNotFound));
const oriented = yield (0, processor_1.orientAttachmentImage)(obsBefore, att.id, imageService, attachmentStore, console);
expect(oriented.patch).toBeUndefined();
expect(imageService.autoOrient).not.toHaveBeenCalled();
expect(attachmentStore.saveContent).not.toHaveBeenCalled();
expect(obsRepo.patchAttachment).not.toHaveBeenCalled();
expect(obsRepo.save).not.toHaveBeenCalled();
}));
it('does not produce an attachment patch if the image service cannot decode the image', () => __awaiter(void 0, void 0, void 0, function* () {
const att = Object.freeze({
id: '1.123.1',
observationFormId: 'form1',
fieldName: 'field1',
oriented: false,
thumbnails: [],
contentType: 'image/jpeg',
name: 'test1.jpeg',
size: 320000,
contentLocator: String(Date.now())
});
const obsBefore = observationWithAttachments('1.123', event1, [att]);
const obsRepo = observationRepos.get(event1.id);
attachmentStore.readContent.and.resolveTo(stream_1.default.Readable.from(Buffer.from('corrupted image')));
const staged = {
id: 'nawgonwork',
tempLocation: jasmine.createSpyObj('mockPendingContent', ['write', 'end'])
};
attachmentStore.stagePendingContent.and.resolveTo(staged);
imageService.autoOrient.and.resolveTo(new Error('wut is this'));
const oriented = yield (0, processor_1.orientAttachmentImage)(obsBefore, att.id, imageService, attachmentStore, console);
expect(oriented.patch).toBeUndefined();
expect(attachmentStore.saveContent).not.toHaveBeenCalled();
expect(staged.tempLocation.write).not.toHaveBeenCalled();
expect(obsRepo.patchAttachment).not.toHaveBeenCalled();
expect(obsRepo.save).not.toHaveBeenCalled();
}));
it('patches the attachment even if saving content did not change attachment meta-data');
it('does not patch the attachment if saving content failed');
});
describe('thumbnail phase', () => {
it('generates the specified thumbnails and produces the attachment patch', () => __awaiter(void 0, void 0, void 0, function* () {
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
const att = Object.freeze({
id: '1.456.1',
observationFormId: 'form1',
fieldName: 'field1',
contentType: 'image/png',
name: 'test1.jpg',
size: 987789,
width: 1200,
height: 1600,
oriented: true,
contentLocator: String(Date.now()),
thumbnails: [],
});
class ExpectedThumb {
constructor(metaData, stagedContentId) {
this.metaData = metaData;
this.stagedContent = new entities_observations_1.StagedAttachmentContent(stagedContentId, new util_spec_1.BufferWriteable());
}
get stagedContentBytes() { return this.stagedContent.tempLocation.content; }
}
const salt = String(Date.now());
const expectedThumbs = new Map()
.set(60, new ExpectedThumb({ minDimension: 60, size: 6000, width: 60, height: 80, name: 'test1-60.jpg', contentType: 'image/jpeg', contentLocator: `${att.contentLocator}-60` }, `${salt}-staged-60`))
.set(120, new ExpectedThumb({ minDimension: 120, size: 12000, width: 120, height: 160, name: 'test1-120.jpg', contentType: 'image/jpeg', contentLocator: `${att.contentLocator}-120` }, `${salt}-staged-120`))
.set(240, new ExpectedThumb({ minDimension: 240, size: 24000, width: 240, height: 320, name: 'test1-240.jpg', contentType: 'image/jpeg', contentLocator: `${att.contentLocator}-240` }, `${salt}-staged-240`));
const obsRepo = observationRepos.get(event1.id);
const obsBefore = observationWithAttachments('1.123', event1, [att]);
const obsStaged = (0, entities_observations_1.patchAttachment)(obsBefore, att.id, { thumbnails: Array.from(expectedThumbs.values()).map(x => lodash_1.default.omit((0, entities_observations_1.copyThumbnailAttrs)(x.metaData), 'contentLocator')) });
const attachmentContent = Buffer.from('big majestic mountains');
const stagedContentStack = [(_a = expectedThumbs.get(240)) === null || _a === void 0 ? void 0 : _a.stagedContent, (_b = expectedThumbs.get(120)) === null || _b === void 0 ? void 0 : _b.stagedContent, (_c = expectedThumbs.get(60)) === null || _c === void 0 ? void 0 : _c.stagedContent];
attachmentStore.readContent.and.callFake((_) => __awaiter(void 0, void 0, void 0, function* () { return stream_1.default.Readable.from(attachmentContent); }));
attachmentStore.stagePendingContent.and.callFake(() => __awaiter(void 0, void 0, void 0, function* () { return stagedContentStack.pop(); }));
attachmentStore.saveThumbnailContent.and.callFake((content, minDimension) => __awaiter(void 0, void 0, void 0, function* () {
return expectedThumbs.get(minDimension).metaData;
}));
imageService.scaleToDimension.and.callFake((minDimension, source, dest) => __awaiter(void 0, void 0, void 0, function* () {
const sourceStream = new util_spec_1.BufferWriteable();
yield util_1.default.promisify(stream_1.default.pipeline)(source.bytes, sourceStream);
const thumbContent = Buffer.from(`${sourceStream.content.toString()} @${minDimension}`);
yield util_1.default.promisify(stream_1.default.pipeline)(stream_1.default.Readable.from(thumbContent), dest);
return {
dimensions: { width: minDimension, height: minDimension * 4 / 3 },
mediaType: 'image/jpeg',
sizeInBytes: minDimension * 100
};
}));
const thumbResult = yield (0, processor_1.thumbnailAttachmentImage)(obsBefore, att.id, [60, 120, 240], imageService, attachmentStore, console);
expect(thumbResult.patch).toEqual({ thumbnails: Array.from(expectedThumbs.values()).map(x => x.metaData) });
expect((_d = expectedThumbs.get(60)) === null || _d === void 0 ? void 0 : _d.stagedContentBytes.toString()).toEqual('big majestic mountains @60');
expect((_e = expectedThumbs.get(120)) === null || _e === void 0 ? void 0 : _e.stagedContentBytes.toString()).toEqual('big majestic mountains @120');
expect((_f = expectedThumbs.get(240)) === null || _f === void 0 ? void 0 : _f.stagedContentBytes.toString()).toEqual('big majestic mountains @240');
expect(attachmentStore.stagePendingContent).toHaveBeenCalledTimes(3);
expect(attachmentStore.readContent).toHaveBeenCalledTimes(3);
expect(attachmentStore.readContent.calls.argsFor(0)).toEqual([att.id, obsBefore]);
expect(attachmentStore.readContent.calls.argsFor(1)).toEqual([att.id, obsBefore]);
expect(attachmentStore.readContent.calls.argsFor(2)).toEqual([att.id, obsBefore]);
expect(imageService.scaleToDimension).toHaveBeenCalledTimes(3);
expect(imageService.scaleToDimension).toHaveBeenCalledWith(60, jasmine.anything(), jasmine.anything());
expect(imageService.scaleToDimension).toHaveBeenCalledWith(120, jasmine.anything(), jasmine.anything());
expect(imageService.scaleToDimension).toHaveBeenCalledWith(240, jasmine.anything(), jasmine.anything());
expect(attachmentStore.saveThumbnailContent).toHaveBeenCalledTimes(3);
expect(attachmentStore.saveThumbnailContent).toHaveBeenCalledWith((_g = expectedThumbs.get(60)) === null || _g === void 0 ? void 0 : _g.stagedContent, 60, att.id, sameAsObservationWithoutDates(obsStaged));
expect(attachmentStore.saveThumbnailContent).toHaveBeenCalledWith((_h = expectedThumbs.get(120)) === null || _h === void 0 ? void 0 : _h.stagedContent, 120, att.id, sameAsObservationWithoutDates(obsStaged));
expect(attachmentStore.saveThumbnailContent).toHaveBeenCalledWith((_j = expectedThumbs.get(240)) === null || _j === void 0 ? void 0 : _j.stagedContent, 240, att.id, sameAsObservationWithoutDates(obsStaged));
expect(attachmentStore.saveContent).not.toHaveBeenCalled();
expect(obsRepo.patchAttachment).not.toHaveBeenCalled();
expect(obsRepo.save).not.toHaveBeenCalled();
}));
it('does not destroy existing thumbnail meta-data during update');
});
describe('automated processing', () => {
class TestPluginStateRepository {
constructor() {
this.state = null;
}
get() {
return __awaiter(this, void 0, void 0, function* () {
return this.state;
});
}
put(x) {
return __awaiter(this, void 0, void 0, function* () {
this.state = Object.assign({}, x);
return this.state;
});
}
patch(state) {
return __awaiter(this, void 0, void 0, function* () {
throw new Error('unimplemented');
});
}
}
let stateRepo;
let attachmentQuery;
let clock;
beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
stateRepo = new TestPluginStateRepository();
attachmentQuery = jasmine.createSpy('attachmentQuery');
clock = jasmine.clock().install();
}));
afterEach(() => {
clock.uninstall();
});
describe('plugin control', () => {
describe('stopping', () => {
it('waits for the current processing interval to finish then stops', () => __awaiter(void 0, void 0, void 0, function* () {
stateRepo.state = Object.assign(Object.assign({}, processor_1.defaultImagePluginConfig), { intervalSeconds: 10 });
const clockTickMillis = stateRepo.state.intervalSeconds * 1000 + 1;
attachmentQuery.and.resolveTo(asyncIterableOf([]));
const plugin = yield (0, processor_1.createImagePluginControl)(stateRepo, eventRepo, observationRepoForEvent, attachmentStore, attachmentQuery, imageService, console);
plugin.start();
clock.tick(clockTickMillis);
yield plugin.stop();
clock.tick(clockTickMillis);
clock.tick(clockTickMillis);
yield new Promise(resolve => {
setTimeout(resolve);
clock.tick(clockTickMillis);
});
expect(attachmentQuery).toHaveBeenCalledTimes(1);
}));
});
it('begins processing for the default configuration when no saved configuration exists', () => __awaiter(void 0, void 0, void 0, function* () {
const clockTickMillis = processor_1.defaultImagePluginConfig.intervalSeconds * 1000 + 1;
attachmentQuery.and.resolveTo(asyncIterableOf([]));
const plugin = yield (0, processor_1.createImagePluginControl)(stateRepo, eventRepo, observationRepoForEvent, attachmentStore, attachmentQuery, imageService, console);
plugin.start();
clock.tick(clockTickMillis);
yield plugin.stop();
clock.tick(clockTickMillis);
clock.tick(clockTickMillis);
yield new Promise(resolve => {
setTimeout(resolve);
clock.tick(clockTickMillis);
});
expect(stateRepo.state).toEqual(Object.assign({}, processor_1.defaultImagePluginConfig));
expect(attachmentQuery).toHaveBeenCalledTimes(1);
}));
it('begins processing for the saved config', () => __awaiter(void 0, void 0, void 0, function* () {
const config = Object.assign(Object.assign({}, processor_1.defaultImagePluginConfig), { intervalSeconds: processor_1.defaultImagePluginConfig.intervalSeconds * 2 });
stateRepo.state = config;
attachmentQuery.and.resolveTo(asyncIterableOf([]));
const plugin = yield (0, processor_1.createImagePluginControl)(stateRepo, eventRepo, observationRepoForEvent, attachmentStore, attachmentQuery, imageService, console);
plugin.start();
clock.tick(processor_1.defaultImagePluginConfig.intervalSeconds * 1000 + 1);
expect(attachmentQuery).not.toHaveBeenCalled();
clock.tick(processor_1.defaultImagePluginConfig.intervalSeconds * 1000);
yield someRunLoops();
expect(attachmentQuery).toHaveBeenCalledTimes(1);
clock.tick(processor_1.defaultImagePluginConfig.intervalSeconds * 1000 + 1);
yield someRunLoops();
expect(attachmentQuery).toHaveBeenCalledTimes(1);
yield plugin.stop();
clock.tick(config.intervalSeconds * 1000);
yield someRunLoops();
clock.tick(config.intervalSeconds * 1000);
yield someRunLoops();
expect(stateRepo.state).toEqual(Object.assign({}, config));
expect(attachmentQuery).toHaveBeenCalledTimes(1);
}));
it('fetches the plugin config from the plugin state repo', () => __awaiter(void 0, void 0, void 0, function* () {
const app = yield (0, processor_1.createImagePluginControl)(stateRepo, eventRepo, observationRepoForEvent, attachmentStore, attachmentQuery, imageService, console);
const config = {
enabled: false,
intervalBatchSize: 10,
intervalSeconds: 100,
thumbnailSizes: []
};
stateRepo.state = config;
const fetched = yield app.getConfig();
expect(fetched).toEqual(Object.assign({}, config));
}));
it('saves a new configuration and restarts automatic processing', () => __awaiter(void 0, void 0, void 0, function* () {
attachmentQuery.and.resolveTo(asyncIterableOf([]));
const plugin = yield (0, processor_1.createImagePluginControl)(stateRepo, eventRepo, observationRepoForEvent, attachmentStore, attachmentQuery, imageService, console);
plugin.start();
clock.tick(processor_1.defaultImagePluginConfig.intervalSeconds * 1000);
yield someRunLoops();
expect(attachmentQuery).toHaveBeenCalledTimes(1);
attachmentQuery.and.returnValue(new Promise(resolve => {
setTimeout(() => resolve(asyncIterableOf([])), processor_1.defaultImagePluginConfig.intervalSeconds * 1000 + 100);
}));
clock.tick(processor_1.defaultImagePluginConfig.intervalSeconds * 1000);
yield someRunLoops();
expect(attachmentQuery).toHaveBeenCalledTimes(2);
const configMod = Object.assign(Object.assign({}, processor_1.defaultImagePluginConfig), { intervalSeconds: processor_1.defaultImagePluginConfig.intervalSeconds * 2 });
plugin.applyConfig(configMod);
yield someRunLoops();
expect(attachmentQuery).toHaveBeenCalledTimes(2);
clock.tick(100);
yield someRunLoops();
expect(attachmentQuery).toHaveBeenCalledTimes(3);
yield plugin.stop();
clock.tick(configMod.intervalSeconds * 1000);
yield someRunLoops();
clock.tick(configMod.intervalSeconds * 1000);
yield someRunLoops();
expect(stateRepo.state).toEqual(Object.assign({}, configMod));
expect(attachmentQuery).toHaveBeenCalledTimes(3);
}));
});
});
it('queries for attachments based on last latest process time of each event and start time of interval', () => __awaiter(void 0, void 0, void 0, function* () {
const eventProcessingStates = new Map([
{ event: allEvents.get(1), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 5 },
{ event: allEvents.get(2), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 2 }
].map(x => [x.event.id, x]));
const pluginState = {
enabled: true,
intervalBatchSize: 1000,
intervalSeconds: 60,
thumbnailSizes: []
};
const findUnprocessedAttachments = jasmine.createSpy('findAttachments').and.resolveTo(asyncIterableOf([]));
yield (0, processor_1.processImageAttachments)(pluginState, eventProcessingStates, findUnprocessedAttachments, imageService, eventRepo, observationRepoForEvent, attachmentStore, console);
expect(findUnprocessedAttachments).toHaveBeenCalledOnceWith(Array.from(eventProcessingStates.values()), null, closeToNow(), pluginState.intervalBatchSize);
}));
it('returns the timestamps of the latest attachments processed for each event', () => __awaiter(void 0, void 0, void 0, function* () {
var _a, _b, _c, _d;
const eventProcessingStates = new Map([
{ event: (0, entities_events_1.copyMageEventAttrs)(event1), latestAttachmentProcessedTimestamp: minutesAgo(5) },
{ event: (0, entities_events_1.copyMageEventAttrs)(event2), latestAttachmentProcessedTimestamp: minutesAgo(4) },
].map(x => [x.event.id, x]));
const pluginState = {
enabled: true,
intervalBatchSize: 1000,
intervalSeconds: 60,
thumbnailSizes: []
};
const event1LatestAttachmentTime = minutesAgo(1);
const event2LatestAttachmentTime = minutesAgo(3);
const eventObservations = new Map()
.set(1, [
observationWithAttachments('1.100', event1, [
{
id: '1.100.1',
fieldName: 'field1',
lastModified: new Date(minutesAgo(6)),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
},
{
id: '1.100.2',
fieldName: 'field1',
lastModified: new Date(event1LatestAttachmentTime),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
},
]),
observationWithAttachments('1.200', event1, [
{
id: '1.200.1',
fieldName: 'field1',
lastModified: new Date(minutesAgo(10)),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
}
])
])
.set(2, [
observationWithAttachments('2.200', event2, [
{
id: '2.200.1',
fieldName: 'field1',
lastModified: new Date(),
observationFormId: 'form1',
oriented: false,
thumbnails: []
}
]),
observationWithAttachments('2.100', event2, [
{
id: '2.100.1',
fieldName: 'field1',
lastModified: new Date(event2LatestAttachmentTime),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
}
])
]);
const unprocessedAttachments = [
{
eventId: 1,
observationId: '1.100',
attachmentId: '1.100.1'
},
{
eventId: 1,
observationId: '1.100',
attachmentId: '1.100.2'
},
{
eventId: 2,
observationId: '2.100',
attachmentId: '2.100.1'
},
{
eventId: 1,
observationId: '1.200',
attachmentId: '1.200.1'
},
];
const findUnprocessedAttachments = jasmine.createSpy('findAttachments').and.resolveTo(asyncIterableOf(unprocessedAttachments));
observationRepos.forEach((repo, eventId) => {
repo.findById.and.callFake((id) => __awaiter(void 0, void 0, void 0, function* () { var _a; return ((_a = eventObservations.get(eventId)) === null || _a === void 0 ? void 0 : _a.find(x => x.id === id)) || null; }));
repo.patchAttachment.and.callFake((o) => __awaiter(void 0, void 0, void 0, function* () { return o; }));
});
imageService.autoOrient.and.resolveTo({
mediaType: 'image/png',
sizeInBytes: 30000,
dimensions: { width: 1000, height: 1000 },
});
imageService.scaleToDimension.and.callFake((minDimension, source, dest) => Promise.resolve({
mediaType: 'image/png',
sizeInBytes: 30000,
dimensions: { width: minDimension, height: Math.round(minDimension * 1.3) },
}));
const stagedContent = new entities_observations_1.StagedAttachmentContent('staged attachment', new util_spec_1.BufferWriteable());
attachmentStore.readContent.and.resolveTo(stream_1.default.Readable.from(Buffer.from('original content')));
attachmentStore.stagePendingContent.and.resolveTo(stagedContent);
attachmentStore.saveContent.and.resolveTo(null);
attachmentStore.saveThumbnailContent.and.resolveTo(null);
const processedEventStates = yield (0, processor_1.processImageAttachments)(pluginState, eventProcessingStates, findUnprocessedAttachments, imageService, eventRepo, observationRepoForEvent, attachmentStore, console);
expect(processedEventStates.size).toEqual(2);
expect((_a = processedEventStates.get(1)) === null || _a === void 0 ? void 0 : _a.event).toEqual((0, entities_events_1.copyMageEventAttrs)(event1));
expect((_b = processedEventStates.get(1)) === null || _b === void 0 ? void 0 : _b.latestAttachmentProcessedTimestamp).toEqual(event1LatestAttachmentTime);
expect((_c = processedEventStates.get(2)) === null || _c === void 0 ? void 0 : _c.event).toEqual((0, entities_events_1.copyMageEventAttrs)(event2));
expect((_d = processedEventStates.get(2)) === null || _d === void 0 ? void 0 : _d.latestAttachmentProcessedTimestamp).toEqual(event2LatestAttachmentTime);
}));
it('processes the attachments serially in the order the query returns', () => __awaiter(void 0, void 0, void 0, function* () {
const eventProcessingStates = new Map([
{ event: (0, entities_events_1.copyMageEventAttrs)(event1), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 5 },
{ event: (0, entities_events_1.copyMageEventAttrs)(event2), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 2 },
].map(x => [x.event.id, x]));
const pluginState = {
enabled: true,
intervalSeconds: 60,
intervalBatchSize: 1000,
thumbnailSizes: [60, 120]
};
const eventObservations = new Map()
.set(1, [
observationWithAttachments('1.100', event1, [
{
id: '1.100.1',
fieldName: 'field1',
lastModified: new Date(minutesAgo(6)),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
},
{
id: '1.100.2',
fieldName: 'field1',
lastModified: new Date(minutesAgo(2)),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
},
]),
observationWithAttachments('1.200', event1, [
{
id: '1.200.1',
fieldName: 'field1',
lastModified: new Date(minutesAgo(10)),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
}
])
])
.set(2, [
observationWithAttachments('2.200', event2, [
{
id: '2.200.1',
fieldName: 'field1',
lastModified: new Date(),
observationFormId: 'form1',
oriented: false,
thumbnails: []
}
]),
observationWithAttachments('2.100', event2, [
{
id: '2.100.1',
fieldName: 'field1',
lastModified: new Date(minutesAgo(3)),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
}
])
]);
const unprocessedAttachments = [
{
eventId: 1,
observationId: '1.100',
attachmentId: '1.100.1'
},
{
eventId: 1,
observationId: '1.100',
attachmentId: '1.100.2'
},
{
eventId: 2,
observationId: '2.100',
attachmentId: '2.100.1'
},
{
eventId: 1,
observationId: '1.200',
attachmentId: '1.200.1'
},
];
const failNextIfCurrentNotProcessed = {
cursor: -1,
get current() {
return unprocessedAttachments[this.cursor] || null;
},
processCurrent() {
const processed = this.current;
processed.processed = true;
return processed;
},
[Symbol.asyncIterator]() {
var _a;
return __asyncGenerator(this, arguments, function* _b() {
while (this.cursor < unprocessedAttachments.length - 1) {
if (this.cursor < 0 || ((_a = this.current) === null || _a === void 0 ? void 0 : _a.processed) === true) {
this.cursor += 1;
yield yield __await(Promise.resolve(unprocessedAttachments[this.cursor]));
}
else {
throw new Error(`attempted to get next attachment before processing current attachment ${this.cursor}`);
}
}
});
}
};
const findUnprocessedAttachments = jasmine.createSpy('findAttachments').and.resolveTo(failNextIfCurrentNotProcessed);
attachmentStore.readContent.and.resolveTo(stream_1.default.Readable.from(Buffer.from(new Date().toISOString())));
attachmentStore.stagePendingContent.and.callFake(() => __awaiter(void 0, void 0, void 0, function* () {
return ({
tempLocation: new util_spec_1.BufferWriteable(),
id: failNextIfCurrentNotProcessed.current.attachmentId
});
}));
imageService.autoOrient.and.callFake((source, dest) => __awaiter(void 0, void 0, void 0, function* () {
return yield Promise.resolve({
mediaType: `image/png`,
dimensions: { width: 100 * (failNextIfCurrentNotProcessed.cursor + 1), height: 120 * (failNextIfCurrentNotProcessed.cursor + 1) },
sizeInBytes: 1000 * (failNextIfCurrentNotProcessed.cursor + 1)
});
}));
imageService.scaleToDimension.and.callFake((minDimension) => __awaiter(void 0, void 0, void 0, function* () {
failNextIfCurrentNotProcessed.processCurrent();
return { mediaType: 'image/jpeg', sizeInBytes: minDimension * 1000, dimensions: { width: minDimension, height: minDimension * 1.5 } };
}));
observationRepos.forEach((repo, eventId) => {
repo.findById.and.callFake((id) => __awaiter(void 0, void 0, void 0, function* () { return eventObservations.get(eventId).find(x => x.id === id) || null; }));
repo.patchAttachment.and.callFake((obs) => __awaiter(void 0, void 0, void 0, function* () { return eventObservations.get(obs.eventId).find(x => x.id === obs.id); }));
});
attachmentStore.saveContent.and.resolveTo(null);
attachmentStore.saveThumbnailContent.and.resolveTo(null);
yield (0, processor_1.processImageAttachments)(pluginState, eventProcessingStates, findUnprocessedAttachments, imageService, eventRepo, observationRepoForEvent, attachmentStore, console);
expect(imageService.autoOrient).toHaveBeenCalledTimes(unprocessedAttachments.length);
for (const a of unprocessedAttachments) {
expect(a.processed).toBe(true);
}
}));
it('marks attachment oriented when orient phase does not produce a patch', () => __awaiter(void 0, void 0, void 0, function* () {
var _e, _f, _g, _h, _j, _k;
const eventProcessingStates = new Map([
{ event: (0, entities_events_1.copyMageEventAttrs)(event1), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 5 },
{ event: (0, entities_events_1.copyMageEventAttrs)(event2), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 2 },
].map(x => [x.event.id, x]));
const pluginState = {
enabled: true,
intervalSeconds: 60,
intervalBatchSize: 1000,
thumbnailSizes: []
};
const attachment1 = {
id: '1.100.1',
fieldName: 'field1',
lastModified: new Date(minutesAgo(6)),
observationFormId: 'form1',
oriented: false,
thumbnails: [],
};
const attachment2 = {
id: '2.200.1',
fieldName: 'field1',
lastModified: new Date(),
observationFormId: 'form1',
oriented: false,
thumbnails: []
};
const obs1 = observationWithAttachments('1.100', event1, [attachment1]);
const obs2 = observationWithAttachments('2.200', event2, [attachment2]);
const unprocessedAttachments = [
{ eventId: obs1.eventId, observationId: obs1.id, attachmentId: attachment1.id },
{ eventId: obs2.eventId, observationId: obs2.id, attachmentId: attachment2.id }
];
const findUnprocessedAttachments = jasmine.createSpy('findAttachments')
.and.resolveTo(asyncIterableOf(unprocessedAttachments));
const invalidImageBytes = stream_1.default.Readable.from(Buffer.from('wut is this'));
const validImageBytes = stream_1.default.Readable.from(Buffer.from('goats.png'));
(_e = observationRepos.get(event1.id)) === null || _e === void 0 ? void 0 : _e.findById.withArgs(obs1.id).and.resolveTo(obs1);
(_f = observationRepos.get(event2.id)) === null || _f === void 0 ? void 0 : _f.findById.withArgs(obs2.id).and.resolveTo(obs2);
(_g = observationRepos.get(event1.id)) === null || _g === void 0 ? void 0 : _g.patchAttachment.and.resolveTo(obs1);
(_h = observationRepos.get(event2.id)) === null || _h === void 0 ? void 0 : _h.patchAttachment.and.resolveTo(obs2);
attachmentStore.readContent.withArgs(jasmine.stringMatching(attachment1.id), jasmine.anything()).and.resolveTo(invalidImageBytes);
attachmentStore.readContent.withArgs(jasmine.stringMatching(attachment2.id), jasmine.anything()).and.resolveTo(validImageBytes);
attachmentStore.stagePendingContent.and.resolveTo({
tempLocation: new util_spec_1.BufferWriteable(),
id: 'pending'
});
imageService.autoOrient.and.callFake((source, dest) => __awaiter(void 0, void 0, void 0, function* () {
if (source.bytes === invalidImageBytes) {
return new Error('bad image data');
}
return yield Promise.resolve({
mediaType: `image/png`,
dimensions: { width: 100, height: 120 },
sizeInBytes: 10000
});
}));
yield (0, processor_1.processImageAttachments)(pluginState, eventProcessingStates, findUnprocessedAttachments, imageService, eventRepo, observationRepoForEvent, attachmentStore, console);
expect((_j = observationRepos.get(event1.id)) === null || _j === void 0 ? void 0 : _j.patchAttachment).toHaveBeenCalledOnceWith(obs1, attachment1.id, { oriented: true });
expect((_k = observationRepos.get(event2.id)) === null || _k === void 0 ? void 0 : _k.patchAttachment).toHaveBeenCalledOnceWith(obs2, attachment2.id, {
oriented: true,
contentType: 'image/png',
size: 10000,
width: 100,
height: 120,
});
}));
it('does not generate thumbnails if orient fails');
it('updates the attachment after orienting and creating thumbnails');
it('skips processing events that do not exist');
it('skips processing events that are complete');
});
function AttachmentContentKey(attachmentId, observationId) {
return `${observationId}::${attachmentId}`;
}
function ThumbnailContentKey(minDimension, attachmentId, observationId) {
return `${observationId}::${attachmentId}@${minDimension}`;
}
class BufferAttachmentStore {
constructor() {
this.pendingContent = new Map();
this.attachmentContent = new Map();
this.thumbnailContent = new Map();
this.nextPendingId = 1;
}
stagePendingContent() {
return __awaiter(this, void 0, void 0, function* () {
const id = `pending::${this.nextPendingId++}`;
const tempLocation = new util_spec_1.BufferWriteable();
const pending = {
id,
tempLocation: tempLocation.on('finish', () => {
this.pendingContent.set(id, tempLocation.content);
})
};
this.pendingContent.set(id, Buffer.alloc(0));
return pending;
});
}
saveContent(content, attachmentId, observation) {
return __awaiter(this, void 0, void 0, function* () {
if (typeof content !== 'string') {
return new entities_observations_1.AttachmentStoreError(entities_observations_1.AttachmentStoreErrorCode.ContentNotFound, 'this store supports saving only staged content');
}
const att = observation.attachmentFor(attachmentId);
if (!att) {
return new entities_observations_1.AttachmentStoreError(entities_observations_1.AttachmentStoreErrorCode.InvalidAttachmentId);
}
const pendingBytes = this.pendingContent.get(content);
if (!pendingBytes) {
return new entities_observations_1.AttachmentStoreError(entities_observations_1.AttachmentStoreErrorCode.ContentNotFound, `pending content not found: ${content}`);
}
const key = AttachmentContentKey(attachmentId, observation.id);
this.pendingContent.delete(content);
this.attachmentContent.set(key, pendingBytes);
return { contentLocator: key, size: pendingBytes.length };
});
}
saveThumbnailContent(content, minDimension, attachmentId, observation) {
return __awaiter(this, void 0, void 0, function* () {
if (typeof content !== 'string') {
return new entities_observations_1.AttachmentStoreError(enti