@cornerstonejs/core
Version:
Cornerstone3D Core
730 lines (729 loc) • 28.3 kB
JavaScript
import { Events as EVENTS, VideoEnums as VideoViewportEnum } from '../enums/index.js';
import { getGenericViewportSourceDataId } from './GenericViewport/genericViewportDisplaySetAccess.js';
import { Transform } from './helpers/cpuFallback/rendering/transform.js';
import triggerEvent from '../utilities/triggerEvent.js';
import Viewport from './Viewport.js';
import { getOrCreateCanvas } from './helpers/index.js';
import CanvasActor from './CanvasActor/index.js';
import cache from '../cache/cache.js';
import uuidv4 from '../utilities/uuidv4.js';
import FrameRange from '../utilities/FrameRange.js';
import { pointInShapeCallback } from '../utilities/pointInShapeCallback.js';
import { getVideoImageDataMetadata, loadVideoStreamMetadata, normalizeVideoPlaybackInfo, } from '../utilities/VideoUtilities.js';
class VideoViewport extends Viewport {
constructor(props) {
super({
...props,
canvas: props.canvas || getOrCreateCanvas(props.element),
});
this.videoWidth = 0;
this.videoHeight = 0;
this.loop = true;
this.mute = true;
this.isPlaying = false;
this.scrollSpeed = 1;
this.playbackRate = 1;
this.frameRange = [0, 0];
this.fps = 30;
this.videoCamera = {
panWorld: [0, 0],
parallelScale: 1,
};
this.voiRange = {
lower: 0,
upper: 255,
};
this.getProperties = () => {
return {
loop: this.videoElement.loop,
muted: this.videoElement.muted,
playbackRate: this.playbackRate,
scrollSpeed: this.scrollSpeed,
voiRange: { ...this.voiRange },
};
};
this.getMiddleSliceData = () => {
throw new Error('Method not implemented.');
};
this.useCustomRenderingPipeline = true;
this.resetCamera = () => {
this.refreshRenderValues();
this.canvasContext.fillRect(0, 0, this.canvas.width, this.canvas.height);
if (!this.isPlaying) {
this.renderFrame();
}
return true;
};
this.getNumberOfSlices = () => {
const computedSlices = Math.round((this.videoElement.duration * this.fps) / this.scrollSpeed);
return isNaN(computedSlices) ? this.numberOfFrames : computedSlices;
};
this.getFrameOfReferenceUID = () => {
return this.videoElement.src;
};
this.resize = () => {
const canvas = this.canvas;
const { clientWidth, clientHeight } = canvas;
if (canvas.width !== clientWidth || canvas.height !== clientHeight) {
canvas.width = clientWidth;
canvas.height = clientHeight;
}
this.refreshRenderValues();
if (!this.isPlaying) {
this.renderFrame();
}
};
this.canvasToWorld = (canvasPos, destPos = [0, 0, 0]) => {
const pan = this.videoCamera.panWorld;
const worldToCanvasRatio = this.getWorldToCanvasRatio();
const panOffsetCanvas = [
pan[0] * worldToCanvasRatio,
pan[1] * worldToCanvasRatio,
];
const subCanvasPos = [
canvasPos[0] - panOffsetCanvas[0],
canvasPos[1] - panOffsetCanvas[1],
];
destPos.splice(0, 2, subCanvasPos[0] / worldToCanvasRatio, subCanvasPos[1] / worldToCanvasRatio);
return destPos;
};
this.worldToCanvas = (worldPos) => {
const pan = this.videoCamera.panWorld;
const worldToCanvasRatio = this.getWorldToCanvasRatio();
const canvasPos = [
(worldPos[0] + pan[0]) * worldToCanvasRatio,
(worldPos[1] + pan[1]) * worldToCanvasRatio,
];
return canvasPos;
};
this.getRotation = () => 0;
this.canvasToIndex = (canvasPos) => {
const transform = this.getTransform();
transform.invert();
return transform.transformPoint(canvasPos.map((it) => it * devicePixelRatio));
};
this.indexToCanvas = (indexPos) => {
const transform = this.getTransform();
return transform
.transformPoint(indexPos)
.map((it) => it / devicePixelRatio);
};
this.customRenderViewportToCanvas = () => {
this.renderFrame();
};
this.renderFrame = () => {
const dpr = window.devicePixelRatio || 1;
const transform = this.getTransform();
const transformationMatrix = transform.getMatrix();
const ctx = this.canvasContext;
ctx.resetTransform();
ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
ctx.transform(transformationMatrix[0] / dpr, transformationMatrix[1] / dpr, transformationMatrix[2] / dpr, transformationMatrix[3] / dpr, transformationMatrix[4] / dpr, transformationMatrix[5] / dpr);
ctx.drawImage(this.videoElement, 0, 0, this.videoWidth, this.videoHeight);
for (const actor of this.getActors()) {
actor.actor.render(this, this.canvasContext);
}
this.canvasContext.resetTransform();
this.setRendered();
triggerEvent(this.element, EVENTS.STACK_NEW_IMAGE, {
element: this.element,
viewportId: this.id,
viewport: this,
renderingEngineId: this.renderingEngineId,
time: this.videoElement.currentTime,
duration: this.videoElement.duration,
});
triggerEvent(this.element, EVENTS.IMAGE_RENDERED, {
element: this.element,
viewportId: this.id,
viewport: this,
imageIndex: this.getCurrentImageIdIndex(),
numberOfSlices: this.numberOfFrames,
renderingEngineId: this.renderingEngineId,
time: this.videoElement.currentTime,
duration: this.videoElement.duration,
});
this.initialRender?.();
const frame = this.getFrameNumber();
if (this.isPlaying) {
if (frame < this.frameRange[0]) {
this.setFrameNumber(this.frameRange[0]);
}
else if (frame > this.frameRange[1]) {
if (this.loop) {
this.setFrameNumber(this.frameRange[0]);
}
else {
this.pause();
}
}
}
};
this.renderWhilstPlaying = () => {
this.renderFrame();
if (this.isPlaying) {
requestAnimationFrame(this.renderWhilstPlaying);
}
};
this.canvasContext = this.canvas.getContext('2d');
this.renderingEngineId = props.renderingEngineId;
this.element.setAttribute('data-viewport-uid', this.id);
this.element.setAttribute('data-rendering-engine-uid', this.renderingEngineId);
this.videoElement = document.createElement('video');
this.videoElement.muted = this.mute;
this.videoElement.loop = this.loop;
this.videoElement.autoplay = true;
this.videoElement.crossOrigin = 'anonymous';
this.addEventListeners();
this.resize();
}
static get useCustomRenderingPipeline() {
return true;
}
addEventListeners() {
this.canvas.addEventListener(EVENTS.ELEMENT_DISABLED, this.elementDisabledHandler);
}
removeEventListeners() {
this.canvas.removeEventListener(EVENTS.ELEMENT_DISABLED, this.elementDisabledHandler);
}
elementDisabledHandler() {
this.removeEventListeners();
this.videoElement.remove();
}
getImageDataMetadata(image) {
const imageId = typeof image === 'string' ? image : image.imageId;
const metadata = getVideoImageDataMetadata(imageId);
this.hasPixelSpacing = metadata.hasPixelSpacing;
return metadata;
}
setDataIds(imageIds, options) {
this.setVideo(imageIds[0], (options.viewReference?.sliceIndex || 0) + 1);
}
setDataList(entries) {
if (!entries.length) {
return;
}
return this.setVideo(entries[0].dataId);
}
async setDisplaySets(...entries) {
await this.mountDisplaySets(entries, async (entry) => {
const sourceDataId = getGenericViewportSourceDataId(entry.displaySetId);
const { frameNumber, viewReference } = entry.options ?? {};
const initialFrame = frameNumber ??
(typeof viewReference?.sliceIndex === 'number'
? viewReference.sliceIndex + 1
: undefined);
await this.setVideo(sourceDataId, initialFrame);
});
}
setVideo(imageId, frameNumber) {
this.clearDisplaySets();
this.imageId = Array.isArray(imageId) ? imageId[0] : imageId;
const stream = loadVideoStreamMetadata(imageId);
this.modality = stream.modality;
this.metadata = stream.metadata;
this.numberOfFrames = stream.numberOfFrames;
return this.setVideoURL(stream.renderedUrl).then(() => {
const playbackInfo = normalizeVideoPlaybackInfo({
durationSeconds: this.videoElement.duration,
cineRate: stream.cineRate,
numberOfFrames: stream.numberOfFrames,
});
this.fps = playbackInfo.fps;
this.numberOfFrames = playbackInfo.numberOfFrames;
this.setFrameRange(playbackInfo.frameRange);
this.initialRender = () => {
this.initialRender = null;
this.pause();
this.setFrameNumber(frameNumber || 1);
};
return new Promise((resolve) => {
window.setTimeout(() => {
this.setFrameNumber(frameNumber || 1);
resolve(this);
}, 25);
});
});
}
async setVideoURL(videoURL) {
return new Promise((resolve) => {
this.videoElement.src = videoURL;
this.videoElement.preload = 'auto';
const loadedMetadataEventHandler = () => {
this.videoWidth = this.videoElement.videoWidth;
this.videoHeight = this.videoElement.videoHeight;
this.videoElement.removeEventListener('loadedmetadata', loadedMetadataEventHandler);
this.refreshRenderValues();
resolve(true);
};
this.videoElement.addEventListener('loadedmetadata', loadedMetadataEventHandler);
});
}
getImageIds() {
const imageIds = new Array(this.numberOfFrames);
const baseImageId = this.imageId.replace(/[0-9]+$/, '');
for (let i = 0; i < this.numberOfFrames; i++) {
imageIds[i] = `${baseImageId}${i + 1}`;
}
return imageIds;
}
togglePlayPause() {
if (this.isPlaying) {
this.pause();
return false;
}
else {
this.play();
return true;
}
}
async play() {
try {
if (!this.isPlaying) {
this.isPlaying = true;
await this.videoElement.play();
this.renderWhilstPlaying();
}
}
catch (_error) {
}
}
pause() {
try {
this.isPlaying = false;
this.videoElement.pause();
}
catch {
}
}
async scroll(delta = 1) {
await this.pause();
const videoElement = this.videoElement;
const renderFrame = this.renderFrame;
const currentTime = videoElement.currentTime;
const newTime = currentTime + (delta * this.scrollSpeed) / this.fps;
videoElement.currentTime = newTime;
const seekEventListener = () => {
renderFrame();
videoElement.removeEventListener('seeked', seekEventListener);
};
videoElement.addEventListener('seeked', seekEventListener);
}
async start() {
const videoElement = this.videoElement;
const renderFrame = this.renderFrame;
videoElement.currentTime = 0;
if (videoElement.paused) {
const seekEventListener = () => {
renderFrame();
videoElement.removeEventListener('seeked', seekEventListener);
};
videoElement.addEventListener('seeked', seekEventListener);
}
}
async end() {
const videoElement = this.videoElement;
const renderFrame = this.renderFrame;
videoElement.currentTime = videoElement.duration;
if (videoElement.paused) {
const seekEventListener = () => {
renderFrame();
videoElement.removeEventListener('seeked', seekEventListener);
};
videoElement.addEventListener('seeked', seekEventListener);
}
}
async setTime(timeInSeconds) {
const videoElement = this.videoElement;
const renderFrame = this.renderFrame;
videoElement.currentTime = timeInSeconds;
if (videoElement.paused) {
const seekEventListener = () => {
renderFrame();
videoElement.removeEventListener('seeked', seekEventListener);
};
videoElement.addEventListener('seeked', seekEventListener);
}
}
getSliceViewInfo() {
throw new Error('Method not implemented.');
}
async setFrameNumber(frame) {
this.setTime((frame - 1) / this.fps);
}
setFrameRange(frameRange) {
if (!frameRange) {
this.frameRange = [1, this.numberOfFrames];
return;
}
if (frameRange.length !== 2 || frameRange[0] === frameRange[1]) {
return;
}
this.frameRange = [frameRange[0], frameRange[1]];
}
getSliceIndexForImage(reference) {
if (!reference) {
return;
}
if (typeof reference === 'string') {
return FrameRange.imageIdToFrameStart(reference);
}
if (reference.referencedImageId) {
return FrameRange.imageIdToFrameStart(reference.referencedImageId);
}
return;
}
getFrameRange() {
return this.frameRange;
}
setProperties(props) {
if (props.loop !== undefined) {
this.videoElement.loop = props.loop;
}
if (props.muted !== undefined) {
this.videoElement.muted = props.muted;
}
if (props.playbackRate !== undefined) {
this.setPlaybackRate(props.playbackRate);
}
if (props.scrollSpeed !== undefined) {
this.setScrollSpeed(props.scrollSpeed);
}
if (props.voiRange) {
this.setVOI(props.voiRange);
}
}
setPlaybackRate(rate = 1) {
this.playbackRate = rate;
if (rate < 0.0625) {
this.pause();
return;
}
if (!this.videoElement) {
return;
}
this.videoElement.playbackRate = rate;
this.play();
}
setScrollSpeed(scrollSpeed = 1, unit = VideoViewportEnum.SpeedUnit.FRAME) {
this.scrollSpeed =
unit === VideoViewportEnum.SpeedUnit.SECOND
? scrollSpeed * this.fps
: scrollSpeed;
}
resetProperties() {
this.setProperties({
loop: false,
muted: true,
voiRange: {
lower: 0,
upper: 255,
},
});
}
getScalarData() {
if (this.scalarData?.frameNumber === this.getFrameNumber()) {
return this.scalarData;
}
if (!this.videoElement ||
!this.videoElement.videoWidth ||
!this.videoElement.videoHeight) {
console.debug('Video not ready yet, returning empty scalar data');
const emptyData = new Uint8ClampedArray();
emptyData.getRange = () => [0, 255];
emptyData.frameNumber = -1;
return emptyData;
}
const canvas = document.createElement('canvas');
canvas.width = this.videoElement.videoWidth;
canvas.height = this.videoElement.videoHeight;
const context = canvas.getContext('2d');
context.drawImage(this.videoElement, 0, 0);
const canvasData = context.getImageData(0, 0, canvas.width, canvas.height);
const scalarData = canvasData.data;
scalarData.getRange = () => [0, 255];
scalarData.frameNumber = this.getFrameNumber();
this.scalarData = scalarData;
return scalarData;
}
getImageData() {
const { metadata } = this;
const spacing = metadata.spacing;
const imageData = {
getDirection: () => metadata.direction,
getDimensions: () => metadata.dimensions,
getRange: () => [0, 255],
getScalarData: () => this.getScalarData(),
getSpacing: () => metadata.spacing,
worldToIndex: (point) => {
const canvasPoint = this.worldToCanvas(point);
const pixelCoord = this.canvasToIndex(canvasPoint);
return [pixelCoord[0], pixelCoord[1], 0];
},
indexToWorld: (point, destPoint) => {
const canvasPoint = this.indexToCanvas([point[0], point[1]]);
return this.canvasToWorld(canvasPoint, destPoint);
},
};
const imageDataForReturn = {
dimensions: metadata.dimensions,
spacing,
origin: metadata.origin,
direction: metadata.direction,
metadata: {
Modality: this.modality,
FrameOfReferenceUID: metadata.FrameOfReferenceUID,
},
getScalarData: () => this.getScalarData(),
scalarData: this.getScalarData(),
imageData,
voxelManager: {
forEach: (callback, options) => {
return pointInShapeCallback(options.imageData, {
pointInShapeFn: options.isInObject ?? (() => true),
callback: callback,
boundsIJK: options.boundsIJK,
returnPoints: options.returnPoints ?? false,
});
},
},
hasPixelSpacing: this.hasPixelSpacing,
calibration: this.calibration,
preScale: {
scaled: false,
},
};
Object.defineProperty(imageData, 'scalarData', {
get: () => this.getScalarData(),
enumerable: true,
});
return imageDataForReturn;
}
hasImageURI(imageURI) {
const framesMatch = imageURI.match(VideoViewport.frameRangeExtractor);
const testURI = framesMatch
? imageURI.substring(0, framesMatch.index)
: imageURI;
return this.imageId.includes(testURI);
}
setVOI(voiRange) {
this.voiRange = voiRange;
const feFilter = this.setColorTransform(voiRange, this.averageWhite);
this.canvas.style.filter = feFilter;
}
setWindowLevel(windowWidth = 256, windowCenter = 128) {
const lower = windowCenter - windowWidth / 2;
const upper = windowCenter + windowWidth / 2 - 1;
this.setVOI({ lower, upper });
this.setColorTransform({ lower, upper }, this.averageWhite);
}
setAverageWhite(averageWhite) {
this.averageWhite = averageWhite;
this.setColorTransform(this.voiRange, averageWhite);
}
setCamera(camera) {
const { parallelScale, focalPoint } = camera;
if (parallelScale) {
this.videoCamera.parallelScale =
this.element.clientHeight / 2 / parallelScale;
}
if (focalPoint !== undefined) {
const focalPointCanvas = this.worldToCanvas(focalPoint);
const canvasCenter = [
this.element.clientWidth / 2,
this.element.clientHeight / 2,
];
const panWorldDelta = [
(focalPointCanvas[0] - canvasCenter[0]) /
this.videoCamera.parallelScale,
(focalPointCanvas[1] - canvasCenter[1]) /
this.videoCamera.parallelScale,
];
this.videoCamera.panWorld = [
this.videoCamera.panWorld[0] - panWorldDelta[0],
this.videoCamera.panWorld[1] - panWorldDelta[1],
];
}
this.canvasContext.fillStyle = 'rgba(0,0,0,1)';
this.canvasContext.fillRect(0, 0, this.canvas.width, this.canvas.height);
if (!this.isPlaying) {
this.renderFrame();
}
}
getCurrentImageId(index = this.getCurrentImageIdIndex()) {
const current = this.imageId?.replace('/frames/1', `/frames/${index + 1}`);
return current;
}
getViewReferenceId(specifier = {}) {
const { sliceIndex } = specifier;
if (sliceIndex === undefined) {
return `videoId:${this.getCurrentImageId()}`;
}
if (Array.isArray(sliceIndex)) {
return `videoId:${this.imageId.substring(0, this.imageId.length - 1)}${sliceIndex[0] + 1}-${sliceIndex[1] + 1}`;
}
const baseTarget = this.imageId.replace('/frames/1', `/frames/${1 + sliceIndex}`);
return `videoId:${baseTarget}`;
}
isReferenceViewable(viewRef, options = {}) {
let { imageURI } = options;
const { referencedImageId, sliceIndex, multiSliceReference } = viewRef;
if (!super.isReferenceViewable(viewRef)) {
return false;
}
const imageId = this.getCurrentImageId();
if (!imageURI) {
const colonIndex = imageId.indexOf(':');
imageURI = imageId.substring(colonIndex + 1, imageId.length - 1);
}
if (options.withNavigation) {
return true;
}
const currentIndex = this.getSliceIndex();
if (multiSliceReference) {
const rangeEndSliceIndex = FrameRange.imageIdToFrameEnd(multiSliceReference.referencedImageId);
return currentIndex >= sliceIndex && currentIndex <= rangeEndSliceIndex;
}
if (sliceIndex !== undefined) {
return currentIndex === sliceIndex;
}
if (!referencedImageId) {
return false;
}
const match = referencedImageId.match(VideoViewport.frameRangeExtractor);
if (!match) {
return true;
}
if (!match[2]) {
return true;
}
const range = match[2].split('-').map((it) => Number(it));
const frame = currentIndex + 1;
return range[0] <= frame && frame <= (range[1] ?? range[0]);
}
setViewReference(viewRef) {
if (typeof viewRef.sliceIndex === 'number') {
this.setFrameNumber(viewRef.sliceIndex + 1);
}
else if (Array.isArray(viewRef.sliceIndex)) {
this.setFrameRange(viewRef.sliceIndex);
}
}
getViewReference(viewRefSpecifier) {
const sliceIndex = viewRefSpecifier?.sliceIndex ??
(this.isPlaying ? this.frameRange[0] : this.getCurrentImageIdIndex());
const rangeEndSliceIndex = viewRefSpecifier?.rangeEndSliceIndex ??
(this.isPlaying ? this.frameRange[1] - 1 : undefined);
const multiSliceReference = rangeEndSliceIndex > sliceIndex
? {
sliceIndex: rangeEndSliceIndex,
referencedImageId: this.getCurrentImageId(rangeEndSliceIndex),
}
: undefined;
return {
...super.getViewReference(viewRefSpecifier),
referencedImageId: this.getViewReferenceId(viewRefSpecifier),
sliceIndex,
multiSliceReference,
};
}
getFrameNumber() {
return 1 + this.getCurrentImageIdIndex();
}
getCurrentImageIdIndex() {
return Math.round(this.videoElement.currentTime * this.fps);
}
getSliceIndex() {
return this.getCurrentImageIdIndex() / this.scrollSpeed;
}
getCamera() {
const { parallelScale } = this.videoCamera;
const canvasCenter = [
this.element.clientWidth / 2,
this.element.clientHeight / 2,
];
const canvasCenterWorld = this.canvasToWorld(canvasCenter);
return {
parallelProjection: true,
focalPoint: canvasCenterWorld,
position: [0, 0, 0],
viewUp: [0, -1, 0],
parallelScale: this.element.clientHeight / 2 / parallelScale,
viewPlaneNormal: [0, 0, 1],
};
}
getFrameRate() {
return this.fps;
}
getPan() {
const panWorld = this.videoCamera.panWorld;
return [panWorld[0], panWorld[1]];
}
refreshRenderValues() {
let worldToCanvasRatio = this.canvas.offsetWidth / this.videoWidth;
if (this.videoHeight * worldToCanvasRatio > this.canvas.height) {
worldToCanvasRatio = this.canvas.offsetHeight / this.videoHeight;
}
const drawWidth = Math.floor(this.videoWidth * worldToCanvasRatio);
const drawHeight = Math.floor(this.videoHeight * worldToCanvasRatio);
const xOffsetCanvas = (this.canvas.offsetWidth - drawWidth) / 2;
const yOffsetCanvas = (this.canvas.offsetHeight - drawHeight) / 2;
const xOffsetWorld = xOffsetCanvas / worldToCanvasRatio;
const yOffsetWorld = yOffsetCanvas / worldToCanvasRatio;
this.videoCamera.panWorld = [xOffsetWorld, yOffsetWorld];
this.videoCamera.parallelScale = worldToCanvasRatio;
}
getWorldToCanvasRatio() {
return this.videoCamera.parallelScale;
}
getCanvasToWorldRatio() {
return 1.0 / this.videoCamera.parallelScale;
}
getTransform() {
const panWorld = this.videoCamera.panWorld;
const devicePixelRatio = window.devicePixelRatio || 1;
const worldToCanvasRatio = this.getWorldToCanvasRatio();
const canvasToWorldRatio = this.getCanvasToWorldRatio();
const halfCanvas = [
this.canvas.offsetWidth / 2,
this.canvas.offsetHeight / 2,
];
const halfCanvasWorldCoordinates = [
halfCanvas[0] * canvasToWorldRatio,
halfCanvas[1] * canvasToWorldRatio,
];
const transform = new Transform();
transform.scale(devicePixelRatio, devicePixelRatio);
transform.translate(halfCanvas[0], halfCanvas[1]);
transform.scale(worldToCanvasRatio, worldToCanvasRatio);
transform.translate(panWorld[0], panWorld[1]);
transform.translate(-halfCanvasWorldCoordinates[0], -halfCanvasWorldCoordinates[1]);
return transform;
}
updateCameraClippingPlanesAndRange() {
}
addImages(stackInputs) {
const actors = this.getActors();
stackInputs.forEach((stackInput) => {
const { imageId, ...rest } = stackInput;
const image = cache.getImage(imageId);
const imageActor = this.createActorMapper(image);
const uid = stackInput.actorUID ?? uuidv4();
if (imageActor) {
actors.push({ uid, actor: imageActor, referencedId: imageId, ...rest });
if (stackInput.callback) {
stackInput.callback({
imageActor: imageActor,
imageId,
});
}
}
});
this.setActors(actors);
}
createActorMapper(image) {
return new CanvasActor(this, image);
}
}
VideoViewport.frameRangeExtractor = /(\/frames\/|[&?]frameNumber=)([^/&?]*)/i;
export default VideoViewport;