labo-components
Version:
908 lines (793 loc) • 30.6 kB
JSX
import React from "react";
import PropTypes from "prop-types";
import FlexPlayerUtil from "../../../util/FlexPlayerUtil";
import MediaObject from "../../../model/MediaObject";
import EditableText from "../../shared/EditableText";
import { ANNOTATION_TARGET } from "../../../util/AnnotationConstants";
import { AnnotationEvents } from "../AnnotationClient";
import MediaEvents from "../_MediaEvents";
import { ResourceViewerContext } from "../ResourceViewerContext";
import Strings from "../_Strings";
import { getSegmentTitle, getSelectionTitle } from "../AnnotationHelpers";
import Timeline from "./timeline/Timeline";
import {
UserSegmentSection,
ContentSegmentSection,
ASRSentenceSection,
ASRWordCloudSection,
} from "./timeline/Sections";
import SectionEdit from "./timeline/SectionEdit";
import KeyboardInteraction from "./KeyboardInteraction";
const LAYER_CONTENT_SEGMENTS_ID = -102;
/**
* This component builds the timeline for the given MediaObject
* and media content duration
*/
export default class TimelineView extends React.PureComponent {
static contextType = ResourceViewerContext;
constructor(props) {
super(props);
this.timelineRef = React.createRef();
this.currentPosition = 0;
this.state = {
userLayers: [],
contentLayers: [],
};
this.annotationEvents = [
AnnotationEvents.ON_EDIT,
AnnotationEvents.ON_SET_ANNOTATION,
AnnotationEvents.ON_SET_SELECTION,
AnnotationEvents.ON_SAVE,
AnnotationEvents.ON_DELETE,
AnnotationEvents.ON_CHANGE_TARGET,
];
}
/**
* React lifecycle functions
*/
componentDidMount = () => {
// Bind to player position updates
this.context.mediaEvents.bind(
MediaEvents.PLAYER_POS,
this.updateTimelinePosition
);
// Bind to annotation updates
this.annotationEvents.forEach((event) => {
this.context.annotationClient.events.bind(
event,
this.updateAnnotations
);
});
// Build layers
this.buildLayers();
// Keyboardinteractions for valid users with active project
if (this.hasValidUserAndProject()) {
this.initKeyboardInteractions();
}
};
componentWillUnmount = () => {
// Unbind player updates
this.context.mediaEvents.unbind(
MediaEvents.PLAYER_POS,
this.updateTimelinePosition
);
// Unbind annotation updates
this.annotationEvents.forEach((event) => {
this.context.annotationClient.events.unbind(
event,
this.updateAnnotations
);
});
// keyboard interaction
if (this.keyboardInteraction) {
this.keyboardInteraction.destroy();
}
};
componentDidUpdate(prevProps) {
// Update the layers in case the mediaObject is changed
// or when the active AnnotationTypes did change
if (prevProps.mediaObject.id !== this.props.mediaObject.id) {
this.buildContentLayers();
}
// Update the layers in case active AnnotationTypes changed
if (
prevProps.activeAnnotationTypes !== this.props.activeAnnotationTypes
) {
this.buildUserLayers();
}
}
initKeyboardInteractions() {
this.keyboardInteraction = new KeyboardInteraction({
context: this.context,
});
}
// Update segment with given selection
// If the selection is null, delete the segment
updateSegment = async (segment, selection) => {
// When the selection is null: delete the segment
if (selection == null) {
this.context.annotationClient.delete(segment);
return;
}
// make the segment the active annotation, so it will be updated
this.context.annotationClient.activeAnnotation = segment;
// set the active selection
this.context.annotationClient.activeSelection = selection;
// Update the segment
await this.context.annotationClient.saveSelection(selection);
};
withinRange = (x) => {
return Math.max(0, Math.min(this.props.duration, x));
};
snapToSegments = (
x,
segments,
currentSegment,
layerId = null,
offset = 1.25
) => {
if (!currentSegment) {
return;
}
// If segments == null, snap to all user segments
if (segments == null) {
segments = this.context.annotationClient.annotations
? this.context.annotationClient.annotations.filter(
(annotation) =>
annotation.target.type ===
ANNOTATION_TARGET.SEGMENT &&
(layerId === null ||
annotation.target.layerId === layerId)
)
: [];
}
offset /= this.getPixelsPerSecond();
segments.some((segment) => {
if (segment.id == currentSegment.id) {
return false;
}
// start
if (
Math.abs(segment.target.selector.refinedBy.start - x) < offset
) {
x = segment.target.selector.refinedBy.start;
return true;
}
// end
if (Math.abs(segment.target.selector.refinedBy.end - x) < offset) {
x = segment.target.selector.refinedBy.end;
return true;
}
return false;
});
return x;
};
/**
* Update annotations (called from events, section edits etc)
*/
updateAnnotations = () => {
this.buildUserLayers();
};
updateUserAnnotationLayer = (layerId) => {
const layer = this.state.userLayers.find(
(layer) => layer.id === layerId
);
// failsafe
if (!layer) {
this.updateAnnotations();
console.error("Could not find layer with id", layerId);
return;
}
const layerTitle = this.context.annotationClient.segmentLayers.getLayerTitle(
layerId
);
const userLayers = [...this.state.userLayers];
userLayers[userLayers.indexOf(layer)] = this.buildUserLayer(
layerId,
layerTitle
);
this.setState({ userLayers });
};
/**
* Operate directly on timeline ref
*/
updateTimelinePosition = (pos) => {
this.currentPosition = pos;
if (this.timelineRef.current) {
this.timelineRef.current.setPositionExternal(pos);
}
};
getPixelsPerSecond = () => {
return this.timelineRef.current
? this.timelineRef.current.getPixelsPerSecond()
: null;
};
updatePlayerPos = (pos) => {
this.context.mediaEvents.trigger(MediaEvents.SET_PLAYER_POS, pos);
};
showContentAnnotations = () => {
this.context.mediaEvents.trigger(MediaEvents.SHOW_CONTENT_ANNOTATIONS);
};
/**
* Timeline layers
*/
selectTimelineLayer = (layerId) => {
if (this.timelineRef.current) {
this.timelineRef.current.selectLayer(layerId);
}
};
getActiveLayerId = () => {
if (this.timelineRef.current) {
return this.timelineRef.current.getActiveLayer();
}
return null;
};
/* ----------------------------- USER LAYERS ------------------------ */
getAnnotateSegmentButton = (
annotationClient,
showAnnotationPopup,
layerId
) => {
// Only render the button when there is an active annotation of type segment
return annotationClient.activeAnnotation &&
annotationClient.activeAnnotation.target &&
annotationClient.activeAnnotation.target.layerId == layerId &&
annotationClient.activeAnnotation.target.type ===
ANNOTATION_TARGET.SEGMENT ? (
<div
className="annotate-segment-button"
title={Strings.ANNOTATE_SEGMENT_BUTTON_TITLE}
onClick={() => {
showAnnotationPopup();
}}
/>
) : null;
};
getDeleteSegmentButton = (annotationClient, layerId) => {
// Only render the button when there is an active annotation of type segment
return annotationClient.activeAnnotation &&
annotationClient.activeAnnotation.target &&
annotationClient.activeAnnotation.target.layerId == layerId &&
annotationClient.activeAnnotation.target.type ===
ANNOTATION_TARGET.SEGMENT ? (
<div
className="delete-segment-button"
title={Strings.DELETE_SEGMENT_BUTTON_TITLE}
onClick={() => {
if (!confirm(Strings.SEGMENT_DELETE_CONFIRM)) {
return;
}
// sending null to the update function results in the segment getting deleted
annotationClient.delete(annotationClient.activeAnnotation);
}}
/>
) : null;
};
getAddSegmentButton = (annotationClient, layerId) => (
<div
className="add-segment-button"
title={Strings.TIMELINE_LAYER_USER_SEGMENTS_ADD_BUTTON_TITLE}
onMouseDown={async (e) => {
// set activeAnnotation to null, so a new root annotation is created & activated
annotationClient.activeAnnotation = null;
let selection;
// shift key = concat new segment
if (e.shiftKey) {
const prevSectionEnd = annotationClient.getSegmentEndBefore(
this.currentPosition,
layerId
);
selection = annotationClient.newTemporalSegment(
prevSectionEnd,
this.currentPosition
);
} else {
selection = annotationClient.newTemporalSegment(
this.currentPosition,
this.currentPosition + 1
);
}
// Create and activate a new segment, provide layerId
await annotationClient.saveSelection(
selection,
true,
true,
layerId
);
}}
onMouseUp={async (e) => {
// on mouse up with Ctrl-key: set active selection end-position to currentPosition
if (
e.ctrlKey &&
annotationClient.activeAnnotation &&
annotationClient.activeSelection &&
annotationClient.activeSelection.end
) {
annotationClient.activeSelection.end = this.currentPosition;
await annotationClient.saveSelection(
annotationClient.activeSelection
);
}
}}
>
+
</div>
);
getUserAnnotationLayerTitle(layerTitle, layerId) {
return (
<EditableText
value={layerTitle}
onChange={async (value) => {
await this.context.annotationClient.segmentLayers.renameLayer(
layerId,
value
);
}}
/>
);
}
getUserSegmentLayer(
segments,
mediaObject,
annotationClient,
layerId,
layerTitle,
duration,
activeAnnotationTypes,
showAnnotationPopup
) {
// Segment buttons
const addSegmentButton = this.getAddSegmentButton(
annotationClient,
layerId
);
const deleteSegmentButton = this.getDeleteSegmentButton(
annotationClient,
layerId
);
const annotateSegmentButton = this.getAnnotateSegmentButton(
annotationClient,
showAnnotationPopup,
layerId
);
// Buttons in layer header
const headerChildren = (
<React.Fragment>
{addSegmentButton}
{deleteSegmentButton}
{annotateSegmentButton}
</React.Fragment>
);
// On layer double click:
// - Add a new segment on Shift+Click
// - Listen to mouse move to set end position
const onLayerDoubleClick = async (position, e) => {
// Check if the double click is on the empty layer
const clickOnEmptyLayer =
e.target.className.indexOf("bg__tl-layer") > -1;
// When not on empty layer; a user segment is clicked
// so just show the Annotation Popup and return
if (!clickOnEmptyLayer) {
showAnnotationPopup();
return;
}
// When clicked in empty layer space; Add a new annotation by dragging:
// set activeAnnotation to null, so a new root annotation is created & activated
annotationClient.activeAnnotation = null;
annotationClient.newTemporalSegment(position, position + 1);
// Check if mouse goes up during async
let cancel = false;
const mouseCancel = () => {
cancel = true;
document.removeEventListener("mouseup", mouseCancel);
};
document.addEventListener("mouseup", mouseCancel);
// Store new segment, include layerId
await annotationClient.saveSelection(
annotationClient.activeSelection,
true,
true,
layerId
);
// Mouse action has been canceled; return
if (cancel) {
return;
}
// Continue editing the new segment
document.removeEventListener("mouseup", mouseCancel);
let lastX = e.pageX;
// Get current segment object in annotation array so we can edit it directly
let segment = null;
annotationClient.annotations.some((annotation) => {
if (
annotation.target &&
annotation.target.type === ANNOTATION_TARGET.SEGMENT &&
annotation.id === annotationClient.activeAnnotation.id
) {
segment = annotation;
return true;
}
return false;
});
if (!segment) {
return;
}
// Set end to start+0.01, so we really start dragging from the original doubleClick position
segment.target.selector.refinedBy.end =
segment.target.selector.refinedBy.start + 0.01;
// Listening to mouse changes and set segment end accordingly
const onMouseMove = (e) => {
if (!segment.target) {
return;
}
const selection = segment.target.selector.refinedBy;
selection.end = this.snapToSegments(
selection.end +
(e.pageX - lastX) / this.getPixelsPerSecond(),
segments,
segment,
layerId
);
// Swap start/end in case of negative length
if (selection.end < selection.start) {
const _start = selection.start;
selection.start = selection.end;
selection.end = _start;
}
selection.start = this.withinRange(selection.start);
selection.end = this.withinRange(selection.end);
lastX = e.pageX;
this.updateAnnotations();
};
const onMouseUp = (e) => {
document.removeEventListener("mousemove", onMouseMove);
document.removeEventListener("mouseup", onMouseUp);
this.updateSegment(segment, segment.target.selector.refinedBy);
};
document.addEventListener("mousemove", onMouseMove);
document.addEventListener("mouseup", onMouseUp);
};
// Return the layer
return {
id: layerId,
title: this.getUserAnnotationLayerTitle(layerTitle, layerId),
className: "user-segment",
headerChildren: headerChildren,
onDoubleClick: onLayerDoubleClick,
sections: segments
.filter((segment) => segment.target.selector.refinedBy)
.sort(
(a, b) =>
a.target.selector.refinedBy.start -
b.target.selector.refinedBy.start
)
.map((segment) => {
// Create title segments (based on active annotations)
const label = getSegmentTitle(
segment,
activeAnnotationTypes,
false
);
const title = getSelectionTitle(segment);
// Return segment data
return {
id:
"tl_user_seg_" +
segment.id +
"_" +
segment.modified,
start: segment.target.selector.refinedBy.start,
end: segment.target.selector.refinedBy.end,
rawData: label.toLowerCase(),
highlight:
annotationClient.activeAnnotation &&
annotationClient.activeAnnotation.id == segment.id,
data: (
<SectionEdit
getPixelsPerSecond={this.getPixelsPerSecond}
updateAnnotationLayer={
this.updateUserAnnotationLayer
}
updatePlayerPos={this.updatePlayerPos}
updateSegment={this.updateSegment}
withinRange={this.withinRange}
snapToSegments={this.snapToSegments}
selectTimelineLayer={this.selectTimelineLayer}
duration={duration}
segment={segment}
mediaObject={mediaObject}
annotationClient={annotationClient}
>
<UserSegmentSection
label={label ? label : title}
title={title}
onClick={
annotationClient.setActiveAnnotation
}
segment={segment}
/>
</SectionEdit>
),
};
}),
};
}
// Build all user layers and update state
buildUserLayers = () => {
// A valid user is required to build the UserLayers
if (!this.hasValidUserAndProject()) {
return;
}
const layers = this.context.annotationClient.segmentLayers.getLayersSorted();
const userLayers = layers.map((layer) =>
this.buildUserLayer(layer.id, layer.title)
);
this.setState({ userLayers });
};
// Build a single user layer with given layerId and title
buildUserLayer(layerId, title) {
const { annotationClient, activeAnnotationTypes } = this.context;
const { mediaObject, duration, showAnnotationPopup } = this.props;
return this.getUserSegmentLayer(
annotationClient.segmentLayers.getSegments(layerId),
mediaObject,
annotationClient,
layerId,
title,
duration,
activeAnnotationTypes,
showAnnotationPopup
);
}
/* ----------------------------- CONTENT LAYERS ------------------------ */
getContentSegmentLayer(mediaObject) {
return {
id: LAYER_CONTENT_SEGMENTS_ID,
title: Strings.TIMELINE_LAYER_CONTENT_SEGMENTS_TITLE,
description: Strings.TIMELINE_LAYER_CONTENT_SEGMENTS_HELP,
className: "content-segment",
sections: mediaObject.segments
? mediaObject.segments
.filter((segment) => !segment.programSegment)
.sort((a, b) => a.start - b.start)
.map((segment, index) => ({
id: "tl_content_seg_" + index,
start: FlexPlayerUtil.timeRelativeToOnAir(
segment.start,
mediaObject
),
end: FlexPlayerUtil.timeRelativeToOnAir(
segment.end,
mediaObject
),
rawData: segment.title
? segment.title.toLowerCase()
: "",
data: <ContentSegmentSection title={segment.title} />,
}))
: [],
};
}
getTranscriptWordCloudLayer = (layerIndex, transcriptType, transcript) => {
if(!transcript || transcript.lines.length === 0) return null;
// ASR wordcloud every %step% seconds
const step = 30; //sec -- if you change this line; you may want to update the description in _Strings.js
const buckets = {};
if (transcript.lines[0].wordTimes) { //for ASR transcripts with timings per word as well
let wordTimes = [];
let words = [];
// collect all words/wordtimes
transcript.lines.forEach(item => {
words = words.concat(item.text.split(" "));
wordTimes = wordTimes.concat(item.wordTimes);
});
// create buckets from wordtimes
wordTimes.forEach((t, index) => {
// relative time
//t = FlexPlayerUtil.timeRelativeToOnAir(t / 1000, mediaObject);
t = t / 1000;
const bucketIndex = Math.floor(t / step);
// create bucket
if (!(bucketIndex in buckets)) {
buckets[bucketIndex] = {
id: "tl_asr_wo_" + bucketIndex,
start: bucketIndex * step,
end: bucketIndex * step + step,
words: [],
};
}
// add word
buckets[bucketIndex].words.push(words[index]);
});
} else { //these transcrips only have a start and end time per block of text
transcript.lines.forEach(item => {
const bucketIndex = Math.floor((item.start / 1000) / step);
if (!(bucketIndex in buckets)) {
buckets[bucketIndex] = {
id: "tl_asr_wo_" + bucketIndex,
start: bucketIndex * step,
end: bucketIndex * step + step,
words: [],
};
}
buckets[bucketIndex].words.push(...item.text.split(" "));
});
}
// create sections from buckets
const bucketSections = Object.values(buckets).map(bucket => {
bucket.rawData = bucket.words.join(" ").toLowerCase();
bucket.data = (
<ASRWordCloudSection
size={20}
words={bucket.words}
onClick={this.showContentAnnotations}
/>
);
return bucket;
});
// create the ASR words layer
return {
id: layerIndex,
title: transcript.title + " " + Strings.TIMELINE_LAYER_ASR_WORDS_TITLE,
description: Strings.TIMELINE_LAYER_ASR_WORDS_HELP,
className: "asr-words",
height: 150,
sections: bucketSections,
};
};
getTranscriptLayer = (layerIndex, transcriptType, transcript) => {
if(!transcript || transcript.lines.length === 0) return null;
const sections = [];
transcript.lines.forEach((line, index) => {
let sectionId = "tl_"+transcriptType+"_se_" + index;
const i = sections.findIndex(x => x.id == sectionId);
if(i <= -1){
sections.push({
id: sectionId,
start: line.start / 1000, //convert to seconds
end: line.end / 1000,
rawData: line.text,
data: (
<ASRSentenceSection
title={line.text}
opacity="90%"
/>
)
});
}
});
return {
id: layerIndex,
title: transcript.title,
description: transcript.title,
className: "asr-sentence",
sections: sections
}
};
buildContentLayers = () => {
const context = this.context;
const { mediaObject } = this.props;
// Default content layers
let contentLayers = [
this.getContentSegmentLayer(mediaObject)
];
// Only apply when using DAAN collection
const activeTranscripts = this.context.getActiveTranscripts();
if(activeTranscripts) {
let layerIndex = -103;
activeTranscripts.forEach(transcript => {
const transcriptLayer = this.getTranscriptLayer(
layerIndex--,
transcript.type,
transcript
);
const transcriptWordCloudLayer = this.getTranscriptWordCloudLayer(
layerIndex--,
transcript.type,
transcript
);
if(transcriptLayer)contentLayers.push(transcriptLayer);
if(transcriptLayer)contentLayers.push(transcriptWordCloudLayer);
});
}
this.setState({
contentLayers: contentLayers
});
};
buildLayers = () => {
this.buildUserLayers();
this.buildContentLayers();
};
renderNewLayerButton = () => {
return (
<div
key="new-layer-button"
className="button-new-layer btn btn-primary"
onClick={
this.context.annotationClient.segmentLayers.addEmptyLayer
}
>
{Strings.BUTTON_ADD_USER_LAYER}
</div>
);
};
hasValidUserAndProject = () => {
return (
this.context.user &&
this.context.user.id !== "ANONYMOUS" &&
this.context.activeProject
);
};
deleteActiveLayer = () => {
const layerId = this.getActiveLayerId();
this.context.annotationClient.segmentLayers.deleteLayerWithCheck(
layerId
);
};
renderDeleteLayerButton = () => {
return (
<div
onClick={this.deleteActiveLayer}
className="delete-layer-button"
key="delete-layer-button"
title={Strings.BUTTON_DELETE_ACTIVE_USER_LAYER_HELP}
/>
);
};
getTimelineActions = () => {
if (this.hasValidUserAndProject()) {
return (
<React.Fragment>
{this.renderNewLayerButton()}
{this.renderDeleteLayerButton()}
</React.Fragment>
);
}
return null;
};
render = () => {
const { mediaObject, duration } = this.props;
// Combine layers
const layers = [
...this.state.userLayers,
...this.state.contentLayers.filter(
(layer) => layer && layer.sections.length
),
];
// If there are no layers, and no logged in user, don't display the timeline
if (layers.length == 0 && !this.hasValidUserAndProject()) {
return null;
}
const actions = this.getTimelineActions();
// render timeline
return (
<Timeline
key={mediaObject.assetId}
ref={this.timelineRef}
start={FlexPlayerUtil.timeRelativeToOnAir(0, mediaObject)}
end={duration}
viewStart={0}
viewEnd={
// Always show full first segment, or at least 60 sec; at max 'duration' seconds
Math.min(
duration,
Math.max(
60,
layers.length > 0 && layers[0].sections.length > 0
? layers[0].sections[0].end
: 60
)
)
}
layers={layers}
setPosition={this.updatePlayerPos}
actions={actions}
/>
);
};
}
TimelineView.propTypes = {
mediaObject: MediaObject.getPropTypes(true), // Current mediaobject
duration: PropTypes.number.isRequired, // duration of media content,
activeAnnotationTypes: PropTypes.arrayOf(PropTypes.string),
};