WIP playback which is handling audio pre-/postroll

www/annotate.js

@@ -109,8 +109,10 @@ class StrokeSlice {
     }
 }
 
-class Annotator {
+class Annotator extends EventTarget {
     constructor(wrapperEl, tags, fileurl) {
+        super();
+
         this.wrapperEl = wrapperEl;
         this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
         this.wrapperEl.appendChild(this.svgEl);
@@ -120,13 +122,13 @@ class Annotator {
         this.controlsEl.classList.add('controls')
         this.wrapperEl.appendChild(this.controlsEl);
 
-        this.scrubberElOld = document.createElement('input');
+        this.playheadEl = document.createElement('input');
-        this.scrubberElOld.type = "range";
+        this.playheadEl.type = "range";
-        this.scrubberElOld.min = 0;
+        this.playheadEl.min = 0;
-        this.scrubberElOld.step = 0.01;
+        this.playheadEl.step = 0.01;
-        this.controlsEl.appendChild(this.scrubberElOld);
+        this.controlsEl.appendChild(this.playheadEl);
 
-        this.scrubberElOld.addEventListener("input", (ev) => {
+        this.playheadEl.addEventListener("input", (ev) => {
             this.scrubTo(ev.target.value);
         })
 
@@ -170,9 +172,11 @@ class Annotator {
         this.controlsEl.appendChild(this.annotationsEl);
 
 
-        this.inPointPosition = null;
+        this.inPointPosition = [0, 0];
+        this.inPointTimeMs = null;
         this.outPointPosition = null;
-        this.currentTime = 0;
+        this.outPointTimeMs = null;
+        this._currentTimeMs = 0;
         this.isPlaying = false;
 
         const groups = ['before', 'annotation', 'after']
@@ -186,7 +190,7 @@ class Annotator {
 
         this.annotations = [];
 
-        this.play(fileurl);
+        this.load(fileurl);
     }
 
     updateAnnotations(save) {
@@ -195,8 +199,8 @@ class Annotator {
         for (let annotation_i in this.annotations) {
             const annotation = this.annotations[annotation_i];
             this.annotationEl = document.createElement('div');
-            const left = (annotation.t_in / this.duration) * 100;
+            const left = (annotation.t_in / this.lastFrameTime) * 100;
-            const right = 100 - (annotation.t_out / this.duration) * 100;
+            const right = 100 - (annotation.t_out / this.lastFrameTime) * 100;
             this.annotationEl.style.left = left + '%';
             this.annotationEl.style.right = right + '%';
 
@@ -253,7 +257,7 @@ class Annotator {
 
     deselectAnnotation(keep_position) {
         if (this.selectedAnnotation)
-            this.currentTime = this.selectedAnnotation.t_out;
+            this._currentTimeMs = this.selectedAnnotation.t_out;
 
         this.wrapperEl.classList.remove('selected-annotation');
 
@@ -266,7 +270,7 @@ class Annotator {
         this.updateAnnotations(false); // selects the right tag & highlights the annotation
     }
 
-    play(file) {
+    load(file) {
         const request = new Request(file, {
             method: 'GET',
         });
@@ -280,7 +284,7 @@ class Annotator {
                 fetch(metadata_req)
                     .then(response => response.ok ? response.json() : null)
                     .then(metadata => {
-                        this.playStrokes(data, metadata)
+                        this.loadStrokes(data, metadata)
                     })
                     .catch(e => console.log(e));
                 // do something with the data sent in the request
@@ -362,26 +366,33 @@ class Annotator {
             this.annotations.push(new Annotation(tag, t_in, t_out));
             this.updateAnnotations(true);
 
-            this.currentTime = t_out;
+            this._currentTimeMs = t_out;
             this.setUpAnnotator();
         }
     }
 
 
     setUpAnnotator() {
+        this.playheadEl.min = this.audioOffset < 0 ? this.audioOffset * 1000 : 0;
+        this.playheadEl.max = this.getEndTimeMs();
+        this.playheadEl.value = this._currentTimeMs;
+
         this.inPointPosition = this.findPositionForTime(this.currentTime);
-        this.outPointPosition = this.findPositionForTime(this.duration);
+        this.inPointTimeMs = this._currentTimeMs;
+        this.outPointPosition = this.findPositionForTime(this.lastFrameTime); // TODO: simplify to get the last frame indexes directly
+        this.outPointTimeMs = null;
 
         if (this.scrubberEl.noUiSlider) {
             this.slider.destroy();
         }
 
+        // console.log(this._currentTimeMs, )
         this.slider = noUiSlider.create(this.scrubberEl, {
-            start: [this.currentTime, this.duration],
+            start: [this._currentTimeMs, this.lastFrameTime],
             connect: true,
             range: {
                 'min': this.audioOffset < 0 ? this.audioOffset * 1000 : 0,
-                'max': this.duration
+                'max': this.getEndTimeMs(),
             },
             tooltips: [
                 this.formatter,
@@ -397,7 +408,9 @@ class Annotator {
         this.slider.on("slide", (values, handle) => {
             this.isPlaying = false;
             this.inPointPosition = this.findPositionForTime(values[0]);
+            this.inPointTimeMs = Number.parseFloat(values[0]);
             this.outPointPosition = this.findPositionForTime(values[1]);
+            this.outPointTimeMs = Number.parseFloat(values[1]);
             this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
 
             // console.log(this.selectedAnnotation);
@@ -417,13 +430,13 @@ class Annotator {
         this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
     }
 
-    playStrokes(drawing, metadata) {
+    loadStrokes(drawing, metadata) {
         this.audioOffset = 0;
         if (metadata) {
             this.annotations = metadata.annotations;
             this.audioFile = metadata.hasOwnProperty('audio') ? metadata.audio.file : null;
             this.audioOffset = metadata.hasOwnProperty('audio') ? Number.parseFloat(metadata.audio.offset) : 0;
-            this.currentTime = this.audioOffset < 0 ? this.audioOffset * 1000 : 0;
+            this._currentTimeMs = this.audioOffset < 0 ? this.audioOffset * 1000 : 0;
             //
             // load any saved metadata
         }
@@ -442,16 +455,16 @@ class Annotator {
         bgEl.classList.add('background');
         this.svgEl.prepend(bgEl);
 
-        this.startTime = window.performance.now() - this.strokes[0].points[0][3];
+        this.firstFrameTime = this.strokes[0].points[0][3];
-        this.duration = this.getDuration();
+        this.lastFrameTime = this.getFinalFrameTime();
-        this.scrubberElOld.max = this.duration;
+        this.playheadEl.max = this.lastFrameTime;
-        this.playTimout = null;
+        this.nextFrameTimeout = null;
 
         this.formatter = wNumb({
             decimals: 2,
             edit: (time) => {
                 let neg = "";
-                if(time < 0) {
+                if (time < 0) {
                     neg = "-";
                     time *= -1;
                 }
@@ -463,68 +476,82 @@ class Annotator {
             }
         });
 
-        this.setUpAnnotator()
-        this.updateAnnotations(false);
 
-        this.setupAudioConfig();
+        this.setupAudioConfig().then(() => {
+            // this.setUpAnnotator()
+            this.updateAnnotations(false);
+        });
 
         // this.playStrokePosition(0, 1);
     }
 
     setupAudioConfig() {
         // audio config
-        let audioConfigEl = document.createElement('div');
+        return new Promise((resolve, reject) => {
-        audioConfigEl.classList.add('audioconfig')
-        this.wrapperEl.appendChild(audioConfigEl);
 
-        let audioSelectEl = document.createElement('select');
+            let audioConfigEl = document.createElement('div');
-        audioSelectEl.classList.add('audioselect');
+            audioConfigEl.classList.add('audioconfig')
-        audioConfigEl.appendChild(audioSelectEl);
+            this.wrapperEl.appendChild(audioConfigEl);
 
-        fetch('/audio')
+            let audioSelectEl = document.createElement('select');
-            .then(response => response.json())
+            audioSelectEl.classList.add('audioselect');
-            .then(data => {
+            audioConfigEl.appendChild(audioSelectEl);
-                data.unshift(''); // add empty, to deselect any file
+
-                data.forEach(audioFile => {
+            fetch('/audio')
-                    let optionEl = document.createElement('option');
+                .then(response => response.json())
-                    optionEl.selected = this.audioFile == audioFile;
+                .then(data => {
-                    optionEl.innerText = audioFile;
+                    data.unshift(''); // add empty, to deselect any file
-                    audioSelectEl.appendChild(optionEl);
+                    data.forEach(audioFile => {
-                });
+                        let optionEl = document.createElement('option');
+                        optionEl.selected = this.audioFile == audioFile;
+                        optionEl.innerText = audioFile;
+                        audioSelectEl.appendChild(optionEl);
+                    });
+                })
+
+            audioSelectEl.addEventListener('change', (ev) => {
+                this.setAudioFile(ev.target.value);
+            });
+
+
+            let audioOffsetTextEl = document.createElement('label');
+            audioOffsetTextEl.innerText = "Offset (s)";
+            audioConfigEl.appendChild(audioOffsetTextEl);
+
+            let audioOffsetEl = document.createElement('input');
+            audioOffsetEl.setAttribute('type', 'number');
+            audioOffsetEl.setAttribute('step', '.01');
+            audioOffsetEl.value = this.audioOffset ?? 0;
+            audioOffsetEl.addEventListener('change', (ev) => {
+                this.setAudioOffset(ev.target.value);
+            });
+            audioOffsetTextEl.appendChild(audioOffsetEl);
+
+
+            this.audioEl = document.createElement('audio');
+            this.audioEl.setAttribute('controls', true);
+            this.audioEl.addEventListener('canplaythrough', (ev) => {
+                console.log('loaded audio', ev);
+                this.audioEl.play();
+            });
+            // this.audioEl.addEventListener('seeked', (ev)=>{
+            //     console.log(ev);
+            // })
+            audioConfigEl.prepend(this.audioEl);
+
+            this.audioEl.addEventListener('loadedmetadata', (ev) => {
+                // resolve the 'set up audio' when metadata has loaded
+                this.setUpAnnotator(); // if offset is negative, annotator starts at negative time
+                resolve();
             })
+            if (this.audioFile) {
+                this.audioEl.setAttribute('src', this.audioFile);
+            } else {
+                this.setUpAnnotator();
+                resolve();
+            }
             
-        audioSelectEl.addEventListener('change', (ev) => {
-            this.setAudioFile(ev.target.value);
         });
-
-
-        let audioOffsetTextEl = document.createElement('label');
-        audioOffsetTextEl.innerText = "Offset (s)";
-        audioConfigEl.appendChild(audioOffsetTextEl);
-
-        let audioOffsetEl = document.createElement('input');
-        audioOffsetEl.setAttribute('type', 'number');
-        audioOffsetEl.setAttribute('step', '.01');
-        audioOffsetEl.value = this.audioOffset ?? 0;
-        audioOffsetEl.addEventListener('change', (ev) => {
-            this.setAudioOffset(ev.target.value);
-        });
-        audioOffsetTextEl.appendChild(audioOffsetEl);
-
-
-        this.audioEl = document.createElement('audio');
-        if (this.audioFile) {
-            this.audioEl.setAttribute('src', this.audioFile);
-        }
-        this.audioEl.setAttribute('controls', true);
-        this.audioEl.addEventListener('canplaythrough', (ev) => {
-            console.log('loaded audio', ev);
-            this.audioEl.play();
-        });
-        // this.audioEl.addEventListener('seeked', (ev)=>{
-        //     console.log(ev);
-        // })
-        audioConfigEl.prepend(this.audioEl);
     }
 
     setAudioFile(audioFile) {
@@ -563,7 +590,7 @@ class Annotator {
 
         // TODO, handle playback delay
         const t_start = this.getAudioTime(t_in); // in ms
-        const t_diff = t_out - t_in; // in ms
+        const t_diff = (t_out ?? this.audioEl.duration * 1000) - t_in; // in ms
 
         this.audioEl.pause();
 
@@ -580,13 +607,13 @@ class Annotator {
         } else {
             this.audioEl.currentTime = t_start / 1000;
             // this.audioEl.play(); // play is done in "seeked" evenlistener
-            console.log(this.audioEl.currentTime, t_start, t_in, t_out)
+            console.log(this.audioEl.currentTime, t_start, t_in, t_out);
         }
 
         this.audioEndTimeout = setTimeout((e) => this.audioEl.pause(), t_diff);
     }
 
-    getDuration() {
+    getFinalFrameTime() {
         const points = this.strokes[this.strokes.length - 1].points;
         return points[points.length - 1][3];
     }
@@ -692,7 +719,7 @@ class Annotator {
         } else {
             this.isPlaying = true;
         }
-        this.drawStrokePosition(path_i, point_i);
+        this.drawStrokePosition(this.inPointPosition, [path_i, point_i]);
 
         const [next_path, next_point] = this.getNextPosition(path_i, point_i);
         if (next_path === null) {
@@ -702,26 +729,136 @@ class Annotator {
 
         const t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3];
 
-        const dt = t - (window.performance.now() - this.startTime);
+        // calculate interval based on playback start to avoid drifting of time
-        this.playTimout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt);
+        const dt = t - (window.performance.now() - this.startTimeMs);
-    }
+        this.nextFrameTimeout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt);
-
-    playUntil(path_i) {
-        // for scrubber
     }
 
     scrubTo(ms) {
-        const [path_i, point_i] = this.findPositionForTime(ms);
+        // const [path_i, point_i] = this.findPositionForTime(ms);
         // console.log(path_i, point_i);
-        clearTimeout(this.playTimout);
+        this.pause();
-        this.playStrokePosition(path_i, point_i);
+        this._seekByTime(ms / 1000);
         // this.playHead = ms;
     }
 
+    /**
+     * Compatibility with HTMLMediaElement API
+     * @returns None
+     */
+    pause() {
+        this._interruptPlayback();
+    }
 
+    _interruptPlayback() {
+        clearTimeout(this.nextFrameTimeout);
+        clearTimeout(this.audioEndTimeout);
+        clearTimeout(this.audioStartTimeout);
+        clearTimeout(this.startVideoTimeout);
+        this.audioEl.pause();
+        this.isPlaying = false;
+    }
+
+    /**
+     * Compatibility with HTMLMediaElement API
+     * @returns Promise
+     */
+    play() {
+        return new Promise((resolve, reject) => {
+            this._interruptPlayback();
+
+            this.startTimeMs = window.performance.now() - this._currentTimeMs;
+
+            if (this._currentTimeMs < 0) {
+                this.startVideoTimeout = setTimeout((e) => this.playStrokePosition(this.currentPathI, this.currentPointI), this._currentTimeMs * -1);
+            } else {
+                this.playStrokePosition(this.currentPathI, this.currentPointI);
+            }
+            console.log(this._currentTimeMs, this.outPointTimeMs);
+            this.playAudioSegment(this._currentTimeMs, this.outPointTimeMs);
+            // this.playStrokePosition(this.currentPathI, this.currentPointI);
+            this.dispatchEvent(new CustomEvent('play', {}));
+            this._animationFrame();
+            resolve();
+        });
+    }
+
+    _animationFrame(timestamp) {
+        // TODO, move time at end of playStrokePosition to here
+        const nextTime = window.performance.now() - this.startTimeMs;
+        const endTime = this.outPointTimeMs ?? this.duration * 1000;
+        if (nextTime > this.duration * 1000) {
+
+        }
+        this.playheadEl.value = this._currentTimeMs;
+        if (this.isPlaying) {
+            window.requestAnimationFrame((timestamp) => this._animationFrame(timestamp));
+        }
+    }
+
+    /**
+     * Note that both t_in and t_out can be negative
+     * @param float|Array t_in in point time, in ms  or array with path/frame points
+     * @param float|Array t_out out point time, in ms or array with path/frame points
+     */
+    playSegment(in_point, out_point) {
+        if (!Array.isArray(in_point)) in_point = this.findPositionForTime(in_point);
+        if (!Array.isArray(out_point)) out_point = this.findPositionForTime(out_point);
+
+        this.inPointPosition = in_point;
+        this.outPointPosition = out_point;
+        this._seekByPoint(in_point);
+
+        this.play();
+    }
+
+    _seekByPoint(point) {
+        this.dispatchEvent(new CustomEvent('seeking', {}));
+        this._currentTimeMs = this.strokes[point[0]].points[point[1]][2];
+        [this.currentPathI, this.currentPointI] = point;
+        this._updateFrame();
+        // TODO set audio, wait for promise to finish
+        this.dispatchEvent(new CustomEvent('seeked', {}));
+
+    }
+    _seekByTime(time) {
+        this.dispatchEvent(new CustomEvent('seeking', { time: time }));
+        this._currentTimeMs = Number.parseFloat(time) * 1000;
+        [this.currentPathI, this.currentPointI] = this.findPositionForTime(this._currentTimeMs);
+        this._updateFrame();
+        this.dispatchEvent(new CustomEvent('seeked', { time: this.currentTime }));
+    }
+
+    _updateFrame() {
+        this.drawStrokePosition(this.inPointPosition, [this.currentPathI, this.currentPointI]);
+    }
+
+    /**
+     * For compatibility with HTMLMediaElement API convert seconds to ms of internal timer
+     */
+    set currentTime(time) {
+        this._seekByTime(time);
+    }
+
+    get currentTime() {
+        return this._currentTimeMs / 1000;
+    }
+
+    getEndTimeMs() {
+        const videoDuration = this.getFinalFrameTime();
+        const audioDuration = (this.audioEl) ? this.audioEl.duration + this.audioOffset : 0;
+        return Math.max(videoDuration, audioDuration * 1000);
+    }
+
+    get duration() {
+
+        const prerollDuration = this.audioOffset < 0 ? this.audioOffset * -1 : 0;
+
+        return prerollDuration + this.getEndTimeMs();
+    }
 
     findPositionForTime(ms) {
-        ms = Math.min(Math.max(ms, 0), this.duration);
+        ms = Math.min(Math.max(ms, 0), this.lastFrameTime);
         // console.log('scrub to', ms)
         let path_i = 0;
         let point_i = 0;