diff --git a/www/annotate.js b/www/annotate.js index 7ce4b77..bd10965 100644 --- a/www/annotate.js +++ b/www/annotate.js @@ -109,8 +109,10 @@ class StrokeSlice { } } -class Annotator { +class Annotator extends EventTarget { constructor(wrapperEl, tags, fileurl) { + super(); + this.wrapperEl = wrapperEl; this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); this.wrapperEl.appendChild(this.svgEl); @@ -120,13 +122,13 @@ class Annotator { this.controlsEl.classList.add('controls') this.wrapperEl.appendChild(this.controlsEl); - this.scrubberElOld = document.createElement('input'); - this.scrubberElOld.type = "range"; - this.scrubberElOld.min = 0; - this.scrubberElOld.step = 0.01; - this.controlsEl.appendChild(this.scrubberElOld); + this.playheadEl = document.createElement('input'); + this.playheadEl.type = "range"; + this.playheadEl.min = 0; + this.playheadEl.step = 0.01; + this.controlsEl.appendChild(this.playheadEl); - this.scrubberElOld.addEventListener("input", (ev) => { + this.playheadEl.addEventListener("input", (ev) => { this.scrubTo(ev.target.value); }) @@ -170,9 +172,11 @@ class Annotator { this.controlsEl.appendChild(this.annotationsEl); - this.inPointPosition = null; + this.inPointPosition = [0, 0]; + this.inPointTimeMs = null; this.outPointPosition = null; - this.currentTime = 0; + this.outPointTimeMs = null; + this._currentTimeMs = 0; this.isPlaying = false; const groups = ['before', 'annotation', 'after'] @@ -186,7 +190,7 @@ class Annotator { this.annotations = []; - this.play(fileurl); + this.load(fileurl); } updateAnnotations(save) { @@ -195,8 +199,8 @@ class Annotator { for (let annotation_i in this.annotations) { const annotation = this.annotations[annotation_i]; this.annotationEl = document.createElement('div'); - const left = (annotation.t_in / this.duration) * 100; - const right = 100 - (annotation.t_out / this.duration) * 100; + const left = (annotation.t_in / this.lastFrameTime) * 100; + const right = 100 - (annotation.t_out / this.lastFrameTime) * 100; this.annotationEl.style.left = left + '%'; this.annotationEl.style.right = right + '%'; @@ -253,7 +257,7 @@ class Annotator { deselectAnnotation(keep_position) { if (this.selectedAnnotation) - this.currentTime = this.selectedAnnotation.t_out; + this._currentTimeMs = this.selectedAnnotation.t_out; this.wrapperEl.classList.remove('selected-annotation'); @@ -266,7 +270,7 @@ class Annotator { this.updateAnnotations(false); // selects the right tag & highlights the annotation } - play(file) { + load(file) { const request = new Request(file, { method: 'GET', }); @@ -280,7 +284,7 @@ class Annotator { fetch(metadata_req) .then(response => response.ok ? response.json() : null) .then(metadata => { - this.playStrokes(data, metadata) + this.loadStrokes(data, metadata) }) .catch(e => console.log(e)); // do something with the data sent in the request @@ -362,26 +366,33 @@ class Annotator { this.annotations.push(new Annotation(tag, t_in, t_out)); this.updateAnnotations(true); - this.currentTime = t_out; + this._currentTimeMs = t_out; this.setUpAnnotator(); } } setUpAnnotator() { + this.playheadEl.min = this.audioOffset < 0 ? this.audioOffset * 1000 : 0; + this.playheadEl.max = this.getEndTimeMs(); + this.playheadEl.value = this._currentTimeMs; + this.inPointPosition = this.findPositionForTime(this.currentTime); - this.outPointPosition = this.findPositionForTime(this.duration); + this.inPointTimeMs = this._currentTimeMs; + this.outPointPosition = this.findPositionForTime(this.lastFrameTime); // TODO: simplify to get the last frame indexes directly + this.outPointTimeMs = null; if (this.scrubberEl.noUiSlider) { this.slider.destroy(); } + // console.log(this._currentTimeMs, ) this.slider = noUiSlider.create(this.scrubberEl, { - start: [this.currentTime, this.duration], + start: [this._currentTimeMs, this.lastFrameTime], connect: true, range: { 'min': this.audioOffset < 0 ? this.audioOffset * 1000 : 0, - 'max': this.duration + 'max': this.getEndTimeMs(), }, tooltips: [ this.formatter, @@ -397,7 +408,9 @@ class Annotator { this.slider.on("slide", (values, handle) => { this.isPlaying = false; this.inPointPosition = this.findPositionForTime(values[0]); + this.inPointTimeMs = Number.parseFloat(values[0]); this.outPointPosition = this.findPositionForTime(values[1]); + this.outPointTimeMs = Number.parseFloat(values[1]); this.drawStrokePosition(this.inPointPosition, this.outPointPosition); // console.log(this.selectedAnnotation); @@ -417,13 +430,13 @@ class Annotator { this.drawStrokePosition(this.inPointPosition, this.outPointPosition); } - playStrokes(drawing, metadata) { + loadStrokes(drawing, metadata) { this.audioOffset = 0; if (metadata) { this.annotations = metadata.annotations; this.audioFile = metadata.hasOwnProperty('audio') ? metadata.audio.file : null; this.audioOffset = metadata.hasOwnProperty('audio') ? Number.parseFloat(metadata.audio.offset) : 0; - this.currentTime = this.audioOffset < 0 ? this.audioOffset * 1000 : 0; + this._currentTimeMs = this.audioOffset < 0 ? this.audioOffset * 1000 : 0; // // load any saved metadata } @@ -442,16 +455,16 @@ class Annotator { bgEl.classList.add('background'); this.svgEl.prepend(bgEl); - this.startTime = window.performance.now() - this.strokes[0].points[0][3]; - this.duration = this.getDuration(); - this.scrubberElOld.max = this.duration; - this.playTimout = null; + this.firstFrameTime = this.strokes[0].points[0][3]; + this.lastFrameTime = this.getFinalFrameTime(); + this.playheadEl.max = this.lastFrameTime; + this.nextFrameTimeout = null; this.formatter = wNumb({ decimals: 2, edit: (time) => { let neg = ""; - if(time < 0) { + if (time < 0) { neg = "-"; time *= -1; } @@ -463,68 +476,82 @@ class Annotator { } }); - this.setUpAnnotator() - this.updateAnnotations(false); - this.setupAudioConfig(); + this.setupAudioConfig().then(() => { + // this.setUpAnnotator() + this.updateAnnotations(false); + }); // this.playStrokePosition(0, 1); } setupAudioConfig() { // audio config - let audioConfigEl = document.createElement('div'); - audioConfigEl.classList.add('audioconfig') - this.wrapperEl.appendChild(audioConfigEl); + return new Promise((resolve, reject) => { - let audioSelectEl = document.createElement('select'); - audioSelectEl.classList.add('audioselect'); - audioConfigEl.appendChild(audioSelectEl); + let audioConfigEl = document.createElement('div'); + audioConfigEl.classList.add('audioconfig') + this.wrapperEl.appendChild(audioConfigEl); - fetch('/audio') - .then(response => response.json()) - .then(data => { - data.unshift(''); // add empty, to deselect any file - data.forEach(audioFile => { - let optionEl = document.createElement('option'); - optionEl.selected = this.audioFile == audioFile; - optionEl.innerText = audioFile; - audioSelectEl.appendChild(optionEl); - }); + let audioSelectEl = document.createElement('select'); + audioSelectEl.classList.add('audioselect'); + audioConfigEl.appendChild(audioSelectEl); + + fetch('/audio') + .then(response => response.json()) + .then(data => { + data.unshift(''); // add empty, to deselect any file + data.forEach(audioFile => { + let optionEl = document.createElement('option'); + optionEl.selected = this.audioFile == audioFile; + optionEl.innerText = audioFile; + audioSelectEl.appendChild(optionEl); + }); + }) + + audioSelectEl.addEventListener('change', (ev) => { + this.setAudioFile(ev.target.value); + }); + + + let audioOffsetTextEl = document.createElement('label'); + audioOffsetTextEl.innerText = "Offset (s)"; + audioConfigEl.appendChild(audioOffsetTextEl); + + let audioOffsetEl = document.createElement('input'); + audioOffsetEl.setAttribute('type', 'number'); + audioOffsetEl.setAttribute('step', '.01'); + audioOffsetEl.value = this.audioOffset ?? 0; + audioOffsetEl.addEventListener('change', (ev) => { + this.setAudioOffset(ev.target.value); + }); + audioOffsetTextEl.appendChild(audioOffsetEl); + + + this.audioEl = document.createElement('audio'); + this.audioEl.setAttribute('controls', true); + this.audioEl.addEventListener('canplaythrough', (ev) => { + console.log('loaded audio', ev); + this.audioEl.play(); + }); + // this.audioEl.addEventListener('seeked', (ev)=>{ + // console.log(ev); + // }) + audioConfigEl.prepend(this.audioEl); + + this.audioEl.addEventListener('loadedmetadata', (ev) => { + // resolve the 'set up audio' when metadata has loaded + this.setUpAnnotator(); // if offset is negative, annotator starts at negative time + resolve(); }) - - audioSelectEl.addEventListener('change', (ev) => { - this.setAudioFile(ev.target.value); + if (this.audioFile) { + this.audioEl.setAttribute('src', this.audioFile); + } else { + this.setUpAnnotator(); + resolve(); + } + }); - - - let audioOffsetTextEl = document.createElement('label'); - audioOffsetTextEl.innerText = "Offset (s)"; - audioConfigEl.appendChild(audioOffsetTextEl); - - let audioOffsetEl = document.createElement('input'); - audioOffsetEl.setAttribute('type', 'number'); - audioOffsetEl.setAttribute('step', '.01'); - audioOffsetEl.value = this.audioOffset ?? 0; - audioOffsetEl.addEventListener('change', (ev) => { - this.setAudioOffset(ev.target.value); - }); - audioOffsetTextEl.appendChild(audioOffsetEl); - - - this.audioEl = document.createElement('audio'); - if (this.audioFile) { - this.audioEl.setAttribute('src', this.audioFile); - } - this.audioEl.setAttribute('controls', true); - this.audioEl.addEventListener('canplaythrough', (ev) => { - console.log('loaded audio', ev); - this.audioEl.play(); - }); - // this.audioEl.addEventListener('seeked', (ev)=>{ - // console.log(ev); - // }) - audioConfigEl.prepend(this.audioEl); } setAudioFile(audioFile) { @@ -563,7 +590,7 @@ class Annotator { // TODO, handle playback delay const t_start = this.getAudioTime(t_in); // in ms - const t_diff = t_out - t_in; // in ms + const t_diff = (t_out ?? this.audioEl.duration * 1000) - t_in; // in ms this.audioEl.pause(); @@ -580,13 +607,13 @@ class Annotator { } else { this.audioEl.currentTime = t_start / 1000; // this.audioEl.play(); // play is done in "seeked" evenlistener - console.log(this.audioEl.currentTime, t_start, t_in, t_out) + console.log(this.audioEl.currentTime, t_start, t_in, t_out); } this.audioEndTimeout = setTimeout((e) => this.audioEl.pause(), t_diff); } - getDuration() { + getFinalFrameTime() { const points = this.strokes[this.strokes.length - 1].points; return points[points.length - 1][3]; } @@ -692,7 +719,7 @@ class Annotator { } else { this.isPlaying = true; } - this.drawStrokePosition(path_i, point_i); + this.drawStrokePosition(this.inPointPosition, [path_i, point_i]); const [next_path, next_point] = this.getNextPosition(path_i, point_i); if (next_path === null) { @@ -702,26 +729,136 @@ class Annotator { const t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3]; - const dt = t - (window.performance.now() - this.startTime); - this.playTimout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt); - } - - playUntil(path_i) { - // for scrubber + // calculate interval based on playback start to avoid drifting of time + const dt = t - (window.performance.now() - this.startTimeMs); + this.nextFrameTimeout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt); } scrubTo(ms) { - const [path_i, point_i] = this.findPositionForTime(ms); + // const [path_i, point_i] = this.findPositionForTime(ms); // console.log(path_i, point_i); - clearTimeout(this.playTimout); - this.playStrokePosition(path_i, point_i); + this.pause(); + this._seekByTime(ms / 1000); // this.playHead = ms; } + /** + * Compatibility with HTMLMediaElement API + * @returns None + */ + pause() { + this._interruptPlayback(); + } + _interruptPlayback() { + clearTimeout(this.nextFrameTimeout); + clearTimeout(this.audioEndTimeout); + clearTimeout(this.audioStartTimeout); + clearTimeout(this.startVideoTimeout); + this.audioEl.pause(); + this.isPlaying = false; + } + + /** + * Compatibility with HTMLMediaElement API + * @returns Promise + */ + play() { + return new Promise((resolve, reject) => { + this._interruptPlayback(); + + this.startTimeMs = window.performance.now() - this._currentTimeMs; + + if (this._currentTimeMs < 0) { + this.startVideoTimeout = setTimeout((e) => this.playStrokePosition(this.currentPathI, this.currentPointI), this._currentTimeMs * -1); + } else { + this.playStrokePosition(this.currentPathI, this.currentPointI); + } + console.log(this._currentTimeMs, this.outPointTimeMs); + this.playAudioSegment(this._currentTimeMs, this.outPointTimeMs); + // this.playStrokePosition(this.currentPathI, this.currentPointI); + this.dispatchEvent(new CustomEvent('play', {})); + this._animationFrame(); + resolve(); + }); + } + + _animationFrame(timestamp) { + // TODO, move time at end of playStrokePosition to here + const nextTime = window.performance.now() - this.startTimeMs; + const endTime = this.outPointTimeMs ?? this.duration * 1000; + if (nextTime > this.duration * 1000) { + + } + this.playheadEl.value = this._currentTimeMs; + if (this.isPlaying) { + window.requestAnimationFrame((timestamp) => this._animationFrame(timestamp)); + } + } + + /** + * Note that both t_in and t_out can be negative + * @param float|Array t_in in point time, in ms or array with path/frame points + * @param float|Array t_out out point time, in ms or array with path/frame points + */ + playSegment(in_point, out_point) { + if (!Array.isArray(in_point)) in_point = this.findPositionForTime(in_point); + if (!Array.isArray(out_point)) out_point = this.findPositionForTime(out_point); + + this.inPointPosition = in_point; + this.outPointPosition = out_point; + this._seekByPoint(in_point); + + this.play(); + } + + _seekByPoint(point) { + this.dispatchEvent(new CustomEvent('seeking', {})); + this._currentTimeMs = this.strokes[point[0]].points[point[1]][2]; + [this.currentPathI, this.currentPointI] = point; + this._updateFrame(); + // TODO set audio, wait for promise to finish + this.dispatchEvent(new CustomEvent('seeked', {})); + + } + _seekByTime(time) { + this.dispatchEvent(new CustomEvent('seeking', { time: time })); + this._currentTimeMs = Number.parseFloat(time) * 1000; + [this.currentPathI, this.currentPointI] = this.findPositionForTime(this._currentTimeMs); + this._updateFrame(); + this.dispatchEvent(new CustomEvent('seeked', { time: this.currentTime })); + } + + _updateFrame() { + this.drawStrokePosition(this.inPointPosition, [this.currentPathI, this.currentPointI]); + } + + /** + * For compatibility with HTMLMediaElement API convert seconds to ms of internal timer + */ + set currentTime(time) { + this._seekByTime(time); + } + + get currentTime() { + return this._currentTimeMs / 1000; + } + + getEndTimeMs() { + const videoDuration = this.getFinalFrameTime(); + const audioDuration = (this.audioEl) ? this.audioEl.duration + this.audioOffset : 0; + return Math.max(videoDuration, audioDuration * 1000); + } + + get duration() { + + const prerollDuration = this.audioOffset < 0 ? this.audioOffset * -1 : 0; + + return prerollDuration + this.getEndTimeMs(); + } findPositionForTime(ms) { - ms = Math.min(Math.max(ms, 0), this.duration); + ms = Math.min(Math.max(ms, 0), this.lastFrameTime); // console.log('scrub to', ms) let path_i = 0; let point_i = 0;