class Annotation { constructor(tag, t_in, t_out) { this.tag = tag; this.t_in = t_in; this.t_out = t_out; } } class StrokeGroup { constructor(group_element, player) { this.g = group_element; this.player = player; } setStrokes(strokes) { const pathEls = this.g.querySelectorAll('path'); let indexes = Object.keys(strokes); for (let pathEl of pathEls) { const i = pathEl.dataset.path_i; if (!indexes.includes(pathEl.dataset.path_i)) { pathEl.parentNode.removeChild(pathEl); } else { // check in and outpoint using pathEl.dataset if (strokes[i].getSliceId() != pathEl.dataset.slice) { const d = this.points2D(strokes[i].points); pathEl.dataset.slice = strokes[i].getSliceId(); pathEl.setAttribute('d', d); } } // this has now been processed indexes.splice(indexes.indexOf(i), 1); } // new strokes indexes.forEach(index => { const stroke = strokes[index]; let pathEl = document.createElementNS('http://www.w3.org/2000/svg', 'path'); pathEl.style.stroke = stroke.color; pathEl.classList.add('path'); pathEl.dataset.path_i = index; pathEl.dataset.slice = stroke.getSliceId(); this.g.appendChild(pathEl); const d = this.points2D(stroke.points); pathEl.setAttribute('d', d); }); } // convert array of points to a d-attribute points2D(strokes) { // strokes to a d attribute for a path let d = ""; let last_stroke = undefined; let cmd = ""; for (let stroke of strokes) { if (!last_stroke) { d += `M${stroke[0] * this.player.dimensions[0]},${stroke[1] * this.player.dimensions[1]} `; cmd = 'M'; } else { if (last_stroke[2] == 1) { d += " m"; cmd = 'm'; } else if (cmd != 'l') { d += ' l '; cmd = 'l'; } let rel_stroke = [stroke[0] - last_stroke[0], stroke[1] - last_stroke[1]]; d += `${rel_stroke[0] * this.player.dimensions[0]},${rel_stroke[1] * this.player.dimensions[1]} `; } last_stroke = stroke; } return d; } } class Stroke { constructor(color, points) { this.color = color; this.points = points; // [[x1,y1,t1], [x2,y2,t2], ...] } getSliceId() { return 'all'; } } class StrokeSlice { constructor(stroke, i_in, i_out) { this.stroke = stroke; // Stroke this.i_in = typeof i_in === 'undefined' ? 0 : i_in; this.i_out = typeof i_out === 'undefined' ? this.stroke.points.length - 1 : i_out; } getSliceId() { return `${this.i_in}-${this.i_out}`; } // compatible with Stroke() get points() { return this.stroke.points.slice(this.i_in, this.i_out + 1); } // compatible with Stroke() get color() { return this.stroke.color; } } class Annotator { constructor(wrapperEl, tags, fileurl) { this.wrapperEl = wrapperEl; this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); this.wrapperEl.appendChild(this.svgEl); this.controlsEl = document.createElement('div'); this.controlsEl.classList.add('controls') this.wrapperEl.appendChild(this.controlsEl); this.scrubberElOld = document.createElement('input'); this.scrubberElOld.type = "range"; this.scrubberElOld.min = 0; this.scrubberElOld.step = 0.01; this.controlsEl.appendChild(this.scrubberElOld); this.scrubberElOld.addEventListener("input", (ev) => { this.scrubTo(ev.target.value); }) this.scrubberEl = document.createElement('div'); this.scrubberEl.classList.add('scrubber') this.controlsEl.appendChild(this.scrubberEl); this.tagsEl = document.createElement('ul'); this.tagsEl.classList.add('tags'); for (let tag of tags) { let tagEl = document.createElement('li'); tagEl.classList.add('tag'); tagEl.dataset.tag = tag; tagEl.innerText = tag; tagEl.addEventListener('click', (e) => { this.addTag(tag, this.inPointPosition, this.outPointPosition); }) let signEl = document.createElement('span'); signEl.classList.add('annotation-' + tag); tagEl.prepend(signEl); this.tagsEl.appendChild(tagEl); } let tagEl = document.createElement('li'); tagEl.classList.add('tag'); tagEl.classList.add('annotation-rm'); tagEl.dataset.tag = 'rm'; tagEl.title = "Remove annotation"; tagEl.innerHTML = "×"; tagEl.addEventListener('click', (e) => { if (this.selectedAnnotation) { this.removeAnnotation(this.selectedAnnotationI); } }); this.tagsEl.appendChild(tagEl); this.controlsEl.appendChild(this.tagsEl); this.annotationsEl = document.createElement('div'); this.annotationsEl.classList.add('annotations') this.controlsEl.appendChild(this.annotationsEl); this.inPointPosition = null; this.outPointPosition = null; this.currentTime = 0; this.isPlaying = false; const groups = ['before', 'annotation', 'after'] this.strokeGroups = {}; groups.forEach(group => { let groupEl = document.createElementNS('http://www.w3.org/2000/svg', 'g'); groupEl.classList.add(group) this.svgEl.appendChild(groupEl); this.strokeGroups[group] = new StrokeGroup(groupEl, this); }); this.annotations = []; this.play(fileurl); } updateAnnotations(save) { this.annotationsEl.innerHTML = ""; for (let annotation_i in this.annotations) { const annotation = this.annotations[annotation_i]; this.annotationEl = document.createElement('div'); const left = (annotation.t_in / this.duration) * 100; const right = 100 - (annotation.t_out / this.duration) * 100; this.annotationEl.style.left = left + '%'; this.annotationEl.style.right = right + '%'; this.annotationEl.classList.add('annotation-' + annotation.tag); if (this.selectedAnnotationI == annotation_i) { this.annotationEl.classList.add('selected'); } this.annotationEl.title = annotation.tag; this.annotationEl.addEventListener('mouseover', (e) => { }); this.annotationEl.addEventListener('mouseout', (e) => { }); this.annotationEl.addEventListener('click', (e) => { if (this.selectedAnnotationI == annotation_i) { this.deselectAnnotation(false); } else { this.selectAnnotation(annotation_i); } }); this.annotationsEl.appendChild(this.annotationEl); } this.tagsEl.childNodes.forEach(tagEl => { if (this.selectedAnnotation && this.selectedAnnotation.tag == tagEl.dataset.tag) { tagEl.classList.add('selected') } else { tagEl.classList.remove('selected') } }); if (save) { this.updateState(); } } selectAnnotation(annotation_i) { this.selectedAnnotationI = annotation_i; this.selectedAnnotation = this.annotations[annotation_i]; this.slider.set([this.selectedAnnotation.t_in, this.selectedAnnotation.t_out]); this.inPointPosition = this.findPositionForTime(this.selectedAnnotation.t_in); this.outPointPosition = this.findPositionForTime(this.selectedAnnotation.t_out); this.drawStrokePosition(this.inPointPosition, this.outPointPosition); this.updateAnnotations(false); //selects the right tag & highlights the annotation this.wrapperEl.classList.add('selected-annotation'); } deselectAnnotation(keep_position) { if (this.selectedAnnotation) this.currentTime = this.selectedAnnotation.t_out; this.wrapperEl.classList.remove('selected-annotation'); this.selectedAnnotationI = null; this.selectedAnnotation = null; if (!keep_position) { this.setUpAnnotator(); } this.updateAnnotations(false); // selects the right tag & highlights the annotation } play(file) { const request = new Request(file, { method: 'GET', }); fetch(request) .then(response => response.json()) .then(data => { const metadata_req = new Request(`/annotations/${data.file}`, { method: 'GET', }); fetch(metadata_req) .then(response => response.ok ? response.json() : null) .then(metadata => { this.playStrokes(data, metadata) }) .catch(e => console.log(e)); // do something with the data sent in the request }); } updateState() { const state = { 'file': this.filename, 'annotations': this.annotations, 'audio': { 'file': this.audioFile, 'offset': this.audioOffset, } } const newState = JSON.stringify(state); if (newState == this.state) { return; } this.wrapperEl.classList.remove('saved'); this.wrapperEl.classList.add('unsaved'); this.state = newState; // autosave on state change: this.save(newState); } setSaved(state) { if (this.state != state) { console.log('already outdated'); } else { this.wrapperEl.classList.add('saved'); this.wrapperEl.classList.remove('unsaved'); } } save(state) { const request = new Request("/annotations/" + this.filename, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: state }); fetch(request) .then((response) => { if (response.ok) { this.setSaved(state); } else { throw Error('Something went wrong'); } }) .catch((error) => { console.log(error); }); } removeAnnotation(annotation_i) { this.deselectAnnotation(true); this.annotations.splice(annotation_i, 1); this.updateAnnotations(true); } addTag(tag) { if (this.selectedAnnotation) { this.selectedAnnotation.tag = tag; this.updateAnnotations(true); } else { // TODO this.slider values for in and out const [t_in, t_out] = this.slider.get(); if (this.slider) { this.slider.destroy(); } this.annotations.push(new Annotation(tag, t_in, t_out)); this.updateAnnotations(true); this.currentTime = t_out; this.setUpAnnotator(); } } setUpAnnotator() { this.inPointPosition = this.findPositionForTime(this.currentTime); this.outPointPosition = this.findPositionForTime(this.duration); if (this.scrubberEl.noUiSlider) { this.slider.destroy(); } this.slider = noUiSlider.create(this.scrubberEl, { start: [this.currentTime, this.duration], connect: true, range: { 'min': 0, 'max': this.duration }, tooltips: [ this.formatter, this.formatter ], // pips: { // mode: 'range', // density: 3, // format: this.formatter // } }); this.slider.on("slide", (values, handle) => { this.isPlaying = false; this.inPointPosition = this.findPositionForTime(values[0]); this.outPointPosition = this.findPositionForTime(values[1]); this.drawStrokePosition(this.inPointPosition, this.outPointPosition); // console.log(this.selectedAnnotation); if (this.selectedAnnotation) { this.selectedAnnotation.t_in = values[0]; this.selectedAnnotation.t_out = values[1]; this.updateAnnotations(false); } }); this.slider.on("end", (values, handle) => { if (this.selectedAnnotation) { this.updateAnnotations(true); } this.playAudioSegment(values[0], values[1]); }) this.drawStrokePosition(this.inPointPosition, this.outPointPosition); } playStrokes(drawing, metadata) { this.audioOffset = 0; if (metadata) { this.annotations = metadata.annotations; this.audioFile = metadata.hasOwnProperty('audio') ? metadata.audio.file : null; this.audioOffset = metadata.hasOwnProperty('audio') ? metadata.audio.offset : 0; // // load any saved metadata } this.filename = drawing.file; this.strokes = drawing.shape.map(s => new Stroke(s['color'], s['points'])); this.currentPathI = null; this.currentPointI = null; this.dimensions = drawing.dimensions; this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`) let bgEl = document.createElementNS('http://www.w3.org/2000/svg', 'rect'); bgEl.setAttribute("x", 0); bgEl.setAttribute("y", 0); bgEl.setAttribute("width", this.dimensions[0]); bgEl.setAttribute("height", this.dimensions[1]); bgEl.classList.add('background'); this.svgEl.prepend(bgEl); this.startTime = window.performance.now() - this.strokes[0].points[0][3]; this.duration = this.getDuration(); this.scrubberElOld.max = this.duration; this.playTimout = null; this.formatter = wNumb({ decimals: 2, edit: (time) => { const s = Math.floor(time / 1000); const minutes = Math.floor(s / 60); const seconds = s - minutes * 60; const ms = Math.floor((time / 1000 - s) * 1000); return `${minutes}:${seconds}:${ms}`; } }); this.setUpAnnotator() this.updateAnnotations(false); this.setupAudioConfig(); // this.playStrokePosition(0, 1); } setupAudioConfig() { // audio config let audioConfigEl = document.createElement('div'); audioConfigEl.classList.add('audioconfig') this.wrapperEl.appendChild(audioConfigEl); let audioSelectEl = document.createElement('select'); audioSelectEl.classList.add('audioselect'); audioConfigEl.appendChild(audioSelectEl); fetch('/audio') .then(response => response.json()) .then(data => { data.unshift(''); // add empty, to deselect any file data.forEach(audioFile => { let optionEl = document.createElement('option'); optionEl.selected = this.audioFile == audioFile; optionEl.innerText = audioFile; audioSelectEl.appendChild(optionEl); }); }) audioSelectEl.addEventListener('change', (ev) => { this.setAudioFile(ev.target.value); }); let audioOffsetTextEl = document.createElement('label'); audioOffsetTextEl.innerText = "Offset (s)"; audioConfigEl.appendChild(audioOffsetTextEl); let audioOffsetEl = document.createElement('input'); audioOffsetEl.setAttribute('type', 'number'); audioOffsetEl.setAttribute('step', '.01'); audioOffsetEl.value = this.audioOffset ?? 0; audioOffsetEl.addEventListener('change', (ev) => { this.setAudioOffset(ev.target.value); }); audioOffsetTextEl.appendChild(audioOffsetEl); this.audioEl = document.createElement('audio'); if (this.audioFile) { this.audioEl.setAttribute('src', this.audioFile); } this.audioEl.setAttribute('controls', true); this.audioEl.addEventListener('canplaythrough', (ev) => { console.log('loaded audio', ev); this.audioEl.play(); }); // this.audioEl.addEventListener('seeked', (ev)=>{ // console.log(ev); // }) audioConfigEl.prepend(this.audioEl); } setAudioFile(audioFile) { this.audioFile = audioFile; this.audioEl.setAttribute('src', this.audioFile); // this.audioEl.play(); // TODO update playhead // TODO update this.duration after load this.updateState(); } setAudioOffset(audioOffset) { this.audioOffset = audioOffset; // TODO update playhead // TODO update this.duration this.updateState(); } /** * @param float time time is ms * @returns float */ getAudioTime(time) { return Number.parseFloat(time) + (this.audioOffset * 1000 ?? 0); } /** * * @param float t_in in point time, in ms * @param float t_out out point time, in ms */ playAudioSegment(t_in, t_out) { if (this.audioStartTimeout) clearTimeout(this.audioStartTimeout); if (this.audioEndTimeout) clearTimeout(this.audioEndTimeout); // TODO, handle playback delay const t_start = this.getAudioTime(t_in); // in ms const t_diff = t_out - t_in; // in ms console.log('set time', t_in, t_start, typeof t_start, typeof t_in, t_start < 0); this.audioEl.pause(); if (t_start < 0) { if (t_diff <= t_start * -1) { console.log('no audio playback in segment', t_start, t_diff); } else { console.log('huh?', t_start, t_diff); // a negative audiooffset delays playback from the start // this.audioStartTimeout = setTimeout((e) => this.audioEl.play(), t*-1000); this.audioStartTimeout = setTimeout((e) => { this.audioEl.currentTime = 0 }, t_start * -1); // triggers play with "seeked" event // this.audioEl.currentTime = 0; } } else { this.audioEl.currentTime = t_start / 1000; // this.audioEl.play(); // play is done in "seeked" evenlistener console.log(this.audioEl.currentTime, t_start, t_in, t_out) } this.audioEndTimeout = setTimeout((e) => this.audioEl.pause(), t_diff); } getDuration() { const points = this.strokes[this.strokes.length - 1].points; return points[points.length - 1][3]; } getStrokesSliceForPathRange(in_point, out_point) { // get paths for given range. Also, split path at in & out if necessary. let slices = {}; for (let i = in_point[0]; i <= out_point[0]; i++) { const stroke = this.strokes[i]; if (typeof stroke === 'undefined') { // out point can be Infinity. So interrupt whenever the end is reached break; } const in_i = (in_point[0] === i) ? in_point[1] : 0; const out_i = (out_point[0] === i) ? out_point[1] : Infinity; slices[i] = new StrokeSlice(stroke, in_i, out_i); } return slices; } // TODO: when drawing, have a group active & inactive. // active is getPathRange(currentIn, currentOut) // inactive is what comes before and after. // then, playing the video is just running pathRanghe(0, playhead) drawStrokePosition(in_point, out_point, show_all) { if (typeof show_all === 'undefined') show_all = true; this.strokeGroups['before'].setStrokes(this.getStrokesSliceForPathRange([0, 0], in_point)); this.strokeGroups['annotation'].setStrokes(this.getStrokesSliceForPathRange(in_point, out_point)); this.strokeGroups['after'].setStrokes(this.getStrokesSliceForPathRange(out_point, [Infinity, Infinity])); // // an inpoint is set, so we're annotating // // make everything coming before translucent // if (this.inPointPosition !== null) { // const [inPath_i, inPoint_i] = this.inPointPosition; // // returns a static NodeList // const currentBeforeEls = this.svgEl.querySelectorAll(`.before_in`); // for (let currentBeforeEl of currentBeforeEls) { // currentBeforeEl.classList.remove('before_in'); // } // for (let index = 0; index < inPath_i; index++) { // const pathEl = this.svgEl.querySelector(`.path${index}`); // if (pathEl) { // pathEl.classList.add('before_in'); // } // } // } // this.currentPathI = path_i; // this.currentPointI = point_i; // const path = this.strokes[path_i]; // // console.log(path); // let pathEl = this.svgEl.querySelector(`.path${path_i}`); // if (!pathEl) { // pathEl = document.createElementNS('http://www.w3.org/2000/svg', 'path'); // pathEl.style.stroke = path.color; // pathEl.classList.add('path' + path_i) // this.svgEl.appendChild(pathEl) // } // const stroke = path.points.slice(0, point_i); // const d = this.strokes2D(stroke); // pathEl.setAttribute('d', d); // this.scrubberElOld.value = path.points[point_i][3]; // this.currentTime = path.points[point_i][3]; } getNextPosition(path_i, point_i) { const path = this.strokes[path_i]; let next_path, next_point; if (path.points.length > point_i + 1) { next_path = path_i; next_point = point_i + 1; // setTimeout(() => this.playStroke(next_path, next_point), dt); } else if (this.strokes.length > path_i + 1) { next_path = path_i + 1; next_point = 1; // use starttime instead of diff, to prevent floating } else { return [null, null]; } // when an outpoint is set, stop playing there if (this.outPointPosition && (next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1])) { return [null, null]; } return [next_path, next_point]; } playStrokePosition(path_i, point_i, allow_interrupt) { if (allow_interrupt) { if (!this.isPlaying) { console.log('not playing because of interrupt'); return; } } else { this.isPlaying = true; } this.drawStrokePosition(path_i, point_i); const [next_path, next_point] = this.getNextPosition(path_i, point_i); if (next_path === null) { console.log('done playing'); return; } const t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3]; const dt = t - (window.performance.now() - this.startTime); this.playTimout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt); } playUntil(path_i) { // for scrubber } scrubTo(ms) { const [path_i, point_i] = this.findPositionForTime(ms); // console.log(path_i, point_i); clearTimeout(this.playTimout); this.playStrokePosition(path_i, point_i); // this.playHead = ms; } findPositionForTime(ms) { ms = Math.min(Math.max(ms, 0), this.duration); // console.log('scrub to', ms) let path_i = 0; let point_i = 0; this.strokes.every((stroke, index) => { const startAt = stroke.points[0][3]; const endAt = stroke.points[stroke.points.length - 1][3]; if (startAt > ms) { return false; // too far } if (endAt > ms) { // we're getting close. Find the right point_i path_i = index; stroke.points.every((point, pi) => { if (point[3] > ms) { // too far return false; } point_i = pi; return true; }); return false; } else { // in case nothings comes after, we store the last best option thus far path_i = index; point_i = stroke.points.length - 1; return true; } }); return [path_i, point_i]; } }