New stroke playing logic for annotating and prep for adding/syncing audio file

files/audio/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

webserver.py

@@ -119,6 +119,22 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
         return client in cls.connections
         
 
+class AudioListingHandler(tornado.web.RequestHandler):
+    def initialize(self, config):
+        self.config = config
+        self.audiodir = os.path.join(self.config.storage, 'audio')
+
+    def get(self):
+        # filename = self.get_argument("file", None)
+        self.set_header("Content-Type", "application/json")
+        if not os.path.exists(self.audiodir):
+            names = []
+        else:
+            names = sorted([f"/audio/{name}" for name in os.listdir(self.audiodir) if name not in ['.gitignore']])
+            print(names)
+        self.write(json.dumps(names))
+            
+
 class AnimationHandler(tornado.web.RequestHandler):
     def initialize(self, config):
         self.config = config
@@ -132,6 +148,7 @@ class AnimationHandler(tornado.web.RequestHandler):
         else:
             path = os.path.join(self.config.storage,os.path.basename(filename)+".csv")
             drawing = {
+                "file": filename,
                 "shape": []
             }
             with open(path, 'r') as fp:
@@ -172,8 +189,53 @@ def strokes2D(strokes):
 
             rel_stroke = [stroke[0] - last_stroke[0], stroke[1] - last_stroke[1]];
             d += f"{rel_stroke[0]},{rel_stroke[1]} "
-        last_stroke = stroke;
+        last_stroke = stroke
-    return d;
+    return d
+
+  
+
+class AnnotationsHandler(tornado.web.RequestHandler):
+    def initialize(self, config):
+        self.config = config
+        self.metadir = os.path.join(self.config.storage, 'metadata')
+
+    def prepare(self):
+        if self.request.headers.get("Content-Type", "").startswith("application/json"):
+            self.json_args = json.loads(self.request.body)
+        else:
+            self.json_args = None
+
+    def get(self, filename):
+        self.set_header("Content-Type", "application/json")
+        filenames = sorted([name[:-4] for name in os.listdir(self.config.storage) if name not in ['.gitignore']])
+
+        if filename not in filenames:
+            raise Exception('Invalid filename')
+
+        
+        meta_file = os.path.join(self.metadir, filename +'.json')
+        if not os.path.exists(meta_file):
+            self.set_status(404)
+            return
+
+        with open(meta_file, 'r') as fp:
+            self.write(json.load(fp))
+
+    def post(self, filename):
+        # filename = self.get_argument("file", None)
+
+        filenames = sorted([name[:-4] for name in os.listdir(self.config.storage) if name not in ['.gitignore']])
+        
+        if filename not in filenames:
+            raise Exception('Invalid filename')
+
+        if not os.path.exists(self.metadir):
+            os.mkdir(self.metadir)
+
+        meta_file = os.path.join(self.metadir, filename +'.json')
+        with open(meta_file, 'w') as fp:
+            json.dump(self.json_args, fp)
+
 
 class Server:
     """
@@ -198,6 +260,12 @@ class Server:
                 
                 (r"/files/(.*)", AnimationHandler,
                     {'config': self.config}),
+                (r"/audio/(.+)", tornado.web.StaticFileHandler,
+                    {"path": os.path.join(self.config.storage, 'audio')}),
+                (r"/audio", AudioListingHandler,
+                    {'config': self.config}),
+                (r"/annotations/(.+)", AnnotationsHandler,
+                    {'config': self.config}),
                 (r"/(.*)", StaticFileWithHeaderHandler,
                     {"path": self.web_root}),
             ], debug=True, autoreload=True)

www/annotate.html

@@ -5,17 +5,27 @@
     <meta charset="utf-8">
     <title>Annotate a line animation</title>
     <style media="screen">
+        body {
+            /* background: black;
+            color: white */
+            background: lightgray;
+        }
+
         #sample,
         svg {
             position: absolute;
-			top: 0;
+            top: 20px;
-			left: 0;
+            left: 20px;
-			bottom: 0;
+            width: calc(100% - 40px);
-			right: 0;
+            height: calc(100% - 200px);
-			width: 100%;
-			height: 100%;
             font-family: sans-serif;
             z-index: 2;
+            /* background: white; */
+            /* border: solid 2px lightgray; */
+        }
+
+        svg .background {
+            fill: white
         }
 
         img {
@@ -36,11 +46,13 @@
             stroke-linecap: round;
         }
 
-		g.before path{
+        g.before path {
             opacity: 0.5;
             stroke: gray !important;
         }
-        g.after path, path.before_in {
+
+        g.after path,
+        path.before_in {
             opacity: .1;
             stroke: gray !important;
         }
@@ -74,31 +86,147 @@
             z-index: 50;
         }
 
-		.playlist li{
+        .playlist li {
             cursor: pointer;
             line-height: 1.5;
         }
-		.playlist li:hover{
+
+        .playlist li:hover {
             color: blue;
         }
 
-		input[type='range']{
+        input[type='range'] {
-			position: absolute;
+            /* position: absolute;
             z-index: 100;
             bottom: 0;
-			left:0;
+            left: 0;
-			right: 0;width: 90%;
+            right: 0; */
-		}
+            width: 100%;
-		.scrubber{
-			position: absolute !important;
-			z-index: 100;
-			bottom: 30px;
-			left:0;
-			right: 0;width: 90%;
         }
 
-        .noUi-horizontal .noUi-touch-area{
+        .controls {
-            cursor:ew-resize;
+            position: absolute !important;
+            z-index: 100;
+            bottom: 10px;
+            left: 5%;
+            right: 0;
+            width: 90%;
+        }
+
+        .scrubber {}
+
+        .tags {
+            line-height: 40px;
+            display: flex;
+            flex-direction: row;
+            padding: 0;
+            margin: 0;
+        }
+
+        .tags li {
+            display: block;
+            padding: 5px;
+            border: solid 1px darkgray;
+            flex-grow: 1;
+            text-align: center;
+        }
+
+        .tags li:hover {
+            cursor: pointer;
+            background: darkgray;
+        }
+
+        .tags li.selected {
+            background: lightsteelblue;
+        }
+
+        .tags li.annotation-rm {
+            /* display: none; */
+            overflow: hidden;
+            color: red;
+            font-size: 30px;
+            width: 0;
+            flex-grow: 0;
+            padding: 5px 0;
+            transition: width .3s;
+            pointer-events: none;
+            border: none;
+            direction: rtl; /* hide behind bar, instead into nothing */
+        }
+
+        .selected-annotation .tags li.annotation-rm {
+            color: red;
+            display: block;
+            width: 30px;
+            pointer-events: all;
+        }
+
+        .tags li span {
+            display: inline-block;
+            width: 20px;
+            height: 20px;
+            margin-right: 10px;
+            vertical-align: middle;
+            border-radius: 5px;
+        }
+
+        .annotations {
+            height: 10px;
+            /* border: solid 1px darkgray; */
+            position: relative;
+        }
+
+        .annotations>div {
+            opacity: .4;
+            background: lightseagreen;
+
+            position: absolute;
+            bottom: 0;
+            top: 0;
+        }
+
+        .annotations>div:hover,
+        .annotations>div.selected {
+            opacity: 1;
+            cursor: pointer;
+        }
+
+        .annotation-test {
+            background-color: red !important;
+        }
+
+        .annotation-another {
+            background-color: blue !important;
+        }
+
+        .annotation-google {
+            background-color: blueviolet !important;
+        }
+
+        .unsaved::before {
+            content: '*';
+            color: red;
+            display: inline-block;
+            text-align: center;
+            font-size: 30px;
+            position: absolute;
+            top: 10px;
+            left: 10px;
+        }
+
+        .saved::before {
+            content: '\2713';
+            display: inline-block;
+            color: green;
+            text-align: center;
+            font-size: 30px;
+            position: absolute;
+            top: 10px;
+            left: 10px;
+        }
+
+        .noUi-horizontal .noUi-touch-area {
+            cursor: ew-resize;
         }
     </style>
     <link rel="stylesheet" href="assets/nouislider-15.5.0.css">
@@ -111,7 +239,10 @@
     <script src="assets/wNumb-1.2.0.min.js"></script>
     <script src="annotate.js"></script>
     <script type='text/javascript'>
-		const player = new Player(document.getElementById("interface"));
+        const player = new Annotator(
+            document.getElementById("interface"),
+            ["test", "another", "google"]
+        );
         player.playlist('/files/');
     </script>
 </body>

www/annotate.js

@@ -1,28 +1,27 @@
-class Annotation{
+class Annotation {
-    constructor(annotation, t_in, t_out) {
+    constructor(tag, t_in, t_out) {
-        this.annotation = annotation;
+        this.tag = tag;
         this.t_in = t_in;
         this.t_out = t_out;
     }
 }
 
-class StrokeGroup{
+class StrokeGroup {
-    constructor(group_element, player){
+    constructor(group_element, player) {
         this.g = group_element;
         this.player = player;
     }
 
-    setStrokes(strokes){
+    setStrokes(strokes) {
-        console.log('set strokes',strokes);
         const pathEls = this.g.querySelectorAll('path');
         let indexes = Object.keys(strokes);
         for (let pathEl of pathEls) {
             const i = pathEl.dataset.path_i;
-            if(!indexes.includes(pathEl.dataset.path_i)){
+            if (!indexes.includes(pathEl.dataset.path_i)) {
                 pathEl.parentNode.removeChild(pathEl);
-            }else{
+            } else {
                 // check in and outpoint using pathEl.dataset
-                if(strokes[i].getSliceId() != pathEl.dataset.slice){
+                if (strokes[i].getSliceId() != pathEl.dataset.slice) {
                     const d = this.points2D(strokes[i].points);
                     pathEl.dataset.slice = strokes[i].getSliceId();
                     pathEl.setAttribute('d', d);
@@ -32,7 +31,6 @@ class StrokeGroup{
             // this has now been processed
             indexes.splice(indexes.indexOf(i), 1);
         }
-        console.log(indexes);
 
         // new strokes
         indexes.forEach(index => {
@@ -78,8 +76,8 @@ class StrokeGroup{
     }
 }
 
-class Stroke{
+class Stroke {
-    constructor(color, points){
+    constructor(color, points) {
         this.color = color;
         this.points = points; // [[x1,y1,t1], [x2,y2,t2], ...]
     }
@@ -89,48 +87,89 @@ class Stroke{
     }
 }
 
-class StrokeSlice{
+class StrokeSlice {
-    constructor(stroke, i_in, i_out){
+    constructor(stroke, i_in, i_out) {
         this.stroke = stroke; // Stroke
         this.i_in = typeof i_in === 'undefined' ? 0 : i_in;
         this.i_out = typeof i_out === 'undefined' ? this.stroke.points.length : i_out;
     }
 
-    getSliceId(){
+    getSliceId() {
         return `${this.i_in}-${this.i_out}`;
     }
 
     // compatible with Stroke()
-    get points(){
+    get points() {
         return this.stroke.points.slice(this.i_in, this.i_out);
     }
 
     // compatible with Stroke()
-    get color(){
+    get color() {
         return this.stroke.color;
     }
 }
 
-class Player {
+class Annotator {
-    constructor(wrapperEl) {
+    constructor(wrapperEl, tags) {
         this.wrapperEl = wrapperEl;
         this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
         this.wrapperEl.appendChild(this.svgEl);
 
+
+        this.controlsEl = document.createElement('div');
+        this.controlsEl.classList.add('controls')
+        this.wrapperEl.appendChild(this.controlsEl);
+
         this.scrubberElOld = document.createElement('input');
         this.scrubberElOld.type = "range";
         this.scrubberElOld.min = 0;
         this.scrubberElOld.step = 0.01;
-        this.wrapperEl.appendChild(this.scrubberElOld);
+        this.controlsEl.appendChild(this.scrubberElOld);
-
-        this.scrubberEl = document.createElement('div');
-        this.scrubberEl.classList.add('scrubber')
-        this.wrapperEl.appendChild(this.scrubberEl);
 
         this.scrubberElOld.addEventListener("input", (ev) => {
             this.scrubTo(ev.target.value);
         })
 
+        this.scrubberEl = document.createElement('div');
+        this.scrubberEl.classList.add('scrubber')
+        this.controlsEl.appendChild(this.scrubberEl);
+
+        this.tagsEl = document.createElement('ul');
+        this.tagsEl.classList.add('tags');
+        for (let tag of tags) {
+            let tagEl = document.createElement('li');
+            tagEl.classList.add('tag');
+            tagEl.dataset.tag = tag;
+            tagEl.innerText = tag;
+            tagEl.addEventListener('click', (e) => {
+                this.addTag(tag, this.inPointPosition, this.outPointPosition);
+            })
+
+            let signEl = document.createElement('span');
+            signEl.classList.add('annotation-' + tag);
+            tagEl.prepend(signEl);
+            this.tagsEl.appendChild(tagEl);
+        }
+        let tagEl = document.createElement('li');
+        tagEl.classList.add('tag');
+        tagEl.classList.add('annotation-rm');
+        tagEl.dataset.tag = 'rm';
+        tagEl.title = "Remove annotation";
+        tagEl.innerHTML = "×";
+        tagEl.addEventListener('click', (e) => {
+            if (this.selectedAnnotation) {
+                this.removeAnnotation(this.selectedAnnotationI);
+            }
+        });
+        this.tagsEl.appendChild(tagEl);
+
+        this.controlsEl.appendChild(this.tagsEl);
+
+        this.annotationsEl = document.createElement('div');
+        this.annotationsEl.classList.add('annotations')
+        this.controlsEl.appendChild(this.annotationsEl);
+
+
         this.inPointPosition = null;
         this.outPointPosition = null;
         this.currentTime = 0;
@@ -148,6 +187,83 @@ class Player {
         this.annotations = []
     }
 
+    updateAnnotations(save) {
+        this.annotationsEl.innerHTML = "";
+        for (let annotation_i in this.annotations) {
+            const annotation = this.annotations[annotation_i];
+            this.annotationEl = document.createElement('div');
+            const left = (annotation.t_in / this.duration) * 100;
+            const right = 100 - (annotation.t_out / this.duration) * 100;
+            this.annotationEl.style.left = left + '%';
+            this.annotationEl.style.right = right + '%';
+
+            this.annotationEl.classList.add('annotation-' + annotation.tag);
+            if (this.selectedAnnotationI == annotation_i) {
+                this.annotationEl.classList.add('selected');
+            }
+            this.annotationEl.title = annotation.tag;
+
+            this.annotationEl.addEventListener('mouseover', (e) => {
+
+            });
+            this.annotationEl.addEventListener('mouseout', (e) => {
+
+            });
+            this.annotationEl.addEventListener('click', (e) => {
+                if (this.selectedAnnotationI == annotation_i) {
+                    this.deselectAnnotation(false);
+                } else {
+                    this.selectAnnotation(annotation_i);
+                }
+            });
+
+            this.annotationsEl.appendChild(this.annotationEl);
+        }
+
+        this.tagsEl.childNodes.forEach(tagEl => {
+            if (this.selectedAnnotation && this.selectedAnnotation.tag == tagEl.dataset.tag) {
+                tagEl.classList.add('selected')
+            } else {
+                tagEl.classList.remove('selected')
+            }
+        });
+
+        if (save) {
+            this.updateState();
+        }
+    }
+
+    selectAnnotation(annotation_i) {
+        this.selectedAnnotationI = annotation_i;
+        this.selectedAnnotation = this.annotations[annotation_i];
+
+        this.slider.set([this.selectedAnnotation.t_in, this.selectedAnnotation.t_out]);
+
+        this.inPointPosition = this.findPositionForTime(this.selectedAnnotation.t_in);
+        this.outPointPosition = this.findPositionForTime(this.selectedAnnotation.t_out);
+        this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
+
+        this.updateAnnotations(false); //selects the right tag & highlights the annotation
+
+        this.wrapperEl.classList.add('selected-annotation');
+    }
+
+    deselectAnnotation(keep_position) {
+        if (this.selectedAnnotation)
+            this.currentTime = this.selectedAnnotation.t_out;
+
+        this.wrapperEl.classList.remove('selected-annotation');
+
+        this.selectedAnnotationI = null;
+        this.selectedAnnotation = null;
+
+        if (!keep_position) {
+            this.setUpAnnotator();
+        }
+        this.updateAnnotations(false); // selects the right tag & highlights the annotation
+    }
+
+    // TODO: to separate class which then instantiates a player for the given file
     playlist(url) {
         const request = new Request(url, {
             method: 'GET',
@@ -190,35 +306,109 @@ class Player {
         fetch(request)
             .then(response => response.json())
             .then(data => {
-                this.playStrokes(data)
+                const metadata_req = new Request(`/annotations/${data.file}`, {
+                    method: 'GET',
+                });
+                fetch(metadata_req)
+                    .then(response => response.ok ? response.json() : null)
+                    .then(metadata => {
+                        this.playStrokes(data, metadata)
+                    })
+                    .catch(e => console.log(e));
                 // do something with the data sent in the request
             });
-
     }
 
-    playStrokes(drawing) {
+    updateState() {
-        this.strokes = drawing.shape.map(s => new Stroke(s['color'], s['points']));
+        const state = {
-        this.currentPathI = null;
+            'file': this.filename,
-        this.currentPointI = null;
+            'annotations': this.annotations,
-        this.dimensions = drawing.dimensions;
+            'audio': {
-        this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
+                'file': this.audioFile,
-        this.startTime = window.performance.now() - this.strokes[0].points[0][3];
+                'offset': this.audioOffset,
-        this.playStrokePosition(0, 1);
-        this.duration = this.getDuration();
-        this.scrubberElOld.max = this.duration;
-        this.playTimout = null;
-
-        const formatter = wNumb({
-            decimals: 2,
-            edit: (time) => {
-                const s = Math.floor(time/1000);
-                const minutes = Math.floor(s / 60);
-                const seconds = s - minutes * 60;
-                const ms = Math.floor((time/1000 - s) * 1000);
-                return `${minutes}:${seconds}:${ms}`;
             }
+        }
+        const newState = JSON.stringify(state);
+        if (newState == this.state) {
+            return;
+        }
+
+        this.wrapperEl.classList.remove('saved');
+        this.wrapperEl.classList.add('unsaved');
+        this.state = newState;
+        // autosave on state change:
+        this.save(newState);
+    }
+
+    setSaved(state) {
+        if (this.state != state) {
+            console.log('already outdated');
+        }
+        else {
+            this.wrapperEl.classList.add('saved');
+            this.wrapperEl.classList.remove('unsaved');
+        }
+    }
+
+    save(state) {
+        const request = new Request("/annotations/" + this.filename, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: state
         });
-        const slider = noUiSlider.create(this.scrubberEl, {
+        fetch(request)
+            .then((response) => {
+                if (response.ok) {
+                    this.setSaved(state);
+                }
+                else {
+                    throw Error('Something went wrong');
+                }
+            })
+            .catch((error) => {
+                console.log(error);
+            });
+
+    }
+
+    removeAnnotation(annotation_i) {
+        this.deselectAnnotation(true);
+        this.annotations.splice(annotation_i, 1);
+        this.updateAnnotations(true);
+    }
+
+    addTag(tag) {
+        if (this.selectedAnnotation) {
+            this.selectedAnnotation.tag = tag;
+            this.updateAnnotations(true);
+        } else {
+
+            // TODO this.slider values for in and out
+            const [t_in, t_out] = this.slider.get();
+            if (this.slider) {
+                this.slider.destroy();
+            }
+
+            this.annotations.push(new Annotation(tag, t_in, t_out));
+            this.updateAnnotations(true);
+
+            this.currentTime = t_out;
+            this.setUpAnnotator();
+        }
+    }
+
+
+    setUpAnnotator() {
+        this.inPointPosition = this.findPositionForTime(this.currentTime);
+        this.outPointPosition = this.findPositionForTime(this.duration);
+
+        if (this.scrubberEl.noUiSlider) {
+            this.slider.destroy();
+        }
+
+        this.slider = noUiSlider.create(this.scrubberEl, {
             start: [this.currentTime, this.duration],
             connect: true,
             range: {
@@ -226,51 +416,144 @@ class Player {
                 'max': this.duration
             },
             tooltips: [
-                formatter, 
+                this.formatter,
-                formatter 
+                this.formatter
             ],
             // pips: {
             //     mode: 'range',
             //     density: 3,
-            //     format: formatter
+            //     format: this.formatter
             // }
         });
 
-        slider.on("slide", (values, handle) => {
+        this.slider.on("slide", (values, handle) => {
             this.isPlaying = false;
-            // console.log(values, handle);
-            // both in and out need to have a value
             this.inPointPosition = this.findPositionForTime(values[0]);
             this.outPointPosition = this.findPositionForTime(values[1]);
-            // if (handle === 0) {
-            //     // in point
-            //     if (
-            //         this.currentPathI < this.inPointPosition[0] ||
-            //         this.currentPointI < this.inPointPosition[1]) {
-            //             this.drawStrokePosition(
-            //                 // this.inPointPosition[0],
-            //                 // this.inPointPosition[1],
-            //                 // always draw at out position, as to see the whole shape of the range
-            //                 this.outPointPosition[0],
-            //                 this.outPointPosition[1],
-            //             );
-            //     }
-            // }
-            // if (handle === 1) {
-            //     // out point
-            //     // this.outPointPosition = this.findPositionForTime(values[1]);
-            //     this.drawStrokePosition(
-            //         this.outPointPosition[0],
-            //         this.outPointPosition[1],
-            //     );
-            // }
             this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
-            // this.inPointPosition = values;
+
-            // this.outPointPosition = vaalues[0];
+            // console.log(this.selectedAnnotation);
-            // this.scrubTo()
+            if (this.selectedAnnotation) {
-            // this.scrubTo(ev.target.value);
+                this.selectedAnnotation.t_in = values[0];
+                this.selectedAnnotation.t_out = values[1];
+                this.updateAnnotations(false);
+            }
+        });
+        this.slider.on("end", (values, handle) => {
+            if (this.selectedAnnotation) {
+                this.updateAnnotations(true);
+            }
+        })
+
+        this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
+    }
+
+    playStrokes(drawing, metadata) {
+        if (metadata) {
+            this.annotations = metadata.annotations;
+            this.audioFile = metadata.hasOwnProperty('audio') ? metadata.audio.file : null;
+            this.audioOffset = metadata.hasOwnProperty('audio') ? metadata.audio.offset : 0;
+            //
+            // load any saved metadata
+        }
+        this.filename = drawing.file;
+        this.strokes = drawing.shape.map(s => new Stroke(s['color'], s['points']));
+        this.currentPathI = null;
+        this.currentPointI = null;
+        this.dimensions = drawing.dimensions;
+        this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
+
+        let bgEl = document.createElementNS('http://www.w3.org/2000/svg', 'rect');
+        bgEl.setAttribute("x", 0);
+        bgEl.setAttribute("y", 0);
+        bgEl.setAttribute("width", this.dimensions[0]);
+        bgEl.setAttribute("height", this.dimensions[1]);
+        bgEl.classList.add('background');
+        this.svgEl.prepend(bgEl);
+
+        this.startTime = window.performance.now() - this.strokes[0].points[0][3];
+        this.duration = this.getDuration();
+        this.scrubberElOld.max = this.duration;
+        this.playTimout = null;
+
+        this.formatter = wNumb({
+            decimals: 2,
+            edit: (time) => {
+                const s = Math.floor(time / 1000);
+                const minutes = Math.floor(s / 60);
+                const seconds = s - minutes * 60;
+                const ms = Math.floor((time / 1000 - s) * 1000);
+                return `${minutes}:${seconds}:${ms}`;
+            }
         });
 
+        this.setUpAnnotator()
+        this.updateAnnotations(false);
+
+        this.setupAudioConfig();
+        
+        // this.playStrokePosition(0, 1);
+    }
+
+    setupAudioConfig(){
+        // audio config
+        let audioConfigEl = document.createElement('div');
+        audioConfigEl.classList.add('audioconfig')
+        this.wrapperEl.appendChild(audioConfigEl);
+
+        let audioSelectEl = document.createElement('select');
+        audioSelectEl.classList.add('audioselect');
+        audioConfigEl.appendChild(audioSelectEl);
+
+        fetch('/audio')
+            .then(response => response.json())
+            .then(data => {
+                data.unshift(''); // add empty, to deselect any file
+                data.forEach(audioFile => {
+                    let optionEl = document.createElement('option');
+                    optionEl.selected = this.audioFile == audioFile;
+                    optionEl.innerText = audioFile;
+                    audioSelectEl.appendChild(optionEl);
+                });
+            })
+
+        audioSelectEl.addEventListener('change', (ev) => {
+            this.setAudioFile(ev.target.value);
+        });
+
+
+        let audioOffsetEl = document.createElement('input');
+        audioOffsetEl.setAttribute('type', 'number');
+        audioOffsetEl.value = this.audioOffset;
+        audioOffsetEl.addEventListener('change', (ev) => {
+            this.setAudioOffset(ev.target.value);
+        });
+        audioConfigEl.appendChild(audioOffsetEl);
+
+
+        this.audioEl = document.createElement('audio');
+        if(this.audioFile) {
+            this.audioEl.setAttribute('src', this.audioFile);
+        }
+        this.audioEl.addEventListener('canplaythrough', (ev) => {
+            console.log('loaded audio', ev);
+        });
+        audioConfigEl.appendChild(this.audioEl);
+    }
+
+    setAudioFile(audioFile) {
+        this.audioFile = audioFile;
+        this.audioEl.setAttribute('src', this.audioFile);
+        // TODO update playhead
+        // TODO update this.duration after load
+        this.updateState();
+    }
+
+    setAudioOffset(audioOffset) {
+        this.audioOffset = audioOffset;
+        // TODO update playhead
+        // TODO update this.duration
+        this.updateState();
     }
 
     getDuration() {
@@ -283,7 +566,7 @@ class Player {
         let slices = {};
         for (let i = in_point[0]; i <= out_point[0]; i++) {
             const stroke = this.strokes[i];
-            if(typeof stroke === 'undefined'){
+            if (typeof stroke === 'undefined') {
                 // out point can be Infinity. So interrupt whenever the end is reached
                 break;
             }
@@ -300,10 +583,10 @@ class Player {
     // inactive is what comes before and after.
     // then, playing the video is just running pathRanghe(0, playhead)
     drawStrokePosition(in_point, out_point, show_all) {
-        if(typeof show_all === 'undefined')
+        if (typeof show_all === 'undefined')
             show_all = true;
 
-        this.strokeGroups['before'].setStrokes(this.getStrokesSliceForPathRange([0,0], in_point));
+        this.strokeGroups['before'].setStrokes(this.getStrokesSliceForPathRange([0, 0], in_point));
         this.strokeGroups['annotation'].setStrokes(this.getStrokesSliceForPathRange(in_point, out_point));
         this.strokeGroups['after'].setStrokes(this.getStrokesSliceForPathRange(out_point, [Infinity, Infinity]));
 
@@ -363,7 +646,7 @@ class Player {
         }
 
         // when an outpoint is set, stop playing there
-        if(this.outPointPosition && (next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1])){
+        if (this.outPointPosition && (next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1])) {
             return [null, null];
         }
 
@@ -371,12 +654,12 @@ class Player {
     }
 
     playStrokePosition(path_i, point_i, allow_interrupt) {
-        if(allow_interrupt) {
+        if (allow_interrupt) {
-            if(!this.isPlaying) {
+            if (!this.isPlaying) {
                 console.log('not playing because of interrupt');
                 return;
             }
-        } else{
+        } else {
             this.isPlaying = true;
         }
         this.drawStrokePosition(path_i, point_i);
@@ -409,7 +692,7 @@ class Player {
 
     findPositionForTime(ms) {
         ms = Math.min(Math.max(ms, 0), this.duration);
-        console.log('scrub to', ms)
+        // console.log('scrub to', ms)
         let path_i = 0;
         let point_i = 0;
         this.strokes.every((stroke, index) => {

www/assets/nouislider-15.5.0.css

@@ -0,0 +1,304 @@
+/* Functional styling;
+ * These styles are required for noUiSlider to function.
+ * You don't need to change these rules to apply your design.
+ */
+.noUi-target,
+.noUi-target * {
+  -webkit-touch-callout: none;
+  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+  -webkit-user-select: none;
+  -ms-touch-action: none;
+  touch-action: none;
+  -ms-user-select: none;
+  -moz-user-select: none;
+  user-select: none;
+  -moz-box-sizing: border-box;
+  box-sizing: border-box;
+}
+.noUi-target {
+  position: relative;
+}
+.noUi-base,
+.noUi-connects {
+  width: 100%;
+  height: 100%;
+  position: relative;
+  z-index: 1;
+}
+/* Wrapper for all connect elements.
+ */
+.noUi-connects {
+  overflow: hidden;
+  z-index: 0;
+}
+.noUi-connect,
+.noUi-origin {
+  will-change: transform;
+  position: absolute;
+  z-index: 1;
+  top: 0;
+  right: 0;
+  height: 100%;
+  width: 100%;
+  -ms-transform-origin: 0 0;
+  -webkit-transform-origin: 0 0;
+  -webkit-transform-style: preserve-3d;
+  transform-origin: 0 0;
+  transform-style: flat;
+}
+/* Offset direction
+ */
+.noUi-txt-dir-rtl.noUi-horizontal .noUi-origin {
+  left: 0;
+  right: auto;
+}
+/* Give origins 0 height/width so they don't interfere with clicking the
+ * connect elements.
+ */
+.noUi-vertical .noUi-origin {
+  top: -100%;
+  width: 0;
+}
+.noUi-horizontal .noUi-origin {
+  height: 0;
+}
+.noUi-handle {
+  -webkit-backface-visibility: hidden;
+  backface-visibility: hidden;
+  position: absolute;
+}
+.noUi-touch-area {
+  height: 100%;
+  width: 100%;
+}
+.noUi-state-tap .noUi-connect,
+.noUi-state-tap .noUi-origin {
+  -webkit-transition: transform 0.3s;
+  transition: transform 0.3s;
+}
+.noUi-state-drag * {
+  cursor: inherit !important;
+}
+/* Slider size and handle placement;
+ */
+.noUi-horizontal {
+  height: 18px;
+}
+.noUi-horizontal .noUi-handle {
+  width: 34px;
+  height: 28px;
+  right: -17px;
+  top: -6px;
+}
+.noUi-vertical {
+  width: 18px;
+}
+.noUi-vertical .noUi-handle {
+  width: 28px;
+  height: 34px;
+  right: -6px;
+  bottom: -17px;
+}
+.noUi-txt-dir-rtl.noUi-horizontal .noUi-handle {
+  left: -17px;
+  right: auto;
+}
+/* Styling;
+ * Giving the connect element a border radius causes issues with using transform: scale
+ */
+.noUi-target {
+  background: #FAFAFA;
+  border-radius: 4px;
+  border: 1px solid #D3D3D3;
+  box-shadow: inset 0 1px 1px #F0F0F0, 0 3px 6px -5px #BBB;
+}
+.noUi-connects {
+  border-radius: 3px;
+}
+.noUi-connect {
+  background: #3FB8AF;
+}
+/* Handles and cursors;
+ */
+.noUi-draggable {
+  cursor: ew-resize;
+}
+.noUi-vertical .noUi-draggable {
+  cursor: ns-resize;
+}
+.noUi-handle {
+  border: 1px solid #D9D9D9;
+  border-radius: 3px;
+  background: #FFF;
+  cursor: default;
+  box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #EBEBEB, 0 3px 6px -3px #BBB;
+}
+.noUi-active {
+  box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #DDD, 0 3px 6px -3px #BBB;
+}
+/* Handle stripes;
+ */
+.noUi-handle:before,
+.noUi-handle:after {
+  content: "";
+  display: block;
+  position: absolute;
+  height: 14px;
+  width: 1px;
+  background: #E8E7E6;
+  left: 14px;
+  top: 6px;
+}
+.noUi-handle:after {
+  left: 17px;
+}
+.noUi-vertical .noUi-handle:before,
+.noUi-vertical .noUi-handle:after {
+  width: 14px;
+  height: 1px;
+  left: 6px;
+  top: 14px;
+}
+.noUi-vertical .noUi-handle:after {
+  top: 17px;
+}
+/* Disabled state;
+ */
+[disabled] .noUi-connect {
+  background: #B8B8B8;
+}
+[disabled].noUi-target,
+[disabled].noUi-handle,
+[disabled] .noUi-handle {
+  cursor: not-allowed;
+}
+/* Base;
+ *
+ */
+.noUi-pips,
+.noUi-pips * {
+  -moz-box-sizing: border-box;
+  box-sizing: border-box;
+}
+.noUi-pips {
+  position: absolute;
+  color: #999;
+}
+/* Values;
+ *
+ */
+.noUi-value {
+  position: absolute;
+  white-space: nowrap;
+  text-align: center;
+}
+.noUi-value-sub {
+  color: #ccc;
+  font-size: 10px;
+}
+/* Markings;
+ *
+ */
+.noUi-marker {
+  position: absolute;
+  background: #CCC;
+}
+.noUi-marker-sub {
+  background: #AAA;
+}
+.noUi-marker-large {
+  background: #AAA;
+}
+/* Horizontal layout;
+ *
+ */
+.noUi-pips-horizontal {
+  padding: 10px 0;
+  height: 80px;
+  top: 100%;
+  left: 0;
+  width: 100%;
+}
+.noUi-value-horizontal {
+  -webkit-transform: translate(-50%, 50%);
+  transform: translate(-50%, 50%);
+}
+.noUi-rtl .noUi-value-horizontal {
+  -webkit-transform: translate(50%, 50%);
+  transform: translate(50%, 50%);
+}
+.noUi-marker-horizontal.noUi-marker {
+  margin-left: -1px;
+  width: 2px;
+  height: 5px;
+}
+.noUi-marker-horizontal.noUi-marker-sub {
+  height: 10px;
+}
+.noUi-marker-horizontal.noUi-marker-large {
+  height: 15px;
+}
+/* Vertical layout;
+ *
+ */
+.noUi-pips-vertical {
+  padding: 0 10px;
+  height: 100%;
+  top: 0;
+  left: 100%;
+}
+.noUi-value-vertical {
+  -webkit-transform: translate(0, -50%);
+  transform: translate(0, -50%);
+  padding-left: 25px;
+}
+.noUi-rtl .noUi-value-vertical {
+  -webkit-transform: translate(0, 50%);
+  transform: translate(0, 50%);
+}
+.noUi-marker-vertical.noUi-marker {
+  width: 5px;
+  height: 2px;
+  margin-top: -1px;
+}
+.noUi-marker-vertical.noUi-marker-sub {
+  width: 10px;
+}
+.noUi-marker-vertical.noUi-marker-large {
+  width: 15px;
+}
+.noUi-tooltip {
+  display: block;
+  position: absolute;
+  border: 1px solid #D9D9D9;
+  border-radius: 3px;
+  background: #fff;
+  color: #000;
+  padding: 5px;
+  text-align: center;
+  white-space: nowrap;
+}
+.noUi-horizontal .noUi-tooltip {
+  -webkit-transform: translate(-50%, 0);
+  transform: translate(-50%, 0);
+  left: 50%;
+  bottom: 120%;
+}
+.noUi-vertical .noUi-tooltip {
+  -webkit-transform: translate(0, -50%);
+  transform: translate(0, -50%);
+  top: 50%;
+  right: 120%;
+}
+.noUi-horizontal .noUi-origin > .noUi-tooltip {
+  -webkit-transform: translate(50%, 0);
+  transform: translate(50%, 0);
+  left: auto;
+  bottom: 10px;
+}
+.noUi-vertical .noUi-origin > .noUi-tooltip {
+  -webkit-transform: translate(0, -18px);
+  transform: translate(0, -18px);
+  top: auto;
+  right: 28px;
+}

www/assets/wNumb-1.2.0.min.js

@@ -0,0 +1 @@
+!function(e){"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?module.exports=e():window.wNumb=e()}(function(){"use strict";var o=["decimals","thousand","mark","prefix","suffix","encoder","decoder","negativeBefore","negative","edit","undo"];function w(e){return e.split("").reverse().join("")}function h(e,t){return e.substring(0,t.length)===t}function f(e,t,n){if((e[t]||e[n])&&e[t]===e[n])throw new Error(t)}function x(e){return"number"==typeof e&&isFinite(e)}function n(e,t,n,r,i,o,f,u,s,c,a,p){var d,l,h,g=p,v="",m="";return o&&(p=o(p)),!!x(p)&&(!1!==e&&0===parseFloat(p.toFixed(e))&&(p=0),p<0&&(d=!0,p=Math.abs(p)),!1!==e&&(p=function(e,t){return e=e.toString().split("e"),(+((e=(e=Math.round(+(e[0]+"e"+(e[1]?+e[1]+t:t)))).toString().split("e"))[0]+"e"+(e[1]?e[1]-t:-t))).toFixed(t)}(p,e)),-1!==(p=p.toString()).indexOf(".")?(h=(l=p.split("."))[0],n&&(v=n+l[1])):h=p,t&&(h=w((h=w(h).match(/.{1,3}/g)).join(w(t)))),d&&u&&(m+=u),r&&(m+=r),d&&s&&(m+=s),m+=h,m+=v,i&&(m+=i),c&&(m=c(m,g)),m)}function r(e,t,n,r,i,o,f,u,s,c,a,p){var d,l="";return a&&(p=a(p)),!(!p||"string"!=typeof p)&&(u&&h(p,u)&&(p=p.replace(u,""),d=!0),r&&h(p,r)&&(p=p.replace(r,"")),s&&h(p,s)&&(p=p.replace(s,""),d=!0),i&&function(e,t){return e.slice(-1*t.length)===t}(p,i)&&(p=p.slice(0,-1*i.length)),t&&(p=p.split(t).join("")),n&&(p=p.replace(n,".")),d&&(l+="-"),""!==(l=(l+=p).replace(/[^0-9\.\-.]/g,""))&&(l=Number(l),f&&(l=f(l)),!!x(l)&&l))}function i(e,t,n){var r,i=[];for(r=0;r<o.length;r+=1)i.push(e[o[r]]);return i.push(n),t.apply("",i)}return function e(t){if(!(this instanceof e))return new e(t);"object"==typeof t&&(t=function(e){var t,n,r,i={};for(void 0===e.suffix&&(e.suffix=e.postfix),t=0;t<o.length;t+=1)if(void 0===(r=e[n=o[t]]))"negative"!==n||i.negativeBefore?"mark"===n&&"."!==i.thousand?i[n]=".":i[n]=!1:i[n]="-";else if("decimals"===n){if(!(0<=r&&r<8))throw new Error(n);i[n]=r}else if("encoder"===n||"decoder"===n||"edit"===n||"undo"===n){if("function"!=typeof r)throw new Error(n);i[n]=r}else{if("string"!=typeof r)throw new Error(n);i[n]=r}return f(i,"mark","thousand"),f(i,"prefix","negative"),f(i,"prefix","negativeBefore"),i}(t),this.to=function(e){return i(t,n,e)},this.from=function(e){return i(t,r,e)})}});

www/draw.js

@@ -17,7 +17,7 @@ class Canvas {
         this.wrapperEl.appendChild(this.filenameEl);
 
 
-        this.colors = ["red", "blue", "green"];
+        this.colors = ["black", "red", "blue", "green"];
 
         this.resize();
 

www/play.html

@@ -36,6 +36,10 @@
 			stroke-linecap: round;
 		}
 
+		path.before_in {
+			opacity: .2;
+		}
+
 		#wrapper {
 			position: absolute;
 			top: 0;
@@ -72,12 +76,29 @@
 		.playlist li:hover{
 			color: blue;
 		}
+
+		input[type='range']{
+			position: absolute;
+			z-index: 100;
+			bottom: 0;
+			left:0;
+			right: 0;width: 90%;
+		}
+		.scrubber{
+			position: absolute;
+			z-index: 100;
+			bottom: 0;
+			left:0;
+			right: 0;width: 90%;
+		}
 	</style>
+	<link rel="stylesheet" href="assets/nouislider-15.5.0.css">
 </head>
 
 <body>
 	<div id='interface'>
 	</div>
+	<script src="assets/nouislider-15.5.0.js"></script>
 	<script src="play.js"></script>
 	<script type='text/javascript'>
 		const player = new Player(document.getElementById("interface"));

www/play.js

@@ -1,11 +1,28 @@
 
-
 class Player {
     constructor(wrapperEl) {
         this.wrapperEl = wrapperEl;
         this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
         this.wrapperEl.appendChild(this.svgEl);
 
+        this.scrubberElOld = document.createElement('input');
+        this.scrubberElOld.type = "range";
+        this.scrubberElOld.min = 0;
+        this.scrubberElOld.step = 0.01;
+        this.wrapperEl.appendChild(this.scrubberElOld);
+
+        this.scrubberEl = document.createElement('div');
+        this.scrubberEl.classList.add('scrubber')
+        this.wrapperEl.appendChild(this.scrubberEl);
+
+        this.scrubberElOld.addEventListener("input", (ev) => {
+            this.scrubTo(ev.target.value);
+        })
+
+        this.inPointPosition = null;
+        this.outPointPosition = null;
+        this.currentTime = 0;
+        this.isPlaying = false;
     }
 
     playlist(url) {
@@ -18,17 +35,17 @@ class Player {
             .then(response => response.json())
             .then(data => {
                 let playlist = this.wrapperEl.querySelector('.playlist');
-            if(!playlist) {
+                if (!playlist) {
                     playlist = document.createElement('nav');
                     playlist.classList.add('playlist');
                     this.wrapperEl.appendChild(playlist)
                 }
-            else{
+                else {
                     playlist.innerHTML = "";
                 }
 
                 const listEl = document.createElement("ul");
-            for(let fileUrl of data) {
+                for (let fileUrl of data) {
                     const liEl = document.createElement("li");
                     liEl.innerText = fileUrl
                     liEl.addEventListener('click', (e) => {
@@ -57,53 +74,237 @@ class Player {
 
     }
 
-    playStrokes(drawing){
+    playStrokes(drawing) {
         this.strokes = drawing.shape;
-        this.currentPath = null;
+        this.currentPathI = null;
+        this.currentPointI = null;
         this.dimensions = drawing.dimensions;
         this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
         this.startTime = window.performance.now() - this.strokes[0].points[0][3];
-        this.playStroke(0,1);
+        this.playStrokePosition(0, 1);
+        this.duration = this.getDuration();
+        this.scrubberElOld.max = this.duration;
+        this.playTimout = null;
+
+        const slider = noUiSlider.create(this.scrubberEl, {
+            start: [this.currentTime, this.duration],
+            connect: true,
+            range: {
+                'min': 0,
+                'max': this.duration
+            }
+        });
+
+        slider.on("slide", (values, handle) => {
+            this.isPlaying = false;
+            // console.log(values, handle);
+            // both in and out need to have a value
+            this.inPointPosition = this.findPositionForTime(values[0]);
+            this.outPointPosition = this.findPositionForTime(values[1]);
+            if (handle === 0) {
+                // in point
+                if (
+                    this.currentPathI < this.inPointPosition[0] ||
+                    this.currentPointI < this.inPointPosition[1]) {
+                        this.drawStrokePosition(
+                            // this.inPointPosition[0],
+                            // this.inPointPosition[1],
+                            // always draw at out position, as to see the whole shape of the range
+                            this.outPointPosition[0],
+                            this.outPointPosition[1],
+                        );
+                }
+            }
+            if (handle === 1) {
+                // out point
+                // this.outPointPosition = this.findPositionForTime(values[1]);
+                this.drawStrokePosition(
+                    this.outPointPosition[0],
+                    this.outPointPosition[1],
+                );
+            }
+            // this.inPointPosition = values;
+            // this.outPointPosition = vaalues[0];
+            // this.scrubTo()
+            // this.scrubTo(ev.target.value);
+        });
+
     }
 
-    playStroke(path_i, point_i){
+    getDuration() {
+        const points = this.strokes[this.strokes.length - 1].points;
+        return points[points.length - 1][3];
+    }
+
+    getStrokesForPathRange(in_point, out_point) {
+        // get paths for given range. Also, split path at in & out if necessary.
+        strokes = {};
+        for (let i = in_point[0]; i <= out_point[0]; i++) {
+            const path = this.strokes[i];
+            const in_i = (in_point[0] === i) ? in_point[1] : 0;
+            const out_i = (out_point[0] === i) ? out_point[1] : Math.inf;
+            const points = path.points.slice(in_i, out_i);
+            strokes[i] = points; // preserve indexes
+        }
+        return strokes;
+    }
+
+    // TODO: when drawing, have a group active & inactive.
+    // active is getPathRange(currentIn, currentOut)
+    // inactive is what comes before and after.
+    // then, playing the video is just running pathRanghe(0, playhead)
+    drawStrokePosition(path_i, point_i, show_all) {
+        if(typeof show_all === 'undefined')
+            show_all = false;
+
+        // check if anything is placed that is in the future from the current playhead
+        if (this.currentPathI !== null && this.currentPoint !== null) {
+            if (this.currentPathI > path_i) {
+                console.log('remove', path_i, ' -> ', this.currentPathI)
+                // remove everything that comes later
+                for (let i = path_i + 1; i <= this.currentPathI; i++) {
+                    // console.log('remove', i);
+                    const pathEl = this.svgEl.querySelector(`.path${i}`);
+                    if (pathEl) {
+                        pathEl.parentNode.removeChild(pathEl);
+                    }
+                }
+            }
+        }
+        // an inpoint is set, so we're annotating
+        // make everything coming before translucent
+        if (this.inPointPosition !== null) {
+            const [inPath_i, inPoint_i] = this.inPointPosition;
+            // returns a static NodeList
+            const currentBeforeEls = this.svgEl.querySelectorAll(`.before_in`);
+            for (let currentBeforeEl of currentBeforeEls) {
+                currentBeforeEl.classList.remove('before_in');
+            }
+
+            for (let index = 0; index < inPath_i; index++) {
+                const pathEl = this.svgEl.querySelector(`.path${index}`);
+                if (pathEl) {
+                    pathEl.classList.add('before_in');
+                }
+            }
+        }
+
+        this.currentPathI = path_i;
+        this.currentPointI = point_i;
+
         const path = this.strokes[path_i];
         // console.log(path);
         let pathEl = this.svgEl.querySelector(`.path${path_i}`);
-        if(!pathEl){
+        if (!pathEl) {
             pathEl = document.createElementNS('http://www.w3.org/2000/svg', 'path');
             pathEl.style.stroke = path.color;
-            pathEl.classList.add('path'+path_i)
+            pathEl.classList.add('path' + path_i)
             this.svgEl.appendChild(pathEl)
         }
 
-        const d = this.strokes2D(path.points.slice(0, point_i));
+        const stroke = path.points.slice(0, point_i);
+        const d = this.strokes2D(stroke);
         pathEl.setAttribute('d', d);
 
-        let next_path, next_point,t;
+        this.scrubberElOld.value = path.points[point_i][3];
-        if(path.points.length > point_i + 1){
+        this.currentTime = path.points[point_i][3];
+    }
+
+    getNextPosition(path_i, point_i) {
+        const path = this.strokes[path_i];
+        let next_path, next_point;
+        if (path.points.length > point_i + 1) {
             next_path = path_i;
             next_point = point_i + 1;
-            t = path.points[next_point][3];// - path.points[point_i][3];
             // setTimeout(() => this.playStroke(next_path, next_point), dt);
-        } else if(this.strokes.length > path_i + 1) {
+        } else if (this.strokes.length > path_i + 1) {
             next_path = path_i + 1;
             next_point = 1;
-            t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3];
             // use starttime instead of diff, to prevent floating
         } else {
-            console.log('done');
+            return [null, null];
-            return
         }
 
+        // when an outpoint is set, stop playing there
+        if(next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1]){
+            return [null, null];
+        }
+
+        return [next_path, next_point];
+    }
+
+    playStrokePosition(path_i, point_i, allow_interrupt) {
+        if(allow_interrupt) {
+            if(!this.isPlaying) {
+                console.log('not playing because of interrupt');
+                return;
+            }
+        } else{
+            this.isPlaying = true;
+        }
+        this.drawStrokePosition(path_i, point_i);
+
+        const [next_path, next_point] = this.getNextPosition(path_i, point_i);
+        if (next_path === null) {
+            console.log('done playing');
+            return;
+        }
+
+        const t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3];
+
         const dt = t - (window.performance.now() - this.startTime);
-        setTimeout(() => this.playStroke(next_path, next_point), dt);
+        this.playTimout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt);
     }
 
-    playUntil(path_i){
+    playUntil(path_i) {
         // for scrubber
     }
 
+    scrubTo(ms) {
+        const [path_i, point_i] = this.findPositionForTime(ms);
+        // console.log(path_i, point_i);
+        clearTimeout(this.playTimout);
+        this.playStrokePosition(path_i, point_i);
+        // this.playHead = ms;
+    }
+
+
+
+    findPositionForTime(ms) {
+        ms = Math.min(Math.max(ms, 0), this.duration);
+        console.log('scrub to', ms)
+        let path_i = 0;
+        let point_i = 0;
+        this.strokes.every((stroke, index) => {
+            const startAt = stroke.points[0][3];
+            const endAt = stroke.points[stroke.points.length - 1][3];
+
+            if (startAt > ms) {
+                return false; // too far
+            }
+            if (endAt > ms) {
+                // we're getting close. Find the right point_i
+                path_i = index;
+                stroke.points.every((point, pi) => {
+                    if (point[3] > ms) {
+                        // too far
+                        return false;
+                    }
+                    point_i = pi;
+                    return true;
+                });
+                return false;
+            } else {
+                // in case nothings comes after, we store the last best option thus far
+                path_i = index;
+                point_i = stroke.points.length - 1;
+                return true;
+            }
+
+        });
+        return [path_i, point_i];
+    }
+
 
     strokes2D(strokes) {
         // strokes to a d attribute for a path