Fix playback and fix playback position

files/audio/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

webserver.py

@@ -11,7 +11,6 @@ import html
 import argparse
 import coloredlogs
 import glob
-import csv
 
 
 
@@ -54,10 +53,9 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
     # the client connected
     def open(self, p = None):
         self.__class__.connections.add(self)
-        self.strokes = []
         self.prefix = datetime.datetime.now().strftime('%Y-%m-%d-')
         self.filename = self.prefix + str(self.check_filenr()) + '-' + uuid.uuid4().hex[:6]
-        print(self.filename)
+        logger.info(f"{self.filename=}")
         self.write_message(json.dumps({
             "filename": self.filename
         }))
@@ -66,6 +64,18 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
         files = glob.glob(os.path.join(self.config.storage, self.prefix +'*'))
         return len(files) + 1
     
+    def appendEvent(self, row):
+        # write to an appendable json format. So basically a file that should be wrapped in [] to be json-parsable
+        with open(os.path.join(self.config.storage,self.filename +'.json_appendable'), 'a') as fp:
+            if not self.hasWritten:
+                #metadata to first row, but only on demand
+                fp.write(json.dumps([datetime.datetime.now().strftime("%Y-%m-%d %T"), self.dimensions[0], self.dimensions[1]]))
+                # writer.writerow()
+                self.hasWritten = True
+
+            fp.write(',\n')
+            # first column is color, rest is points
+            fp.write(json.dumps(row))
 
     # the client sent the message
     def on_message(self, message):
@@ -73,26 +83,15 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
 
         try:
             msg = json.loads(message)
-            if msg['action'] == 'stroke':
-                print('stroke!')
-                self.strokes.append([msg['color'], msg['points']])
-
-                with open(os.path.join(self.config.storage,self.filename +'.csv'), 'a') as fp:
-                    writer = csv.writer(fp, delimiter=';')
-                    if not self.hasWritten:
-                        #metadata to first row, but only on demand
-                        writer.writerow([datetime.datetime.now().strftime("%Y-%m-%d %T"), self.dimensions[0], self.dimensions[1]])
-                        self.hasWritten = True
-
-                    # first column is color, rest is points
-                    writer.writerow([msg['color']] +[coordinate for point in msg['points'] for coordinate in point[:4]])
-
-
-            elif msg['action'] == 'dimensions':
+            if msg['event'] == 'stroke':
+                logger.info('stroke')
+                self.appendEvent(msg)
+            elif msg['event'] == 'dimensions':
                 self.dimensions = [int(msg['width']), int(msg['height'])]
                 logger.info(f"{self.dimensions=}")
-
-
+            elif msg['event'] == 'viewbox':
+                logger.info('move or resize')
+                self.appendEvent(msg)
             else:
                 # self.send({'alert': 'Unknown request: {}'.format(message)})
                 logger.warn('Unknown request: {}'.format(message))
@@ -119,6 +118,22 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
         return client in cls.connections
         
 
+class AudioListingHandler(tornado.web.RequestHandler):
+    def initialize(self, config):
+        self.config = config
+        self.audiodir = os.path.join(self.config.storage, 'audio')
+
+    def get(self):
+        # filename = self.get_argument("file", None)
+        self.set_header("Content-Type", "application/json")
+        if not os.path.exists(self.audiodir):
+            names = []
+        else:
+            names = sorted([f"/audio/{name}" for name in os.listdir(self.audiodir) if name not in ['.gitignore']])
+            print(names)
+        self.write(json.dumps(names))
+            
+
 class AnimationHandler(tornado.web.RequestHandler):
     def initialize(self, config):
         self.config = config
@@ -127,29 +142,47 @@ class AnimationHandler(tornado.web.RequestHandler):
         self.set_header("Content-Type", "application/json")
         # filename = self.get_argument("file", None)
         if filename == '':
-            names = sorted([f"/files/{name[:-4]}" for name in os.listdir(self.config.storage) if name not in ['.gitignore']])
-            self.write(json.dumps(names))
+            files = []
+            names = [name for name in os.listdir(self.config.storage) if name.endswith('json_appendable')]
+            for name in names:
+                with open(os.path.join(self.config.storage, name), 'r') as fp:
+                    first_line = fp.readline().strip()
+                    if first_line.endswith(','):
+                        first_line = first_line[:-1]
+                    print(first_line)
+                    metadata = json.loads(first_line)
+                    files.append({
+                        'name': f"/files/{name[:-16]}",
+                        "time": metadata[0],
+                        "dimensions": [metadata[1], metadata[2]],
+                    })
+            
+            files.sort(key=lambda k: k['time'])
+            self.write(json.dumps(files))
         else:
-            path = os.path.join(self.config.storage,os.path.basename(filename)+".csv")
+            path = os.path.join(self.config.storage,os.path.basename(filename)+".json_appendable")
             drawing = {
+                "file": filename,
                 "shape": []
             }
             with open(path, 'r') as fp:
-                strokes = csv.reader(fp,delimiter=';')
-                for i, stroke in enumerate(strokes):
+                events = json.loads('['+fp.read()+']')
+                for i, event in enumerate(events):
                     if i == 0:
                         # metadata on first line
-                        drawing['time'] = stroke[0]
-                        drawing['dimensions'] = [stroke[1], stroke[2]]
-                        continue
-                    color = stroke.pop(0)
-                    points = []
-                    for i in range(int(len(stroke) / 4)):
-                        p =  stroke[i*4:i*4+4]
-                        points.append([float(p[0]), float(p[1]), int(p[2]), float(p[3])])
+                        drawing['time'] = event[0]
+                        drawing['dimensions'] = [event[1], event[2]]
+                    else:
+                        if event['event'] == 'viewbox':
+                            pass
+                        if event['event'] == 'stroke':
+                            # points = []
+                            # for i in range(int(len(stroke) / 4)):
+                            #     p =  stroke[i*4:i*4+4]
+                            #     points.append([float(p[0]), float(p[1]), int(p[2]), float(p[3])])
                             drawing['shape'].append({
-                        'color': color,
-                        'points': points
+                                'color': event['color'],
+                                'points': event['points']
                             }) 
             self.write(json.dumps(drawing))
 
@@ -172,8 +205,59 @@ def strokes2D(strokes):
 
             rel_stroke = [stroke[0] - last_stroke[0], stroke[1] - last_stroke[1]];
             d += f"{rel_stroke[0]},{rel_stroke[1]} "
-        last_stroke = stroke;
-    return d;
+        last_stroke = stroke
+    return d
+
+  
+
+class AnnotationsHandler(tornado.web.RequestHandler):
+    def initialize(self, config):
+        self.config = config
+        self.metadir = os.path.join(self.config.storage, 'metadata')
+
+    def prepare(self):
+        if self.request.headers.get("Content-Type", "").startswith("application/json"):
+            self.json_args = json.loads(self.request.body)
+        else:
+            self.json_args = None
+    
+    def get_filenames(self):
+        return [name[:-16] for name in os.listdir(self.config.storage) if name.endswith('json_appendable')]
+
+    def get(self, filename):
+        self.set_header("Content-Type", "application/json")
+        filenames = self.get_filenames()
+
+        print(filenames, filename)
+
+        if filename not in filenames:
+            raise tornado.web.HTTPError(404)
+
+        
+        meta_file = os.path.join(self.metadir, filename +'.json')
+        if not os.path.exists(meta_file):
+            self.set_status(404)
+            return
+
+        with open(meta_file, 'r') as fp:
+            self.write(json.load(fp))
+
+    def post(self, filename):
+        # filename = self.get_argument("file", None)
+
+        filenames = self.get_filenames()
+        print(filenames, filename)
+        
+        if filename not in filenames:
+            raise tornado.web.HTTPError(404)
+
+        if not os.path.exists(self.metadir):
+            os.mkdir(self.metadir)
+
+        meta_file = os.path.join(self.metadir, filename +'.json')
+        with open(meta_file, 'w') as fp:
+            json.dump(self.json_args, fp)
+
 
 class Server:
     """
@@ -198,6 +282,12 @@ class Server:
                 
                 (r"/files/(.*)", AnimationHandler,
                     {'config': self.config}),
+                (r"/audio/(.+)", tornado.web.StaticFileHandler,
+                    {"path": os.path.join(self.config.storage, 'audio')}),
+                (r"/audio", AudioListingHandler,
+                    {'config': self.config}),
+                (r"/annotations/(.+)", AnnotationsHandler,
+                    {'config': self.config}),
                 (r"/(.*)", StaticFileWithHeaderHandler,
                     {"path": self.web_root}),
             ], debug=True, autoreload=True)

www/annotate.html

@@ -5,17 +5,27 @@
     <meta charset="utf-8">
     <title>Annotate a line animation</title>
     <style media="screen">
+        body {
+            /* background: black;
+            color: white */
+            background: lightgray;
+        }
+
         #sample,
         svg {
             position: absolute;
-			top: 0;
-			left: 0;
-			bottom: 0;
-			right: 0;
-			width: 100%;
-			height: 100%;
+            top: 20px;
+            left: 20px;
+            width: calc(100% - 40px);
+            height: calc(100% - 200px);
             font-family: sans-serif;
             z-index: 2;
+            /* background: white; */
+            /* border: solid 2px lightgray; */
+        }
+
+        svg .background {
+            fill: white
         }
 
         img {
@@ -40,7 +50,9 @@
             opacity: 0.5;
             stroke: gray !important;
         }
-        g.after path, path.before_in {
+
+        g.after path,
+        path.before_in {
             opacity: .1;
             stroke: gray !important;
         }
@@ -59,49 +71,165 @@
             background: rgba(255, 255, 255, 0.7);
         }
 
-		html,
-		body {
-			height: 100%;
-			width: 100%;
-			margin: 0;
-			font-family: sans-serif;
-		}
         
-		.playlist {
-			position: absolute;
-			left: 0;
-			top: 0;
-			z-index: 50;
-		}
-
-		.playlist li{
-			cursor: pointer;
-			line-height: 1.5;
-		}
-		.playlist li:hover{
-			color: blue;
-		}
 
         input[type='range'] {
-			position: absolute;
+            /* position: absolute;
             z-index: 100;
             bottom: 0;
             left: 0;
-			right: 0;width: 90%;
+            right: 0; */
+            width: 100%;
         }
-		.scrubber{
+
+        .controls {
             position: absolute !important;
             z-index: 100;
-			bottom: 30px;
-			left:0;
-			right: 0;width: 90%;
+            bottom: 10px;
+            left: 5%;
+            right: 0;
+            width: 90%;
+        }
+
+        .scrubber {}
+
+        .tags {
+            line-height: 40px;
+            display: flex;
+            flex-direction: row;
+            padding: 0;
+            margin: 0;
+        }
+
+        .tags li {
+            display: block;
+            padding: 5px;
+            border: solid 1px darkgray;
+            flex-grow: 1;
+            text-align: center;
+        }
+
+        .tags li:hover {
+            cursor: pointer;
+            background: darkgray;
+        }
+
+        .tags li.selected {
+            background: lightsteelblue;
+        }
+
+        .tags li.annotation-rm {
+            /* display: none; */
+            overflow: hidden;
+            color: red;
+            font-size: 30px;
+            width: 0;
+            flex-grow: 0;
+            padding: 5px 0;
+            transition: width .3s;
+            pointer-events: none;
+            border: none;
+            direction: rtl;
+            /* hide behind bar, instead into nothing */
+        }
+
+        .selected-annotation .tags li.annotation-rm {
+            color: red;
+            display: block;
+            width: 30px;
+            pointer-events: all;
+        }
+
+        .tags li span {
+            display: inline-block;
+            width: 20px;
+            height: 20px;
+            margin-right: 10px;
+            vertical-align: middle;
+            border-radius: 5px;
+        }
+
+        .annotations {
+            height: 10px;
+            /* border: solid 1px darkgray; */
+            position: relative;
+        }
+
+        .annotations>div {
+            opacity: .4;
+            background: lightseagreen;
+
+            position: absolute;
+            bottom: 0;
+            top: 0;
+        }
+
+        .annotations>div:hover,
+        .annotations>div.selected {
+            opacity: 1;
+            cursor: pointer;
+        }
+
+        .annotation-test {
+            background-color: red !important;
+        }
+
+        .annotation-another {
+            background-color: blue !important;
+        }
+
+        .annotation-google {
+            background-color: blueviolet !important;
+        }
+
+        .unsaved::before {
+            content: '*';
+            color: red;
+            display: inline-block;
+            text-align: center;
+            font-size: 30px;
+            position: absolute;
+            top: 10px;
+            left: 10px;
+        }
+
+        .saved::before {
+            content: '\2713';
+            display: inline-block;
+            color: green;
+            text-align: center;
+            font-size: 30px;
+            position: absolute;
+            top: 10px;
+            left: 10px;
         }
 
         .noUi-horizontal .noUi-touch-area {
             cursor: ew-resize;
         }
+
+        .audioconfig{
+            z-index: 9;
+            background:black;
+            color: white;
+            position: relative;
+            width: 100px; /* as wide as audio controls only */
+            overflow: hidden;
+            white-space: nowrap;
+        }
+        .audioconfig:hover{
+            width: auto;
+        }
+        .audioconfig select, .audioconfig input{
+            margin:10px;
+        }
+        audio{
+            vertical-align: middle;
+            width: 100px; /* hides seek head */
+        }
     </style>
     <link rel="stylesheet" href="assets/nouislider-15.5.0.css">
+    <link rel="stylesheet" href="core.css">
 </head>
 
 <body>
@@ -110,9 +238,18 @@
     <script src="assets/nouislider-15.5.0.js"></script>
     <script src="assets/wNumb-1.2.0.min.js"></script>
     <script src="annotate.js"></script>
+	<script src="playlist.js"></script>
     <script type='text/javascript'>
-		const player = new Player(document.getElementById("interface"));
-		player.playlist('/files/');
+        let ann;
+        if (location.search) {
+            ann = new Annotator(
+                document.getElementById("interface"),
+                ["test", "another", "google"],
+                location.search.substring(1)
+            );
+        } else {
+            const playlist = new Playlist(document.getElementById("interface"), '/files/');
+        }
     </script>
 </body>
 

www/annotate.js

@@ -1,6 +1,6 @@
 class Annotation {
-    constructor(annotation, t_in, t_out) {
-        this.annotation = annotation;
+    constructor(tag, t_in, t_out) {
+        this.tag = tag;
         this.t_in = t_in;
         this.t_out = t_out;
     }
@@ -13,7 +13,6 @@ class StrokeGroup{
     }
 
     setStrokes(strokes) {
-        console.log('set strokes',strokes);
         const pathEls = this.g.querySelectorAll('path');
         let indexes = Object.keys(strokes);
         for (let pathEl of pathEls) {
@@ -32,7 +31,6 @@ class StrokeGroup{
             // this has now been processed
             indexes.splice(indexes.indexOf(i), 1);
         }
-        console.log(indexes);
 
         // new strokes
         indexes.forEach(index => {
@@ -93,7 +91,7 @@ class StrokeSlice{
     constructor(stroke, i_in, i_out) {
         this.stroke = stroke; // Stroke
         this.i_in = typeof i_in === 'undefined' ? 0 : i_in;
-        this.i_out = typeof i_out === 'undefined' ? this.stroke.points.length : i_out;
+        this.i_out = typeof i_out === 'undefined' ? this.stroke.points.length - 1 : i_out;
     }
 
     getSliceId() {
@@ -102,7 +100,7 @@ class StrokeSlice{
 
     // compatible with Stroke()
     get points() {
-        return this.stroke.points.slice(this.i_in, this.i_out);
+        return this.stroke.points.slice(this.i_in, this.i_out + 1);
     }
 
     // compatible with Stroke()
@@ -111,26 +109,67 @@ class StrokeSlice{
     }
 }
 
-class Player {
-    constructor(wrapperEl) {
+class Annotator {
+    constructor(wrapperEl, tags, fileurl) {
         this.wrapperEl = wrapperEl;
         this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
         this.wrapperEl.appendChild(this.svgEl);
 
+
+        this.controlsEl = document.createElement('div');
+        this.controlsEl.classList.add('controls')
+        this.wrapperEl.appendChild(this.controlsEl);
+
         this.scrubberElOld = document.createElement('input');
         this.scrubberElOld.type = "range";
         this.scrubberElOld.min = 0;
         this.scrubberElOld.step = 0.01;
-        this.wrapperEl.appendChild(this.scrubberElOld);
-
-        this.scrubberEl = document.createElement('div');
-        this.scrubberEl.classList.add('scrubber')
-        this.wrapperEl.appendChild(this.scrubberEl);
+        this.controlsEl.appendChild(this.scrubberElOld);
 
         this.scrubberElOld.addEventListener("input", (ev) => {
             this.scrubTo(ev.target.value);
         })
 
+        this.scrubberEl = document.createElement('div');
+        this.scrubberEl.classList.add('scrubber')
+        this.controlsEl.appendChild(this.scrubberEl);
+
+        this.tagsEl = document.createElement('ul');
+        this.tagsEl.classList.add('tags');
+        for (let tag of tags) {
+            let tagEl = document.createElement('li');
+            tagEl.classList.add('tag');
+            tagEl.dataset.tag = tag;
+            tagEl.innerText = tag;
+            tagEl.addEventListener('click', (e) => {
+                this.addTag(tag, this.inPointPosition, this.outPointPosition);
+            })
+
+            let signEl = document.createElement('span');
+            signEl.classList.add('annotation-' + tag);
+            tagEl.prepend(signEl);
+            this.tagsEl.appendChild(tagEl);
+        }
+        let tagEl = document.createElement('li');
+        tagEl.classList.add('tag');
+        tagEl.classList.add('annotation-rm');
+        tagEl.dataset.tag = 'rm';
+        tagEl.title = "Remove annotation";
+        tagEl.innerHTML = "×";
+        tagEl.addEventListener('click', (e) => {
+            if (this.selectedAnnotation) {
+                this.removeAnnotation(this.selectedAnnotationI);
+            }
+        });
+        this.tagsEl.appendChild(tagEl);
+
+        this.controlsEl.appendChild(this.tagsEl);
+
+        this.annotationsEl = document.createElement('div');
+        this.annotationsEl.classList.add('annotations')
+        this.controlsEl.appendChild(this.annotationsEl);
+
+
         this.inPointPosition = null;
         this.outPointPosition = null;
         this.currentTime = 0;
@@ -145,41 +184,86 @@ class Player {
             this.strokeGroups[group] = new StrokeGroup(groupEl, this);
         });
 
-        this.annotations = []
+        this.annotations = [];
+
+        this.play(fileurl);
     }
 
-    playlist(url) {
-        const request = new Request(url, {
-            method: 'GET',
+    updateAnnotations(save) {
+
+        this.annotationsEl.innerHTML = "";
+        for (let annotation_i in this.annotations) {
+            const annotation = this.annotations[annotation_i];
+            this.annotationEl = document.createElement('div');
+            const left = (annotation.t_in / this.duration) * 100;
+            const right = 100 - (annotation.t_out / this.duration) * 100;
+            this.annotationEl.style.left = left + '%';
+            this.annotationEl.style.right = right + '%';
+
+            this.annotationEl.classList.add('annotation-' + annotation.tag);
+            if (this.selectedAnnotationI == annotation_i) {
+                this.annotationEl.classList.add('selected');
+            }
+            this.annotationEl.title = annotation.tag;
+
+            this.annotationEl.addEventListener('mouseover', (e) => {
+
+            });
+            this.annotationEl.addEventListener('mouseout', (e) => {
+
+            });
+            this.annotationEl.addEventListener('click', (e) => {
+                if (this.selectedAnnotationI == annotation_i) {
+                    this.deselectAnnotation(false);
+                } else {
+                    this.selectAnnotation(annotation_i);
+                }
             });
 
-
-        fetch(request)
-            .then(response => response.json())
-            .then(data => {
-                let playlist = this.wrapperEl.querySelector('.playlist');
-                if (!playlist) {
-                    playlist = document.createElement('nav');
-                    playlist.classList.add('playlist');
-                    this.wrapperEl.appendChild(playlist)
-                }
-                else {
-                    playlist.innerHTML = "";
+            this.annotationsEl.appendChild(this.annotationEl);
         }
 
-                const listEl = document.createElement("ul");
-                for (let fileUrl of data) {
-                    const liEl = document.createElement("li");
-                    liEl.innerText = fileUrl
-                    liEl.addEventListener('click', (e) => {
-                        this.play(fileUrl);
-                        playlist.style.display = "none";
-                    });
-                    listEl.appendChild(liEl);
+        this.tagsEl.childNodes.forEach(tagEl => {
+            if (this.selectedAnnotation && this.selectedAnnotation.tag == tagEl.dataset.tag) {
+                tagEl.classList.add('selected')
+            } else {
+                tagEl.classList.remove('selected')
             }
-                playlist.appendChild(listEl);
-                // do something with the data sent in the request
         });
+
+        if (save) {
+            this.updateState();
+        }
+    }
+
+    selectAnnotation(annotation_i) {
+        this.selectedAnnotationI = annotation_i;
+        this.selectedAnnotation = this.annotations[annotation_i];
+
+        this.slider.set([this.selectedAnnotation.t_in, this.selectedAnnotation.t_out]);
+
+        this.inPointPosition = this.findPositionForTime(this.selectedAnnotation.t_in);
+        this.outPointPosition = this.findPositionForTime(this.selectedAnnotation.t_out);
+        this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
+
+        this.updateAnnotations(false); //selects the right tag & highlights the annotation
+
+        this.wrapperEl.classList.add('selected-annotation');
+    }
+
+    deselectAnnotation(keep_position) {
+        if (this.selectedAnnotation)
+            this.currentTime = this.selectedAnnotation.t_out;
+
+        this.wrapperEl.classList.remove('selected-annotation');
+
+        this.selectedAnnotationI = null;
+        this.selectedAnnotation = null;
+
+        if (!keep_position) {
+            this.setUpAnnotator();
+        }
+        this.updateAnnotations(false); // selects the right tag & highlights the annotation
     }
 
     play(file) {
@@ -190,25 +274,179 @@ class Player {
         fetch(request)
             .then(response => response.json())
             .then(data => {
-                this.playStrokes(data)
+                const metadata_req = new Request(`/annotations/${data.file}`, {
+                    method: 'GET',
+                });
+                fetch(metadata_req)
+                    .then(response => response.ok ? response.json() : null)
+                    .then(metadata => {
+                        this.playStrokes(data, metadata)
+                    })
+                    .catch(e => console.log(e));
                 // do something with the data sent in the request
             });
+    }
+
+    updateState() {
+        const state = {
+            'file': this.filename,
+            'annotations': this.annotations,
+            'audio': {
+                'file': this.audioFile,
+                'offset': this.audioOffset,
+            }
+        }
+        const newState = JSON.stringify(state);
+        if (newState == this.state) {
+            return;
+        }
+
+        this.wrapperEl.classList.remove('saved');
+        this.wrapperEl.classList.add('unsaved');
+        this.state = newState;
+        // autosave on state change:
+        this.save(newState);
+    }
+
+    setSaved(state) {
+        if (this.state != state) {
+            console.log('already outdated');
+        }
+        else {
+            this.wrapperEl.classList.add('saved');
+            this.wrapperEl.classList.remove('unsaved');
+        }
+    }
+
+    save(state) {
+        const request = new Request("/annotations/" + this.filename, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: state
+        });
+        fetch(request)
+            .then((response) => {
+                if (response.ok) {
+                    this.setSaved(state);
+                }
+                else {
+                    throw Error('Something went wrong');
+                }
+            })
+            .catch((error) => {
+                console.log(error);
+            });
 
     }
 
-    playStrokes(drawing) {
+    removeAnnotation(annotation_i) {
+        this.deselectAnnotation(true);
+        this.annotations.splice(annotation_i, 1);
+        this.updateAnnotations(true);
+    }
+
+    addTag(tag) {
+        if (this.selectedAnnotation) {
+            this.selectedAnnotation.tag = tag;
+            this.updateAnnotations(true);
+        } else {
+
+            // TODO this.slider values for in and out
+            const [t_in, t_out] = this.slider.get();
+            if (this.slider) {
+                this.slider.destroy();
+            }
+
+            this.annotations.push(new Annotation(tag, t_in, t_out));
+            this.updateAnnotations(true);
+
+            this.currentTime = t_out;
+            this.setUpAnnotator();
+        }
+    }
+
+
+    setUpAnnotator() {
+        this.inPointPosition = this.findPositionForTime(this.currentTime);
+        this.outPointPosition = this.findPositionForTime(this.duration);
+
+        if (this.scrubberEl.noUiSlider) {
+            this.slider.destroy();
+        }
+
+        this.slider = noUiSlider.create(this.scrubberEl, {
+            start: [this.currentTime, this.duration],
+            connect: true,
+            range: {
+                'min': 0,
+                'max': this.duration
+            },
+            tooltips: [
+                this.formatter,
+                this.formatter
+            ],
+            // pips: {
+            //     mode: 'range',
+            //     density: 3,
+            //     format: this.formatter
+            // }
+        });
+
+        this.slider.on("slide", (values, handle) => {
+            this.isPlaying = false;
+            this.inPointPosition = this.findPositionForTime(values[0]);
+            this.outPointPosition = this.findPositionForTime(values[1]);
+            this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
+
+            // console.log(this.selectedAnnotation);
+            if (this.selectedAnnotation) {
+                this.selectedAnnotation.t_in = values[0];
+                this.selectedAnnotation.t_out = values[1];
+                this.updateAnnotations(false);
+            }
+        });
+        this.slider.on("end", (values, handle) => {
+            if (this.selectedAnnotation) {
+                this.updateAnnotations(true);
+            }
+            this.playAudioSegment(values[0], values[1]);
+        })
+
+        this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
+    }
+
+    playStrokes(drawing, metadata) {
+        this.audioOffset = 0;
+        if (metadata) {
+            this.annotations = metadata.annotations;
+            this.audioFile = metadata.hasOwnProperty('audio') ? metadata.audio.file : null;
+            this.audioOffset = metadata.hasOwnProperty('audio') ? metadata.audio.offset : 0;
+            //
+            // load any saved metadata
+        }
+        this.filename = drawing.file;
         this.strokes = drawing.shape.map(s => new Stroke(s['color'], s['points']));
         this.currentPathI = null;
         this.currentPointI = null;
         this.dimensions = drawing.dimensions;
         this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
+
+        let bgEl = document.createElementNS('http://www.w3.org/2000/svg', 'rect');
+        bgEl.setAttribute("x", 0);
+        bgEl.setAttribute("y", 0);
+        bgEl.setAttribute("width", this.dimensions[0]);
+        bgEl.setAttribute("height", this.dimensions[1]);
+        bgEl.classList.add('background');
+        this.svgEl.prepend(bgEl);
+
         this.startTime = window.performance.now() - this.strokes[0].points[0][3];
-        this.playStrokePosition(0, 1);
         this.duration = this.getDuration();
         this.scrubberElOld.max = this.duration;
         this.playTimout = null;
 
-        const formatter = wNumb({
+        this.formatter = wNumb({
             decimals: 2,
             edit: (time) => {
                 const s = Math.floor(time / 1000);
@@ -218,59 +456,128 @@ class Player {
                 return `${minutes}:${seconds}:${ms}`;
             }
         });
-        const slider = noUiSlider.create(this.scrubberEl, {
-            start: [this.currentTime, this.duration],
-            connect: true,
-            range: {
-                'min': 0,
-                'max': this.duration
-            },
-            tooltips: [
-                formatter, 
-                formatter 
-            ],
-            // pips: {
-            //     mode: 'range',
-            //     density: 3,
-            //     format: formatter
-            // }
+
+        this.setUpAnnotator()
+        this.updateAnnotations(false);
+
+        this.setupAudioConfig();
+
+        // this.playStrokePosition(0, 1);
+    }
+
+    setupAudioConfig() {
+        // audio config
+        let audioConfigEl = document.createElement('div');
+        audioConfigEl.classList.add('audioconfig')
+        this.wrapperEl.appendChild(audioConfigEl);
+
+        let audioSelectEl = document.createElement('select');
+        audioSelectEl.classList.add('audioselect');
+        audioConfigEl.appendChild(audioSelectEl);
+
+        fetch('/audio')
+            .then(response => response.json())
+            .then(data => {
+                data.unshift(''); // add empty, to deselect any file
+                data.forEach(audioFile => {
+                    let optionEl = document.createElement('option');
+                    optionEl.selected = this.audioFile == audioFile;
+                    optionEl.innerText = audioFile;
+                    audioSelectEl.appendChild(optionEl);
+                });
+            })
+
+        audioSelectEl.addEventListener('change', (ev) => {
+            this.setAudioFile(ev.target.value);
         });
 
-        slider.on("slide", (values, handle) => {
-            this.isPlaying = false;
-            // console.log(values, handle);
-            // both in and out need to have a value
-            this.inPointPosition = this.findPositionForTime(values[0]);
-            this.outPointPosition = this.findPositionForTime(values[1]);
-            // if (handle === 0) {
-            //     // in point
-            //     if (
-            //         this.currentPathI < this.inPointPosition[0] ||
-            //         this.currentPointI < this.inPointPosition[1]) {
-            //             this.drawStrokePosition(
-            //                 // this.inPointPosition[0],
-            //                 // this.inPointPosition[1],
-            //                 // always draw at out position, as to see the whole shape of the range
-            //                 this.outPointPosition[0],
-            //                 this.outPointPosition[1],
-            //             );
-            //     }
-            // }
-            // if (handle === 1) {
-            //     // out point
-            //     // this.outPointPosition = this.findPositionForTime(values[1]);
-            //     this.drawStrokePosition(
-            //         this.outPointPosition[0],
-            //         this.outPointPosition[1],
-            //     );
-            // }
-            this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
-            // this.inPointPosition = values;
-            // this.outPointPosition = vaalues[0];
-            // this.scrubTo()
-            // this.scrubTo(ev.target.value);
-        });
 
+        let audioOffsetTextEl = document.createElement('label');
+        audioOffsetTextEl.innerText = "Offset (s)";
+        audioConfigEl.appendChild(audioOffsetTextEl);
+
+        let audioOffsetEl = document.createElement('input');
+        audioOffsetEl.setAttribute('type', 'number');
+        audioOffsetEl.setAttribute('step', '.01');
+        audioOffsetEl.value = this.audioOffset ?? 0;
+        audioOffsetEl.addEventListener('change', (ev) => {
+            this.setAudioOffset(ev.target.value);
+        });
+        audioOffsetTextEl.appendChild(audioOffsetEl);
+
+
+        this.audioEl = document.createElement('audio');
+        if (this.audioFile) {
+            this.audioEl.setAttribute('src', this.audioFile);
+        }
+        this.audioEl.setAttribute('controls', true);
+        this.audioEl.addEventListener('canplaythrough', (ev) => {
+            console.log('loaded audio', ev);
+            this.audioEl.play();
+        });
+        // this.audioEl.addEventListener('seeked', (ev)=>{
+        //     console.log(ev);
+        // })
+        audioConfigEl.prepend(this.audioEl);
+    }
+
+    setAudioFile(audioFile) {
+        this.audioFile = audioFile;
+        this.audioEl.setAttribute('src', this.audioFile);
+        // this.audioEl.play();
+        // TODO update playhead
+        // TODO update this.duration after load
+        this.updateState();
+    }
+
+    setAudioOffset(audioOffset) {
+        this.audioOffset = audioOffset;
+        // TODO update playhead
+        // TODO update this.duration
+        this.updateState();
+    }
+
+    /**
+     * @param float time time is ms
+     * @returns float
+     */
+    getAudioTime(time) {
+        return Number.parseFloat(time) + (this.audioOffset * 1000 ?? 0);
+    }
+
+    /**
+     * 
+     * @param float t_in in point time, in ms
+     * @param float t_out out point time, in ms
+     */
+    playAudioSegment(t_in, t_out) {
+        if (this.audioStartTimeout) clearTimeout(this.audioStartTimeout);
+        if (this.audioEndTimeout) clearTimeout(this.audioEndTimeout);
+
+        // TODO, handle playback delay
+        const t_start = this.getAudioTime(t_in); // in ms
+        const t_diff = t_out - t_in; // in ms
+
+        console.log('set time', t_in, t_start, typeof t_start, typeof t_in, t_start < 0);
+        this.audioEl.pause();
+
+        if (t_start < 0) {
+            if (t_diff <= t_start * -1) {
+                console.log('no audio playback in segment', t_start, t_diff);
+            } else {
+                console.log('huh?', t_start, t_diff);
+                // a negative audiooffset delays playback from the start
+                // this.audioStartTimeout = setTimeout((e) => this.audioEl.play(), t*-1000);
+                this.audioStartTimeout = setTimeout((e) => { this.audioEl.currentTime = 0 }, t_start * -1); // triggers play with "seeked" event
+                // this.audioEl.currentTime = 0;
+            }
+        } else {
+            this.audioEl.currentTime = t_start / 1000;
+            // this.audioEl.play(); // play is done in "seeked" evenlistener
+            console.log(this.audioEl.currentTime, t_start, t_in, t_out)
+        }
+
+        this.audioEndTimeout = setTimeout((e) => this.audioEl.pause(), t_diff);
     }
 
     getDuration() {
@@ -409,7 +716,7 @@ class Player {
 
     findPositionForTime(ms) {
         ms = Math.min(Math.max(ms, 0), this.duration);
-        console.log('scrub to', ms)
+        // console.log('scrub to', ms)
         let path_i = 0;
         let point_i = 0;
         this.strokes.every((stroke, index) => {

www/assets/nouislider-15.5.0.css

@@ -0,0 +1,304 @@
+/* Functional styling;
+ * These styles are required for noUiSlider to function.
+ * You don't need to change these rules to apply your design.
+ */
+.noUi-target,
+.noUi-target * {
+  -webkit-touch-callout: none;
+  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+  -webkit-user-select: none;
+  -ms-touch-action: none;
+  touch-action: none;
+  -ms-user-select: none;
+  -moz-user-select: none;
+  user-select: none;
+  -moz-box-sizing: border-box;
+  box-sizing: border-box;
+}
+.noUi-target {
+  position: relative;
+}
+.noUi-base,
+.noUi-connects {
+  width: 100%;
+  height: 100%;
+  position: relative;
+  z-index: 1;
+}
+/* Wrapper for all connect elements.
+ */
+.noUi-connects {
+  overflow: hidden;
+  z-index: 0;
+}
+.noUi-connect,
+.noUi-origin {
+  will-change: transform;
+  position: absolute;
+  z-index: 1;
+  top: 0;
+  right: 0;
+  height: 100%;
+  width: 100%;
+  -ms-transform-origin: 0 0;
+  -webkit-transform-origin: 0 0;
+  -webkit-transform-style: preserve-3d;
+  transform-origin: 0 0;
+  transform-style: flat;
+}
+/* Offset direction
+ */
+.noUi-txt-dir-rtl.noUi-horizontal .noUi-origin {
+  left: 0;
+  right: auto;
+}
+/* Give origins 0 height/width so they don't interfere with clicking the
+ * connect elements.
+ */
+.noUi-vertical .noUi-origin {
+  top: -100%;
+  width: 0;
+}
+.noUi-horizontal .noUi-origin {
+  height: 0;
+}
+.noUi-handle {
+  -webkit-backface-visibility: hidden;
+  backface-visibility: hidden;
+  position: absolute;
+}
+.noUi-touch-area {
+  height: 100%;
+  width: 100%;
+}
+.noUi-state-tap .noUi-connect,
+.noUi-state-tap .noUi-origin {
+  -webkit-transition: transform 0.3s;
+  transition: transform 0.3s;
+}
+.noUi-state-drag * {
+  cursor: inherit !important;
+}
+/* Slider size and handle placement;
+ */
+.noUi-horizontal {
+  height: 18px;
+}
+.noUi-horizontal .noUi-handle {
+  width: 34px;
+  height: 28px;
+  right: -17px;
+  top: -6px;
+}
+.noUi-vertical {
+  width: 18px;
+}
+.noUi-vertical .noUi-handle {
+  width: 28px;
+  height: 34px;
+  right: -6px;
+  bottom: -17px;
+}
+.noUi-txt-dir-rtl.noUi-horizontal .noUi-handle {
+  left: -17px;
+  right: auto;
+}
+/* Styling;
+ * Giving the connect element a border radius causes issues with using transform: scale
+ */
+.noUi-target {
+  background: #FAFAFA;
+  border-radius: 4px;
+  border: 1px solid #D3D3D3;
+  box-shadow: inset 0 1px 1px #F0F0F0, 0 3px 6px -5px #BBB;
+}
+.noUi-connects {
+  border-radius: 3px;
+}
+.noUi-connect {
+  background: #3FB8AF;
+}
+/* Handles and cursors;
+ */
+.noUi-draggable {
+  cursor: ew-resize;
+}
+.noUi-vertical .noUi-draggable {
+  cursor: ns-resize;
+}
+.noUi-handle {
+  border: 1px solid #D9D9D9;
+  border-radius: 3px;
+  background: #FFF;
+  cursor: default;
+  box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #EBEBEB, 0 3px 6px -3px #BBB;
+}
+.noUi-active {
+  box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #DDD, 0 3px 6px -3px #BBB;
+}
+/* Handle stripes;
+ */
+.noUi-handle:before,
+.noUi-handle:after {
+  content: "";
+  display: block;
+  position: absolute;
+  height: 14px;
+  width: 1px;
+  background: #E8E7E6;
+  left: 14px;
+  top: 6px;
+}
+.noUi-handle:after {
+  left: 17px;
+}
+.noUi-vertical .noUi-handle:before,
+.noUi-vertical .noUi-handle:after {
+  width: 14px;
+  height: 1px;
+  left: 6px;
+  top: 14px;
+}
+.noUi-vertical .noUi-handle:after {
+  top: 17px;
+}
+/* Disabled state;
+ */
+[disabled] .noUi-connect {
+  background: #B8B8B8;
+}
+[disabled].noUi-target,
+[disabled].noUi-handle,
+[disabled] .noUi-handle {
+  cursor: not-allowed;
+}
+/* Base;
+ *
+ */
+.noUi-pips,
+.noUi-pips * {
+  -moz-box-sizing: border-box;
+  box-sizing: border-box;
+}
+.noUi-pips {
+  position: absolute;
+  color: #999;
+}
+/* Values;
+ *
+ */
+.noUi-value {
+  position: absolute;
+  white-space: nowrap;
+  text-align: center;
+}
+.noUi-value-sub {
+  color: #ccc;
+  font-size: 10px;
+}
+/* Markings;
+ *
+ */
+.noUi-marker {
+  position: absolute;
+  background: #CCC;
+}
+.noUi-marker-sub {
+  background: #AAA;
+}
+.noUi-marker-large {
+  background: #AAA;
+}
+/* Horizontal layout;
+ *
+ */
+.noUi-pips-horizontal {
+  padding: 10px 0;
+  height: 80px;
+  top: 100%;
+  left: 0;
+  width: 100%;
+}
+.noUi-value-horizontal {
+  -webkit-transform: translate(-50%, 50%);
+  transform: translate(-50%, 50%);
+}
+.noUi-rtl .noUi-value-horizontal {
+  -webkit-transform: translate(50%, 50%);
+  transform: translate(50%, 50%);
+}
+.noUi-marker-horizontal.noUi-marker {
+  margin-left: -1px;
+  width: 2px;
+  height: 5px;
+}
+.noUi-marker-horizontal.noUi-marker-sub {
+  height: 10px;
+}
+.noUi-marker-horizontal.noUi-marker-large {
+  height: 15px;
+}
+/* Vertical layout;
+ *
+ */
+.noUi-pips-vertical {
+  padding: 0 10px;
+  height: 100%;
+  top: 0;
+  left: 100%;
+}
+.noUi-value-vertical {
+  -webkit-transform: translate(0, -50%);
+  transform: translate(0, -50%);
+  padding-left: 25px;
+}
+.noUi-rtl .noUi-value-vertical {
+  -webkit-transform: translate(0, 50%);
+  transform: translate(0, 50%);
+}
+.noUi-marker-vertical.noUi-marker {
+  width: 5px;
+  height: 2px;
+  margin-top: -1px;
+}
+.noUi-marker-vertical.noUi-marker-sub {
+  width: 10px;
+}
+.noUi-marker-vertical.noUi-marker-large {
+  width: 15px;
+}
+.noUi-tooltip {
+  display: block;
+  position: absolute;
+  border: 1px solid #D9D9D9;
+  border-radius: 3px;
+  background: #fff;
+  color: #000;
+  padding: 5px;
+  text-align: center;
+  white-space: nowrap;
+}
+.noUi-horizontal .noUi-tooltip {
+  -webkit-transform: translate(-50%, 0);
+  transform: translate(-50%, 0);
+  left: 50%;
+  bottom: 120%;
+}
+.noUi-vertical .noUi-tooltip {
+  -webkit-transform: translate(0, -50%);
+  transform: translate(0, -50%);
+  top: 50%;
+  right: 120%;
+}
+.noUi-horizontal .noUi-origin > .noUi-tooltip {
+  -webkit-transform: translate(50%, 0);
+  transform: translate(50%, 0);
+  left: auto;
+  bottom: 10px;
+}
+.noUi-vertical .noUi-origin > .noUi-tooltip {
+  -webkit-transform: translate(0, -18px);
+  transform: translate(0, -18px);
+  top: auto;
+  right: 28px;
+}

www/assets/wNumb-1.2.0.min.js

@@ -0,0 +1 @@
+!function(e){"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?module.exports=e():window.wNumb=e()}(function(){"use strict";var o=["decimals","thousand","mark","prefix","suffix","encoder","decoder","negativeBefore","negative","edit","undo"];function w(e){return e.split("").reverse().join("")}function h(e,t){return e.substring(0,t.length)===t}function f(e,t,n){if((e[t]||e[n])&&e[t]===e[n])throw new Error(t)}function x(e){return"number"==typeof e&&isFinite(e)}function n(e,t,n,r,i,o,f,u,s,c,a,p){var d,l,h,g=p,v="",m="";return o&&(p=o(p)),!!x(p)&&(!1!==e&&0===parseFloat(p.toFixed(e))&&(p=0),p<0&&(d=!0,p=Math.abs(p)),!1!==e&&(p=function(e,t){return e=e.toString().split("e"),(+((e=(e=Math.round(+(e[0]+"e"+(e[1]?+e[1]+t:t)))).toString().split("e"))[0]+"e"+(e[1]?e[1]-t:-t))).toFixed(t)}(p,e)),-1!==(p=p.toString()).indexOf(".")?(h=(l=p.split("."))[0],n&&(v=n+l[1])):h=p,t&&(h=w((h=w(h).match(/.{1,3}/g)).join(w(t)))),d&&u&&(m+=u),r&&(m+=r),d&&s&&(m+=s),m+=h,m+=v,i&&(m+=i),c&&(m=c(m,g)),m)}function r(e,t,n,r,i,o,f,u,s,c,a,p){var d,l="";return a&&(p=a(p)),!(!p||"string"!=typeof p)&&(u&&h(p,u)&&(p=p.replace(u,""),d=!0),r&&h(p,r)&&(p=p.replace(r,"")),s&&h(p,s)&&(p=p.replace(s,""),d=!0),i&&function(e,t){return e.slice(-1*t.length)===t}(p,i)&&(p=p.slice(0,-1*i.length)),t&&(p=p.split(t).join("")),n&&(p=p.replace(n,".")),d&&(l+="-"),""!==(l=(l+=p).replace(/[^0-9\.\-.]/g,""))&&(l=Number(l),f&&(l=f(l)),!!x(l)&&l))}function i(e,t,n){var r,i=[];for(r=0;r<o.length;r+=1)i.push(e[o[r]]);return i.push(n),t.apply("",i)}return function e(t){if(!(this instanceof e))return new e(t);"object"==typeof t&&(t=function(e){var t,n,r,i={};for(void 0===e.suffix&&(e.suffix=e.postfix),t=0;t<o.length;t+=1)if(void 0===(r=e[n=o[t]]))"negative"!==n||i.negativeBefore?"mark"===n&&"."!==i.thousand?i[n]=".":i[n]=!1:i[n]="-";else if("decimals"===n){if(!(0<=r&&r<8))throw new Error(n);i[n]=r}else if("encoder"===n||"decoder"===n||"edit"===n||"undo"===n){if("function"!=typeof r)throw new Error(n);i[n]=r}else{if("string"!=typeof r)throw new Error(n);i[n]=r}return f(i,"mark","thousand"),f(i,"prefix","negative"),f(i,"prefix","negativeBefore"),i}(t),this.to=function(e){return i(t,n,e)},this.from=function(e){return i(t,r,e)})}});

www/core.css

@@ -0,0 +1,57 @@
+html,
+body {
+    height: 100%;
+    width: 100%;
+    margin: 0;
+    font-family: sans-serif;
+}
+
+.playlist {
+    position: absolute;
+    left: 0;
+    top: 0;
+    z-index: 50;
+    /* font-size: 15px; */
+}
+
+.playlist::before {
+    content: 'files';
+    padding: 20px;
+    margin: 20px;
+    font-size: 300%;
+    /* display: list-item?; */
+}
+
+.playlist li {
+    /* cursor: pointer; */
+    list-style: none;
+    ;
+    line-height: 1.5;
+    /* margin: 20px; */
+    border-bottom: solid darkgray 1px;
+    padding: 20px 0;
+}
+
+.playlist li a:hover {
+    color: blue;
+}
+
+.playlist .links {
+    display: block;
+}
+
+.playlist li .name {
+    padding-left: 20px
+}
+
+.playlist .links::before {
+    content: '[';
+}
+
+.playlist .links::after {
+    content: ']';
+}
+
+.playlist .links a {
+    margin: 0 10px;
+}

www/draw.js

@@ -1,7 +1,10 @@
 class Canvas {
     constructor(wrapperEl) {
         this.allowDrawing = false;
+        this.viewbox = { "x": 0, "y": 0, "width": null, "height": null };
         this.url = window.location.origin.replace('http', 'ws') + '/ws?' + window.location.search.substring(1);
+
+        // build the interface
         this.wrapperEl = wrapperEl;
         this.wrapperEl.classList.add('closed');
 
@@ -17,7 +20,7 @@ class Canvas {
         this.wrapperEl.appendChild(this.filenameEl);
 
 
-        this.colors = ["red", "blue", "green"];
+        this.colors = ["black", "red", "blue", "green"];
 
         this.resize();
 
@@ -25,15 +28,38 @@ class Canvas {
 
 
         this.paths = [];
+        this.viewboxes = [];
+        this.events = []; // all paths & viewboxes events
         this.isDrawing = false;
         this.hasMouseDown = false;
         this.currentStrokeEl = null;
 
         this.startTime = null;
 
-        document.body.addEventListener('pointermove', this.draw.bind(this));
-        document.body.addEventListener('pointerup', this.penup.bind(this));
-        this.svgEl.addEventListener('pointerdown', this.startStroke.bind(this));
+        this.isMoving = false;
+        document.body.addEventListener('pointermove', (ev) => {
+            if (ev.pointerType == "touch" || ev.buttons & 4) { // 4: middle mouse button
+                this.moveCanvas(ev);
+            } else { // pointerType == pen or mouse
+                this.draw(ev);
+            }
+        });
+        document.body.addEventListener('pointerup', (ev) => {
+            if (ev.pointerType == "touch" || ev.buttons & 4 || this.isMoving) { // buttons is 0 on pointerup
+                this.endMoveCanvas(ev);
+                this.isMoving = false;
+            } else { // pointerType == pen or mouse
+                this.penup(ev);
+            }
+        });
+        this.svgEl.addEventListener('pointerdown', (ev) => {
+            if (ev.pointerType == "touch" || ev.buttons & 4) { // 4: middle mouse button
+                this.isMoving = true;
+                this.startMoveCanvas(ev);
+            } else { // pointerType == pen or mouse
+                this.startStroke(ev);
+            }
+        });
 
         this.createToolbox();
 
@@ -42,12 +68,13 @@ class Canvas {
 
         this.socket = new WebSocket(this.url);
         this.socket.addEventListener('open', (e) => {
-
-            this.socket.send(JSON.stringify({
-                'action': 'dimensions',
-                'width': this.width,
-                'height': this.height
-            }));
+            const d = {
+                'event': 'dimensions',
+                'width': this.viewbox.width,
+                'height': this.viewbox.height
+            };
+            console.log('send', d);
+            this.socket.send(JSON.stringify(d));
         })
         this.socket.addEventListener('message', (e) => {
             let msg = JSON.parse(e.data);
@@ -60,6 +87,40 @@ class Canvas {
         });
     }
 
+    startMoveCanvas(ev) {
+        this.moveCanvasPrevPoint = { "x": ev.x, "y": ev.y };
+        this.currentMoves = [];
+    }
+
+    endMoveCanvas(ev) {
+        this.moveCanvasPrevPoint = null;
+
+        // sync viewpoints
+        const d = {
+            'event': 'viewbox',
+            'viewboxes': this.currentMoves
+        };
+        console.log('send', d);
+        this.socket.send(JSON.stringify(d));
+    }
+
+    moveCanvas(ev) {
+        if (this.moveCanvasPrevPoint === null) {
+            return
+        }
+        const diff = {
+            "x": ev.x - this.moveCanvasPrevPoint.x,
+            "y": ev.y - this.moveCanvasPrevPoint.y,
+        }
+        this.viewbox.x -= diff.x;
+        this.viewbox.y -= diff.y;
+        this.moveCanvasPrevPoint = { "x": ev.x, "y": ev.y };
+        this.currentMoves.push(Object.assign({'t': window.performance.now() - this.startTime}, this.viewbox));
+
+        this.applyViewBox()
+    }
+
+
     openTheFloor() {
         this.wrapperEl.classList.remove('closed');
     }
@@ -100,12 +161,27 @@ class Canvas {
     }
 
     resize() {
-        this.width = window.innerWidth;
-        this.height = window.innerHeight;
-        const viewBox = `0 0 ${this.width} ${this.height}`;
+        this.viewbox.width = window.innerWidth;
+        this.viewbox.height = window.innerHeight;
+
+        this.applyViewBox();
+    }
+
+    applyViewBox() {
+        const viewBox = `${this.viewbox.x} ${this.viewbox.y} ${this.viewbox.width} ${this.viewbox.height}`;
         this.svgEl.setAttribute('viewBox', viewBox);
-        this.svgEl.setAttribute('width', this.width + 'mm');
-        this.svgEl.setAttribute('height', this.height + 'mm');
+        this.svgEl.setAttribute('width', this.viewbox.width + 'mm');
+        this.svgEl.setAttribute('height', this.viewbox.height + 'mm');
+
+        // todo save drag event;
+        // const newViewbox = Object.assign({}, this.viewbox, {'t': window.performance.now() - this.startTime});
+        // const lastViewbox = this.viewboxes[this.viewboxes.length - 1];
+        // if(newViewbox.x == lastViewbox.x && newViewbox.y == lastViewbox.y && newViewbox.width == lastViewbox.width && newViewbox.height == lastViewbox.height){
+        //     // do nothing, avoiding duplicate
+        // } else {
+        //     this.viewboxes.push(newViewbox);
+        //     this.events.push(newViewbox);
+        // }
     }
 
     requestResize() {
@@ -116,8 +192,8 @@ class Canvas {
     getCoordinates(e) {
         // convert event coordinates into relative positions on x & y axis
         let box = this.svgEl.getBoundingClientRect();
-        let x = (e.x - box['left']) / box['width'];
-        let y = (e.y - box['top']) / box['height'];
+        let x = (e.x - box['left'] + this.viewbox.x) / box['width'];
+        let y = (e.y - box['top'] + this.viewbox.y) / box['height'];
         return { 'x': x, 'y': y };
     }
 
@@ -125,27 +201,11 @@ class Canvas {
         return !(pos['x'] < 0 || pos['y'] < 0 || pos['x'] > 1 || pos['y'] > 1);
     }
 
-    // isInsideDrawingBounds (pos) {
-    //     if(pos['x'] > xPadding && pos['x'] < (xPadding+drawWidthFactor) && pos['y'] > yPadding && pos['y'] < yPadding+drawHeightFactor) {
-    //         return true;
-    //     }
-    //     return false;
-    // }
 
     draw(e) {
         let pos = this.getCoordinates(e);
-        if (this.hasMouseDown)
-            console.log(pos, e);
 
-        // if(!isInsideBounds(pos)) {
-        //     // outside of bounds
-        //     return;
-        // }
-
-        // if(!e.buttons || (this.isDrawing && !this.isInsideBounds(pos))){
-        //     this.endStroke(pos);
-        // }
-        if (!this.isDrawing && this.hasMouseDown && this.isInsideBounds(pos)) {
+        if (!this.isDrawing && this.hasMouseDown /*&& this.isInsideBounds(pos)*/) {
             this.isDrawing = true;
         }
 
@@ -154,28 +214,22 @@ class Canvas {
             let d = this.strokes2D(this.paths[this.paths.length - 1].points);
             this.currentStrokeEl.setAttribute('d', d);
         }
-
-        //console.log([pos['x'], pos['y']], isDrawing);
-        // socket.send(JSON.stringify({
-        //     'action': 'move',
-        //     'direction': [pos['x'], pos['y']],
-        //     'mouse': isDrawing,
-        // }));
     }
 
     startStroke(e) {
         this.hasMouseDown = true;
-        console.log('start');
 
         const strokeEl = document.createElementNS('http://www.w3.org/2000/svg', 'path');
         strokeEl.style.stroke = this.currentColor;
         this.svgEl.appendChild(strokeEl);
         this.currentStrokeEl = strokeEl;
-        this.paths.push({
+        let path = {
             'el': strokeEl,
             'color': this.currentColor,
             'points': []
-        });
+        };
+        this.paths.push(path);
+        this.events.push(path); // same ref.
 
         if (this.startTime === null) {
             // initiate timer on first stroke
@@ -184,11 +238,10 @@ class Canvas {
     }
 
     endStroke(pos) {
-        console.log(this.isDrawing)
         if (!this.isDrawing) {
             return;
         }
-        console.log("send!")
+        
         this.isDrawing = false;
         //document.body.removeEventListener('mousemove', draw);
 
@@ -198,11 +251,13 @@ class Canvas {
             this.paths[this.paths.length - 1].points[this.paths[this.paths.length - 1].points.length - 1][2] = 1;
         }
         const stroke = this.paths[this.paths.length - 1];
-        this.socket.send(JSON.stringify({
-            'action': 'stroke',
+        const d = {
+            'event': 'stroke',
             'color': stroke.color,
             'points': stroke.points
-        }));
+        };
+        console.log('send', d);
+        this.socket.send(JSON.stringify(d));
     }
 
     penup(e) {
@@ -222,7 +277,7 @@ class Canvas {
         let cmd = "";
         for (let stroke of strokes) {
             if (!last_stroke) {
-                d += `M${stroke[0] * this.width},${stroke[1] * this.height} `;
+                d += `M${stroke[0] * this.viewbox.width},${stroke[1] * this.viewbox.height} `;
                 cmd = 'M';
             } else {
                 if (last_stroke[2] == 1) {
@@ -233,7 +288,7 @@ class Canvas {
                     cmd = 'l';
                 }
                 let rel_stroke = [stroke[0] - last_stroke[0], stroke[1] - last_stroke[1]];
-                d += `${rel_stroke[0] * this.width},${rel_stroke[1] * this.height} `;
+                d += `${rel_stroke[0] * this.viewbox.width},${rel_stroke[1] * this.viewbox.height} `;
             }
             last_stroke = stroke;
 

www/play.html

@@ -36,6 +36,10 @@
 			stroke-linecap: round;
 		}
 
+		path.before_in {
+			opacity: .2;
+		}
+
 		#wrapper {
 			position: absolute;
 			top: 0;
@@ -58,30 +62,44 @@
 			font-family: sans-serif;
 		}
 
-		.playlist {
+		
+		input[type='range'] {
 			position: absolute;
+			z-index: 100;
+			bottom: 0;
 			left: 0;
-			top: 0;
-			z-index: 50;
+			right: 0;
+			width: 90%;
 		}
 
-		.playlist li{
-			cursor: pointer;
-			line-height: 1.5;
-		}
-		.playlist li:hover{
-			color: blue;
+		.scrubber {
+			position: absolute;
+			z-index: 100;
+			bottom: 0;
+			left: 0;
+			right: 0;
+			width: 90%;
 		}
 	</style>
+	<link rel="stylesheet" href="assets/nouislider-15.5.0.css">
+	<link rel="stylesheet" href="core.css">
 </head>
 
 <body>
 	<div id='interface'>
 	</div>
+	<script src="assets/nouislider-15.5.0.js"></script>
 	<script src="play.js"></script>
+	<script src="playlist.js"></script>
 	<script type='text/javascript'>
-		const player = new Player(document.getElementById("interface"));
-		player.playlist('/files/');
+		if (location.search) {
+			const player = new Player(
+				document.getElementById("interface"),
+				location.search.substring(1)
+			);
+		} else {
+			const playlist = new Playlist(document.getElementById("interface"), '/files/');
+		}
 	</script>
 </body>
 

www/play.js

@@ -1,45 +1,30 @@
 
-
 class Player {
-    constructor(wrapperEl) {
+    constructor(wrapperEl, fileurl) {
         this.wrapperEl = wrapperEl;
         this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
         this.wrapperEl.appendChild(this.svgEl);
 
-    }
+        this.scrubberElOld = document.createElement('input');
+        this.scrubberElOld.type = "range";
+        this.scrubberElOld.min = 0;
+        this.scrubberElOld.step = 0.01;
+        this.wrapperEl.appendChild(this.scrubberElOld);
 
-    playlist(url) {
-        const request = new Request(url, {
-            method: 'GET',
-        });
+        this.scrubberEl = document.createElement('div');
+        this.scrubberEl.classList.add('scrubber')
+        this.wrapperEl.appendChild(this.scrubberEl);
 
+        this.scrubberElOld.addEventListener("input", (ev) => {
+            this.scrubTo(ev.target.value);
+        })
 
-        fetch(request)
-        .then(response => response.json())
-        .then(data => {
-            let playlist = this.wrapperEl.querySelector('.playlist');
-            if(!playlist) {
-                playlist = document.createElement('nav');
-                playlist.classList.add('playlist');
-                this.wrapperEl.appendChild(playlist)
-            }
-            else{
-                playlist.innerHTML = "";
-            }
+        this.inPointPosition = null;
+        this.outPointPosition = null;
+        this.currentTime = 0;
+        this.isPlaying = false;
 
-            const listEl = document.createElement("ul");
-            for(let fileUrl of data) {
-                const liEl = document.createElement("li");
-                liEl.innerText = fileUrl
-                liEl.addEventListener('click', (e) => {
-                    this.play(fileUrl);
-                    playlist.style.display = "none";
-                });
-                listEl.appendChild(liEl);
-            }
-            playlist.appendChild(listEl);
-            // do something with the data sent in the request
-        });
+        this.play(fileurl);
     }
 
     play(file) {
@@ -59,14 +44,122 @@ class Player {
 
     playStrokes(drawing) {
         this.strokes = drawing.shape;
-        this.currentPath = null;
+        this.currentPathI = null;
+        this.currentPointI = null;
         this.dimensions = drawing.dimensions;
         this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
         this.startTime = window.performance.now() - this.strokes[0].points[0][3];
-        this.playStroke(0,1);
+        this.playStrokePosition(0, 1);
+        this.duration = this.getDuration();
+        this.scrubberElOld.max = this.duration;
+        this.playTimout = null;
+
+        const slider = noUiSlider.create(this.scrubberEl, {
+            start: [this.currentTime, this.duration],
+            connect: true,
+            range: {
+                'min': 0,
+                'max': this.duration
+            }
+        });
+
+        slider.on("slide", (values, handle) => {
+            this.isPlaying = false;
+            // console.log(values, handle);
+            // both in and out need to have a value
+            this.inPointPosition = this.findPositionForTime(values[0]);
+            this.outPointPosition = this.findPositionForTime(values[1]);
+            if (handle === 0) {
+                // in point
+                if (
+                    this.currentPathI < this.inPointPosition[0] ||
+                    this.currentPointI < this.inPointPosition[1]) {
+                        this.drawStrokePosition(
+                            // this.inPointPosition[0],
+                            // this.inPointPosition[1],
+                            // always draw at out position, as to see the whole shape of the range
+                            this.outPointPosition[0],
+                            this.outPointPosition[1],
+                        );
+                }
+            }
+            if (handle === 1) {
+                // out point
+                // this.outPointPosition = this.findPositionForTime(values[1]);
+                this.drawStrokePosition(
+                    this.outPointPosition[0],
+                    this.outPointPosition[1],
+                );
+            }
+            // this.inPointPosition = values;
+            // this.outPointPosition = vaalues[0];
+            // this.scrubTo()
+            // this.scrubTo(ev.target.value);
+        });
+
     }
 
-    playStroke(path_i, point_i){
+    getDuration() {
+        const points = this.strokes[this.strokes.length - 1].points;
+        return points[points.length - 1][3];
+    }
+
+    getStrokesForPathRange(in_point, out_point) {
+        // get paths for given range. Also, split path at in & out if necessary.
+        strokes = {};
+        for (let i = in_point[0]; i <= out_point[0]; i++) {
+            const path = this.strokes[i];
+            const in_i = (in_point[0] === i) ? in_point[1] : 0;
+            const out_i = (out_point[0] === i) ? out_point[1] : Math.inf;
+            const points = path.points.slice(in_i, out_i);
+            strokes[i] = points; // preserve indexes
+        }
+        return strokes;
+    }
+
+    // TODO: when drawing, have a group active & inactive.
+    // active is getPathRange(currentIn, currentOut)
+    // inactive is what comes before and after.
+    // then, playing the video is just running pathRanghe(0, playhead)
+    drawStrokePosition(path_i, point_i, show_all) {
+        if(typeof show_all === 'undefined')
+            show_all = false;
+
+        // check if anything is placed that is in the future from the current playhead
+        if (this.currentPathI !== null && this.currentPoint !== null) {
+            if (this.currentPathI > path_i) {
+                console.log('remove', path_i, ' -> ', this.currentPathI)
+                // remove everything that comes later
+                for (let i = path_i + 1; i <= this.currentPathI; i++) {
+                    // console.log('remove', i);
+                    const pathEl = this.svgEl.querySelector(`.path${i}`);
+                    if (pathEl) {
+                        pathEl.parentNode.removeChild(pathEl);
+                    }
+                }
+            }
+        }
+        // an inpoint is set, so we're annotating
+        // make everything coming before translucent
+        if (this.inPointPosition !== null) {
+            const [inPath_i, inPoint_i] = this.inPointPosition;
+            // returns a static NodeList
+            const currentBeforeEls = this.svgEl.querySelectorAll(`.before_in`);
+            for (let currentBeforeEl of currentBeforeEls) {
+                currentBeforeEl.classList.remove('before_in');
+            }
+
+            for (let index = 0; index < inPath_i; index++) {
+                const pathEl = this.svgEl.querySelector(`.path${index}`);
+                if (pathEl) {
+                    pathEl.classList.add('before_in');
+                }
+            }
+        }
+
+        this.currentPathI = path_i;
+        this.currentPointI = point_i;
+
         const path = this.strokes[path_i];
         // console.log(path);
         let pathEl = this.svgEl.querySelector(`.path${path_i}`);
@@ -77,33 +170,109 @@ class Player {
             this.svgEl.appendChild(pathEl)
         }
 
-        const d = this.strokes2D(path.points.slice(0, point_i));
+        const stroke = path.points.slice(0, point_i);
+        const d = this.strokes2D(stroke);
         pathEl.setAttribute('d', d);
 
-        let next_path, next_point,t;
+        this.scrubberElOld.value = path.points[point_i][3];
+        this.currentTime = path.points[point_i][3];
+    }
+
+    getNextPosition(path_i, point_i) {
+        const path = this.strokes[path_i];
+        let next_path, next_point;
         if (path.points.length > point_i + 1) {
             next_path = path_i;
             next_point = point_i + 1;
-            t = path.points[next_point][3];// - path.points[point_i][3];
             // setTimeout(() => this.playStroke(next_path, next_point), dt);
         } else if (this.strokes.length > path_i + 1) {
             next_path = path_i + 1;
             next_point = 1;
-            t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3];
             // use starttime instead of diff, to prevent floating
         } else {
-            console.log('done');
-            return
+            return [null, null];
         }
 
+        // when an outpoint is set, stop playing there
+        if(next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1]){
+            return [null, null];
+        }
+
+        return [next_path, next_point];
+    }
+
+    playStrokePosition(path_i, point_i, allow_interrupt) {
+        if(allow_interrupt) {
+            if(!this.isPlaying) {
+                console.log('not playing because of interrupt');
+                return;
+            }
+        } else{
+            this.isPlaying = true;
+        }
+        this.drawStrokePosition(path_i, point_i);
+
+        const [next_path, next_point] = this.getNextPosition(path_i, point_i);
+        if (next_path === null) {
+            console.log('done playing');
+            return;
+        }
+
+        const t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3];
+
         const dt = t - (window.performance.now() - this.startTime);
-        setTimeout(() => this.playStroke(next_path, next_point), dt);
+        this.playTimout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt);
     }
 
     playUntil(path_i) {
         // for scrubber
     }
 
+    scrubTo(ms) {
+        const [path_i, point_i] = this.findPositionForTime(ms);
+        // console.log(path_i, point_i);
+        clearTimeout(this.playTimout);
+        this.playStrokePosition(path_i, point_i);
+        // this.playHead = ms;
+    }
+
+
+
+    findPositionForTime(ms) {
+        ms = Math.min(Math.max(ms, 0), this.duration);
+        console.log('scrub to', ms)
+        let path_i = 0;
+        let point_i = 0;
+        this.strokes.every((stroke, index) => {
+            const startAt = stroke.points[0][3];
+            const endAt = stroke.points[stroke.points.length - 1][3];
+
+            if (startAt > ms) {
+                return false; // too far
+            }
+            if (endAt > ms) {
+                // we're getting close. Find the right point_i
+                path_i = index;
+                stroke.points.every((point, pi) => {
+                    if (point[3] > ms) {
+                        // too far
+                        return false;
+                    }
+                    point_i = pi;
+                    return true;
+                });
+                return false;
+            } else {
+                // in case nothings comes after, we store the last best option thus far
+                path_i = index;
+                point_i = stroke.points.length - 1;
+                return true;
+            }
+
+        });
+        return [path_i, point_i];
+    }
+
 
     strokes2D(strokes) {
         // strokes to a d attribute for a path

www/playlist.js

@@ -0,0 +1,67 @@
+class Playlist {
+    constructor(wrapperEl, url) {
+        this.wrapperEl = wrapperEl;
+
+        const request = new Request(url, {
+            method: 'GET',
+        });
+
+        fetch(request)
+            .then(response => response.json())
+            .then(data => {
+                let playlist = this.wrapperEl.querySelector('.playlist');
+                if (!playlist) {
+                    playlist = document.createElement('nav');
+                    playlist.classList.add('playlist');
+                    this.wrapperEl.appendChild(playlist)
+                }
+                else {
+                    playlist.innerHTML = "";
+                }
+
+                const listEl = document.createElement("ul");
+                for (let file of data) {
+                    const liEl = document.createElement("li");
+
+                    const dateEl = document.createElement("span");
+                    dateEl.classList.add('date');
+                    dateEl.innerText = file.time;
+                    liEl.append(dateEl);
+
+                    const nameEl = document.createElement("span");
+                    nameEl.classList.add('name');
+                    nameEl.innerText = file.name;
+                    liEl.append(nameEl);
+
+                    const linksEl = document.createElement("span");
+                    linksEl.classList.add('links');
+                    liEl.append(linksEl);
+
+                    const playEl = document.createElement("a");
+                    playEl.classList.add('play');
+                    playEl.innerText = "Play";
+                    playEl.href = location;
+                    playEl.pathname = "play.html";
+                    playEl.search = "?"+file.name;
+                    linksEl.append(playEl);
+                    
+                    const annotateEl = document.createElement("a");
+                    annotateEl.classList.add('annotate');
+                    annotateEl.innerText = "Annotate";
+                    annotateEl.href = location;
+                    annotateEl.pathname = "annotate.html";
+                    annotateEl.search = "?"+file.name;
+                    linksEl.append(annotateEl);
+                    
+                    // liEl.addEventListener('click', (e) => {
+                    //     this.play(fileUrl);
+                    //     playlist.style.display = "none";
+                    // });
+                    listEl.appendChild(liEl);
+                }
+                playlist.appendChild(listEl);
+                // do something with the data sent in the request
+            });
+
+    }
+}