security_vision/chronodiagram
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

New stroke playing logic for annotating and prep for adding/syncing audio file

Browse source
This commit is contained in:
Ruben van de Ven 2021-12-20 13:36:18 +01:00
parent 864cf95b8b
commit 9be9f11ba2
10 changed files with 3520 additions and 255 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
files/audio/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore

72
webserver.py
View file

@ -119,6 +119,22 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
return client in cls.connections
class AudioListingHandler(tornado.web.RequestHandler):
def initialize(self, config):
self.config = config
self.audiodir = os.path.join(self.config.storage, 'audio')
def get(self):
# filename = self.get_argument("file", None)
self.set_header("Content-Type", "application/json")
if not os.path.exists(self.audiodir):
names = []
else:
names = sorted([f"/audio/{name}" for name in os.listdir(self.audiodir) if name not in ['.gitignore']])
print(names)
self.write(json.dumps(names))
class AnimationHandler(tornado.web.RequestHandler):
def initialize(self, config):
self.config = config
@ -132,6 +148,7 @@ class AnimationHandler(tornado.web.RequestHandler):
else:
path = os.path.join(self.config.storage,os.path.basename(filename)+".csv")
drawing = {
"file": filename,
"shape": []
}
with open(path, 'r') as fp:
@ -172,8 +189,53 @@ def strokes2D(strokes):
rel_stroke = [stroke[0] - last_stroke[0], stroke[1] - last_stroke[1]];
d += f"{rel_stroke[0]},{rel_stroke[1]} "
last_stroke = stroke;
return d;
last_stroke = stroke
return d
class AnnotationsHandler(tornado.web.RequestHandler):
def initialize(self, config):
self.config = config
self.metadir = os.path.join(self.config.storage, 'metadata')
def prepare(self):
if self.request.headers.get("Content-Type", "").startswith("application/json"):
self.json_args = json.loads(self.request.body)
else:
self.json_args = None
def get(self, filename):
self.set_header("Content-Type", "application/json")
filenames = sorted([name[:-4] for name in os.listdir(self.config.storage) if name not in ['.gitignore']])
if filename not in filenames:
raise Exception('Invalid filename')
meta_file = os.path.join(self.metadir, filename +'.json')
if not os.path.exists(meta_file):
self.set_status(404)
return
with open(meta_file, 'r') as fp:
self.write(json.load(fp))
def post(self, filename):
# filename = self.get_argument("file", None)
filenames = sorted([name[:-4] for name in os.listdir(self.config.storage) if name not in ['.gitignore']])
if filename not in filenames:
raise Exception('Invalid filename')
if not os.path.exists(self.metadir):
os.mkdir(self.metadir)
meta_file = os.path.join(self.metadir, filename +'.json')
with open(meta_file, 'w') as fp:
json.dump(self.json_args, fp)
class Server:
"""
@ -198,6 +260,12 @@ class Server:
(r"/files/(.*)", AnimationHandler,
{'config': self.config}),
(r"/audio/(.+)", tornado.web.StaticFileHandler,
{"path": os.path.join(self.config.storage, 'audio')}),
(r"/audio", AudioListingHandler,
{'config': self.config}),
(r"/annotations/(.+)", AnnotationsHandler,
{'config': self.config}),
(r"/(.*)", StaticFileWithHeaderHandler,
{"path": self.web_root}),
], debug=True, autoreload=True)

333
www/annotate.html
View file

@ -2,118 +2,249 @@
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8">
<title>Annotate a line animation</title>
<style media="screen">
#sample,
svg {
position: absolute;
top: 0;
left: 0;
bottom: 0;
right: 0;
width: 100%;
height: 100%;
font-family: sans-serif;
z-index: 2;
}
<meta charset="utf-8">
<title>Annotate a line animation</title>
<style media="screen">
body {
/* background: black;
color: white */
background: lightgray;
}
img {
position: absolute;
top: 0;
bottom: 0;
right: 0;
left: 0;
width: 100%;
height: 100%;
z-index: 1;
}
#sample,
svg {
position: absolute;
top: 20px;
left: 20px;
width: calc(100% - 40px);
height: calc(100% - 200px);
font-family: sans-serif;
z-index: 2;
/* background: white; */
/* border: solid 2px lightgray; */
}
path {
fill: none;
stroke: gray;
stroke-width: 1mm;
stroke-linecap: round;
}
svg .background {
fill: white
}
g.before path{
img {
position: absolute;
top: 0;
bottom: 0;
right: 0;
left: 0;
width: 100%;
height: 100%;
z-index: 1;
}
path {
fill: none;
stroke: gray;
stroke-width: 1mm;
stroke-linecap: round;
}
g.before path {
opacity: 0.5;
stroke: gray !important;
}
g.after path, path.before_in {
opacity: .1;
g.after path,
path.before_in {
opacity: .1;
stroke: gray !important;
}
#wrapper {
position: absolute;
top: 0;
right: 0;
bottom: 0;
left: 0;
background: none;
}
.gray {
position: absolute;
background: rgba(255, 255, 255, 0.7);
}
html,
body {
height: 100%;
width: 100%;
margin: 0;
font-family: sans-serif;
}
.playlist {
position: absolute;
left: 0;
top: 0;
z-index: 50;
}
.playlist li{
cursor: pointer;
line-height: 1.5;
}
.playlist li:hover{
color: blue;
}
input[type='range']{
position: absolute;
z-index: 100;
bottom: 0;
left:0;
right: 0;width: 90%;
}
.scrubber{
position: absolute !important;
z-index: 100;
bottom: 30px;
left:0;
right: 0;width: 90%;
}
.noUi-horizontal .noUi-touch-area{
cursor:ew-resize;
}
</style>
<link rel="stylesheet" href="assets/nouislider-15.5.0.css">
#wrapper {
position: absolute;
top: 0;
right: 0;
bottom: 0;
left: 0;
background: none;
}
.gray {
position: absolute;
background: rgba(255, 255, 255, 0.7);
}
html,
body {
height: 100%;
width: 100%;
margin: 0;
font-family: sans-serif;
}
.playlist {
position: absolute;
left: 0;
top: 0;
z-index: 50;
}
.playlist li {
cursor: pointer;
line-height: 1.5;
}
.playlist li:hover {
color: blue;
}
input[type='range'] {
/* position: absolute;
z-index: 100;
bottom: 0;
left: 0;
right: 0; */
width: 100%;
}
.controls {
position: absolute !important;
z-index: 100;
bottom: 10px;
left: 5%;
right: 0;
width: 90%;
}
.scrubber {}
.tags {
line-height: 40px;
display: flex;
flex-direction: row;
padding: 0;
margin: 0;
}
.tags li {
display: block;
padding: 5px;
border: solid 1px darkgray;
flex-grow: 1;
text-align: center;
}
.tags li:hover {
cursor: pointer;
background: darkgray;
}
.tags li.selected {
background: lightsteelblue;
}
.tags li.annotation-rm {
/* display: none; */
overflow: hidden;
color: red;
font-size: 30px;
width: 0;
flex-grow: 0;
padding: 5px 0;
transition: width .3s;
pointer-events: none;
border: none;
direction: rtl; /* hide behind bar, instead into nothing */
}
.selected-annotation .tags li.annotation-rm {
color: red;
display: block;
width: 30px;
pointer-events: all;
}
.tags li span {
display: inline-block;
width: 20px;
height: 20px;
margin-right: 10px;
vertical-align: middle;
border-radius: 5px;
}
.annotations {
height: 10px;
/* border: solid 1px darkgray; */
position: relative;
}
.annotations>div {
opacity: .4;
background: lightseagreen;
position: absolute;
bottom: 0;
top: 0;
}
.annotations>div:hover,
.annotations>div.selected {
opacity: 1;
cursor: pointer;
}
.annotation-test {
background-color: red !important;
}
.annotation-another {
background-color: blue !important;
}
.annotation-google {
background-color: blueviolet !important;
}
.unsaved::before {
content: '*';
color: red;
display: inline-block;
text-align: center;
font-size: 30px;
position: absolute;
top: 10px;
left: 10px;
}
.saved::before {
content: '\2713';
display: inline-block;
color: green;
text-align: center;
font-size: 30px;
position: absolute;
top: 10px;
left: 10px;
}
.noUi-horizontal .noUi-touch-area {
cursor: ew-resize;
}
</style>
<link rel="stylesheet" href="assets/nouislider-15.5.0.css">
</head>
<body>
<div id='interface'>
</div>
<script src="assets/nouislider-15.5.0.js"></script>
<script src="assets/wNumb-1.2.0.min.js"></script>
<script src="annotate.js"></script>
<script type='text/javascript'>
const player = new Player(document.getElementById("interface"));
player.playlist('/files/');
</script>
<div id='interface'>
</div>
<script src="assets/nouislider-15.5.0.js"></script>
<script src="assets/wNumb-1.2.0.min.js"></script>
<script src="annotate.js"></script>
<script type='text/javascript'>
const player = new Annotator(
document.getElementById("interface"),
["test", "another", "google"]
);
player.playlist('/files/');
</script>
</body>
</html>

463
www/annotate.js
View file

@ -1,39 +1,37 @@
class Annotation{
constructor(annotation, t_in, t_out) {
this.annotation = annotation;
class Annotation {
constructor(tag, t_in, t_out) {
this.tag = tag;
this.t_in = t_in;
this.t_out = t_out;
}
}
class StrokeGroup{
constructor(group_element, player){
class StrokeGroup {
constructor(group_element, player) {
this.g = group_element;
this.player = player;
}
setStrokes(strokes){
console.log('set strokes',strokes);
setStrokes(strokes) {
const pathEls = this.g.querySelectorAll('path');
let indexes = Object.keys(strokes);
for (let pathEl of pathEls) {
const i = pathEl.dataset.path_i;
if(!indexes.includes(pathEl.dataset.path_i)){
if (!indexes.includes(pathEl.dataset.path_i)) {
pathEl.parentNode.removeChild(pathEl);
}else{
} else {
// check in and outpoint using pathEl.dataset
if(strokes[i].getSliceId() != pathEl.dataset.slice){
if (strokes[i].getSliceId() != pathEl.dataset.slice) {
const d = this.points2D(strokes[i].points);
pathEl.dataset.slice = strokes[i].getSliceId();
pathEl.setAttribute('d', d);
}
}
// this has now been processed
indexes.splice(indexes.indexOf(i), 1);
}
console.log(indexes);
// new strokes
indexes.forEach(index => {
const stroke = strokes[index];
@ -78,8 +76,8 @@ class StrokeGroup{
}
}
class Stroke{
constructor(color, points){
class Stroke {
constructor(color, points) {
this.color = color;
this.points = points; // [[x1,y1,t1], [x2,y2,t2], ...]
}
@ -89,48 +87,89 @@ class Stroke{
}
}
class StrokeSlice{
constructor(stroke, i_in, i_out){
class StrokeSlice {
constructor(stroke, i_in, i_out) {
this.stroke = stroke; // Stroke
this.i_in = typeof i_in === 'undefined' ? 0 : i_in;
this.i_out = typeof i_out === 'undefined' ? this.stroke.points.length : i_out;
}
getSliceId(){
getSliceId() {
return `${this.i_in}-${this.i_out}`;
}
// compatible with Stroke()
get points(){
get points() {
return this.stroke.points.slice(this.i_in, this.i_out);
}
// compatible with Stroke()
get color(){
get color() {
return this.stroke.color;
}
}
class Player {
constructor(wrapperEl) {
class Annotator {
constructor(wrapperEl, tags) {
this.wrapperEl = wrapperEl;
this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
this.wrapperEl.appendChild(this.svgEl);
this.controlsEl = document.createElement('div');
this.controlsEl.classList.add('controls')
this.wrapperEl.appendChild(this.controlsEl);
this.scrubberElOld = document.createElement('input');
this.scrubberElOld.type = "range";
this.scrubberElOld.min = 0;
this.scrubberElOld.step = 0.01;
this.wrapperEl.appendChild(this.scrubberElOld);
this.scrubberEl = document.createElement('div');
this.scrubberEl.classList.add('scrubber')
this.wrapperEl.appendChild(this.scrubberEl);
this.controlsEl.appendChild(this.scrubberElOld);
this.scrubberElOld.addEventListener("input", (ev) => {
this.scrubTo(ev.target.value);
})
this.scrubberEl = document.createElement('div');
this.scrubberEl.classList.add('scrubber')
this.controlsEl.appendChild(this.scrubberEl);
this.tagsEl = document.createElement('ul');
this.tagsEl.classList.add('tags');
for (let tag of tags) {
let tagEl = document.createElement('li');
tagEl.classList.add('tag');
tagEl.dataset.tag = tag;
tagEl.innerText = tag;
tagEl.addEventListener('click', (e) => {
this.addTag(tag, this.inPointPosition, this.outPointPosition);
})
let signEl = document.createElement('span');
signEl.classList.add('annotation-' + tag);
tagEl.prepend(signEl);
this.tagsEl.appendChild(tagEl);
}
let tagEl = document.createElement('li');
tagEl.classList.add('tag');
tagEl.classList.add('annotation-rm');
tagEl.dataset.tag = 'rm';
tagEl.title = "Remove annotation";
tagEl.innerHTML = "&times;";
tagEl.addEventListener('click', (e) => {
if (this.selectedAnnotation) {
this.removeAnnotation(this.selectedAnnotationI);
}
});
this.tagsEl.appendChild(tagEl);
this.controlsEl.appendChild(this.tagsEl);
this.annotationsEl = document.createElement('div');
this.annotationsEl.classList.add('annotations')
this.controlsEl.appendChild(this.annotationsEl);
this.inPointPosition = null;
this.outPointPosition = null;
this.currentTime = 0;
@ -148,6 +187,83 @@ class Player {
this.annotations = []
}
updateAnnotations(save) {
this.annotationsEl.innerHTML = "";
for (let annotation_i in this.annotations) {
const annotation = this.annotations[annotation_i];
this.annotationEl = document.createElement('div');
const left = (annotation.t_in / this.duration) * 100;
const right = 100 - (annotation.t_out / this.duration) * 100;
this.annotationEl.style.left = left + '%';
this.annotationEl.style.right = right + '%';
this.annotationEl.classList.add('annotation-' + annotation.tag);
if (this.selectedAnnotationI == annotation_i) {
this.annotationEl.classList.add('selected');
}
this.annotationEl.title = annotation.tag;
this.annotationEl.addEventListener('mouseover', (e) => {
});
this.annotationEl.addEventListener('mouseout', (e) => {
});
this.annotationEl.addEventListener('click', (e) => {
if (this.selectedAnnotationI == annotation_i) {
this.deselectAnnotation(false);
} else {
this.selectAnnotation(annotation_i);
}
});
this.annotationsEl.appendChild(this.annotationEl);
}
this.tagsEl.childNodes.forEach(tagEl => {
if (this.selectedAnnotation && this.selectedAnnotation.tag == tagEl.dataset.tag) {
tagEl.classList.add('selected')
} else {
tagEl.classList.remove('selected')
}
});
if (save) {
this.updateState();
}
}
selectAnnotation(annotation_i) {
this.selectedAnnotationI = annotation_i;
this.selectedAnnotation = this.annotations[annotation_i];
this.slider.set([this.selectedAnnotation.t_in, this.selectedAnnotation.t_out]);
this.inPointPosition = this.findPositionForTime(this.selectedAnnotation.t_in);
this.outPointPosition = this.findPositionForTime(this.selectedAnnotation.t_out);
this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
this.updateAnnotations(false); //selects the right tag & highlights the annotation
this.wrapperEl.classList.add('selected-annotation');
}
deselectAnnotation(keep_position) {
if (this.selectedAnnotation)
this.currentTime = this.selectedAnnotation.t_out;
this.wrapperEl.classList.remove('selected-annotation');
this.selectedAnnotationI = null;
this.selectedAnnotation = null;
if (!keep_position) {
this.setUpAnnotator();
}
this.updateAnnotations(false); // selects the right tag & highlights the annotation
}
// TODO: to separate class which then instantiates a player for the given file
playlist(url) {
const request = new Request(url, {
method: 'GET',
@ -190,35 +306,109 @@ class Player {
fetch(request)
.then(response => response.json())
.then(data => {
this.playStrokes(data)
const metadata_req = new Request(`/annotations/${data.file}`, {
method: 'GET',
});
fetch(metadata_req)
.then(response => response.ok ? response.json() : null)
.then(metadata => {
this.playStrokes(data, metadata)
})
.catch(e => console.log(e));
// do something with the data sent in the request
});
}
updateState() {
const state = {
'file': this.filename,
'annotations': this.annotations,
'audio': {
'file': this.audioFile,
'offset': this.audioOffset,
}
}
const newState = JSON.stringify(state);
if (newState == this.state) {
return;
}
this.wrapperEl.classList.remove('saved');
this.wrapperEl.classList.add('unsaved');
this.state = newState;
// autosave on state change:
this.save(newState);
}
setSaved(state) {
if (this.state != state) {
console.log('already outdated');
}
else {
this.wrapperEl.classList.add('saved');
this.wrapperEl.classList.remove('unsaved');
}
}
save(state) {
const request = new Request("/annotations/" + this.filename, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: state
});
fetch(request)
.then((response) => {
if (response.ok) {
this.setSaved(state);
}
else {
throw Error('Something went wrong');
}
})
.catch((error) => {
console.log(error);
});
}
playStrokes(drawing) {
this.strokes = drawing.shape.map(s => new Stroke(s['color'], s['points']));
this.currentPathI = null;
this.currentPointI = null;
this.dimensions = drawing.dimensions;
this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
this.startTime = window.performance.now() - this.strokes[0].points[0][3];
this.playStrokePosition(0, 1);
this.duration = this.getDuration();
this.scrubberElOld.max = this.duration;
this.playTimout = null;
removeAnnotation(annotation_i) {
this.deselectAnnotation(true);
this.annotations.splice(annotation_i, 1);
this.updateAnnotations(true);
}
const formatter = wNumb({
decimals: 2,
edit: (time) => {
const s = Math.floor(time/1000);
const minutes = Math.floor(s / 60);
const seconds = s - minutes * 60;
const ms = Math.floor((time/1000 - s) * 1000);
return `${minutes}:${seconds}:${ms}`;
addTag(tag) {
if (this.selectedAnnotation) {
this.selectedAnnotation.tag = tag;
this.updateAnnotations(true);
} else {
// TODO this.slider values for in and out
const [t_in, t_out] = this.slider.get();
if (this.slider) {
this.slider.destroy();
}
});
const slider = noUiSlider.create(this.scrubberEl, {
this.annotations.push(new Annotation(tag, t_in, t_out));
this.updateAnnotations(true);
this.currentTime = t_out;
this.setUpAnnotator();
}
}
setUpAnnotator() {
this.inPointPosition = this.findPositionForTime(this.currentTime);
this.outPointPosition = this.findPositionForTime(this.duration);
if (this.scrubberEl.noUiSlider) {
this.slider.destroy();
}
this.slider = noUiSlider.create(this.scrubberEl, {
start: [this.currentTime, this.duration],
connect: true,
range: {
@ -226,51 +416,144 @@ class Player {
'max': this.duration
},
tooltips: [
formatter,
formatter
this.formatter,
this.formatter
],
// pips: {
// mode: 'range',
// density: 3,
// format: formatter
// format: this.formatter
// }
});
slider.on("slide", (values, handle) => {
this.slider.on("slide", (values, handle) => {
this.isPlaying = false;
// console.log(values, handle);
// both in and out need to have a value
this.inPointPosition = this.findPositionForTime(values[0]);
this.outPointPosition = this.findPositionForTime(values[1]);
// if (handle === 0) {
// // in point
// if (
// this.currentPathI < this.inPointPosition[0] ||
// this.currentPointI < this.inPointPosition[1]) {
// this.drawStrokePosition(
// // this.inPointPosition[0],
// // this.inPointPosition[1],
// // always draw at out position, as to see the whole shape of the range
// this.outPointPosition[0],
// this.outPointPosition[1],
// );
// }
// }
// if (handle === 1) {
// // out point
// // this.outPointPosition = this.findPositionForTime(values[1]);
// this.drawStrokePosition(
// this.outPointPosition[0],
// this.outPointPosition[1],
// );
// }
this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
// this.inPointPosition = values;
// this.outPointPosition = vaalues[0];
// this.scrubTo()
// this.scrubTo(ev.target.value);
// console.log(this.selectedAnnotation);
if (this.selectedAnnotation) {
this.selectedAnnotation.t_in = values[0];
this.selectedAnnotation.t_out = values[1];
this.updateAnnotations(false);
}
});
this.slider.on("end", (values, handle) => {
if (this.selectedAnnotation) {
this.updateAnnotations(true);
}
})
this.drawStrokePosition(this.inPointPosition, this.outPointPosition);
}
playStrokes(drawing, metadata) {
if (metadata) {
this.annotations = metadata.annotations;
this.audioFile = metadata.hasOwnProperty('audio') ? metadata.audio.file : null;
this.audioOffset = metadata.hasOwnProperty('audio') ? metadata.audio.offset : 0;
//
// load any saved metadata
}
this.filename = drawing.file;
this.strokes = drawing.shape.map(s => new Stroke(s['color'], s['points']));
this.currentPathI = null;
this.currentPointI = null;
this.dimensions = drawing.dimensions;
this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
let bgEl = document.createElementNS('http://www.w3.org/2000/svg', 'rect');
bgEl.setAttribute("x", 0);
bgEl.setAttribute("y", 0);
bgEl.setAttribute("width", this.dimensions[0]);
bgEl.setAttribute("height", this.dimensions[1]);
bgEl.classList.add('background');
this.svgEl.prepend(bgEl);
this.startTime = window.performance.now() - this.strokes[0].points[0][3];
this.duration = this.getDuration();
this.scrubberElOld.max = this.duration;
this.playTimout = null;
this.formatter = wNumb({
decimals: 2,
edit: (time) => {
const s = Math.floor(time / 1000);
const minutes = Math.floor(s / 60);
const seconds = s - minutes * 60;
const ms = Math.floor((time / 1000 - s) * 1000);
return `${minutes}:${seconds}:${ms}`;
}
});
this.setUpAnnotator()
this.updateAnnotations(false);
this.setupAudioConfig();
// this.playStrokePosition(0, 1);
}
setupAudioConfig(){
// audio config
let audioConfigEl = document.createElement('div');
audioConfigEl.classList.add('audioconfig')
this.wrapperEl.appendChild(audioConfigEl);
let audioSelectEl = document.createElement('select');
audioSelectEl.classList.add('audioselect');
audioConfigEl.appendChild(audioSelectEl);
fetch('/audio')
.then(response => response.json())
.then(data => {
data.unshift(''); // add empty, to deselect any file
data.forEach(audioFile => {
let optionEl = document.createElement('option');
optionEl.selected = this.audioFile == audioFile;
optionEl.innerText = audioFile;
audioSelectEl.appendChild(optionEl);
});
})
audioSelectEl.addEventListener('change', (ev) => {
this.setAudioFile(ev.target.value);
});
let audioOffsetEl = document.createElement('input');
audioOffsetEl.setAttribute('type', 'number');
audioOffsetEl.value = this.audioOffset;
audioOffsetEl.addEventListener('change', (ev) => {
this.setAudioOffset(ev.target.value);
});
audioConfigEl.appendChild(audioOffsetEl);
this.audioEl = document.createElement('audio');
if(this.audioFile) {
this.audioEl.setAttribute('src', this.audioFile);
}
this.audioEl.addEventListener('canplaythrough', (ev) => {
console.log('loaded audio', ev);
});
audioConfigEl.appendChild(this.audioEl);
}
setAudioFile(audioFile) {
this.audioFile = audioFile;
this.audioEl.setAttribute('src', this.audioFile);
// TODO update playhead
// TODO update this.duration after load
this.updateState();
}
setAudioOffset(audioOffset) {
this.audioOffset = audioOffset;
// TODO update playhead
// TODO update this.duration
this.updateState();
}
getDuration() {
@ -283,7 +566,7 @@ class Player {
let slices = {};
for (let i = in_point[0]; i <= out_point[0]; i++) {
const stroke = this.strokes[i];
if(typeof stroke === 'undefined'){
if (typeof stroke === 'undefined') {
// out point can be Infinity. So interrupt whenever the end is reached
break;
}
@ -300,14 +583,14 @@ class Player {
// inactive is what comes before and after.
// then, playing the video is just running pathRanghe(0, playhead)
drawStrokePosition(in_point, out_point, show_all) {
if(typeof show_all === 'undefined')
if (typeof show_all === 'undefined')
show_all = true;
this.strokeGroups['before'].setStrokes(this.getStrokesSliceForPathRange([0,0], in_point));
this.strokeGroups['before'].setStrokes(this.getStrokesSliceForPathRange([0, 0], in_point));
this.strokeGroups['annotation'].setStrokes(this.getStrokesSliceForPathRange(in_point, out_point));
this.strokeGroups['after'].setStrokes(this.getStrokesSliceForPathRange(out_point, [Infinity, Infinity]));
// // an inpoint is set, so we're annotating
// // make everything coming before translucent
// if (this.inPointPosition !== null) {
@ -363,7 +646,7 @@ class Player {
}
// when an outpoint is set, stop playing there
if(this.outPointPosition && (next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1])){
if (this.outPointPosition && (next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1])) {
return [null, null];
}
@ -371,12 +654,12 @@ class Player {
}
playStrokePosition(path_i, point_i, allow_interrupt) {
if(allow_interrupt) {
if(!this.isPlaying) {
if (allow_interrupt) {
if (!this.isPlaying) {
console.log('not playing because of interrupt');
return;
}
} else{
} else {
this.isPlaying = true;
}
this.drawStrokePosition(path_i, point_i);
@ -409,7 +692,7 @@ class Player {
findPositionForTime(ms) {
ms = Math.min(Math.max(ms, 0), this.duration);
console.log('scrub to', ms)
// console.log('scrub to', ms)
let path_i = 0;
let point_i = 0;
this.strokes.every((stroke, index) => {

304
www/assets/nouislider-15.5.0.css Normal file
View file

@ -0,0 +1,304 @@
/* Functional styling;
* These styles are required for noUiSlider to function.
* You don't need to change these rules to apply your design.
*/
.noUi-target,
.noUi-target * {
-webkit-touch-callout: none;
-webkit-tap-highlight-color: rgba(0, 0, 0, 0);
-webkit-user-select: none;
-ms-touch-action: none;
touch-action: none;
-ms-user-select: none;
-moz-user-select: none;
user-select: none;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
.noUi-target {
position: relative;
}
.noUi-base,
.noUi-connects {
width: 100%;
height: 100%;
position: relative;
z-index: 1;
}
/* Wrapper for all connect elements.
*/
.noUi-connects {
overflow: hidden;
z-index: 0;
}
.noUi-connect,
.noUi-origin {
will-change: transform;
position: absolute;
z-index: 1;
top: 0;
right: 0;
height: 100%;
width: 100%;
-ms-transform-origin: 0 0;
-webkit-transform-origin: 0 0;
-webkit-transform-style: preserve-3d;
transform-origin: 0 0;
transform-style: flat;
}
/* Offset direction
*/
.noUi-txt-dir-rtl.noUi-horizontal .noUi-origin {
left: 0;
right: auto;
}
/* Give origins 0 height/width so they don't interfere with clicking the
* connect elements.
*/
.noUi-vertical .noUi-origin {
top: -100%;
width: 0;
}
.noUi-horizontal .noUi-origin {
height: 0;
}
.noUi-handle {
-webkit-backface-visibility: hidden;
backface-visibility: hidden;
position: absolute;
}
.noUi-touch-area {
height: 100%;
width: 100%;
}
.noUi-state-tap .noUi-connect,
.noUi-state-tap .noUi-origin {
-webkit-transition: transform 0.3s;
transition: transform 0.3s;
}
.noUi-state-drag * {
cursor: inherit !important;
}
/* Slider size and handle placement;
*/
.noUi-horizontal {
height: 18px;
}
.noUi-horizontal .noUi-handle {
width: 34px;
height: 28px;
right: -17px;
top: -6px;
}
.noUi-vertical {
width: 18px;
}
.noUi-vertical .noUi-handle {
width: 28px;
height: 34px;
right: -6px;
bottom: -17px;
}
.noUi-txt-dir-rtl.noUi-horizontal .noUi-handle {
left: -17px;
right: auto;
}
/* Styling;
* Giving the connect element a border radius causes issues with using transform: scale
*/
.noUi-target {
background: #FAFAFA;
border-radius: 4px;
border: 1px solid #D3D3D3;
box-shadow: inset 0 1px 1px #F0F0F0, 0 3px 6px -5px #BBB;
}
.noUi-connects {
border-radius: 3px;
}
.noUi-connect {
background: #3FB8AF;
}
/* Handles and cursors;
*/
.noUi-draggable {
cursor: ew-resize;
}
.noUi-vertical .noUi-draggable {
cursor: ns-resize;
}
.noUi-handle {
border: 1px solid #D9D9D9;
border-radius: 3px;
background: #FFF;
cursor: default;
box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #EBEBEB, 0 3px 6px -3px #BBB;
}
.noUi-active {
box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #DDD, 0 3px 6px -3px #BBB;
}
/* Handle stripes;
*/
.noUi-handle:before,
.noUi-handle:after {
content: "";
display: block;
position: absolute;
height: 14px;
width: 1px;
background: #E8E7E6;
left: 14px;
top: 6px;
}
.noUi-handle:after {
left: 17px;
}
.noUi-vertical .noUi-handle:before,
.noUi-vertical .noUi-handle:after {
width: 14px;
height: 1px;
left: 6px;
top: 14px;
}
.noUi-vertical .noUi-handle:after {
top: 17px;
}
/* Disabled state;
*/
[disabled] .noUi-connect {
background: #B8B8B8;
}
[disabled].noUi-target,
[disabled].noUi-handle,
[disabled] .noUi-handle {
cursor: not-allowed;
}
/* Base;
*
*/
.noUi-pips,
.noUi-pips * {
-moz-box-sizing: border-box;
box-sizing: border-box;
}
.noUi-pips {
position: absolute;
color: #999;
}
/* Values;
*
*/
.noUi-value {
position: absolute;
white-space: nowrap;
text-align: center;
}
.noUi-value-sub {
color: #ccc;
font-size: 10px;
}
/* Markings;
*
*/
.noUi-marker {
position: absolute;
background: #CCC;
}
.noUi-marker-sub {
background: #AAA;
}
.noUi-marker-large {
background: #AAA;
}
/* Horizontal layout;
*
*/
.noUi-pips-horizontal {
padding: 10px 0;
height: 80px;
top: 100%;
left: 0;
width: 100%;
}
.noUi-value-horizontal {
-webkit-transform: translate(-50%, 50%);
transform: translate(-50%, 50%);
}
.noUi-rtl .noUi-value-horizontal {
-webkit-transform: translate(50%, 50%);
transform: translate(50%, 50%);
}
.noUi-marker-horizontal.noUi-marker {
margin-left: -1px;
width: 2px;
height: 5px;
}
.noUi-marker-horizontal.noUi-marker-sub {
height: 10px;
}
.noUi-marker-horizontal.noUi-marker-large {
height: 15px;
}
/* Vertical layout;
*
*/
.noUi-pips-vertical {
padding: 0 10px;
height: 100%;
top: 0;
left: 100%;
}
.noUi-value-vertical {
-webkit-transform: translate(0, -50%);
transform: translate(0, -50%);
padding-left: 25px;
}
.noUi-rtl .noUi-value-vertical {
-webkit-transform: translate(0, 50%);
transform: translate(0, 50%);
}
.noUi-marker-vertical.noUi-marker {
width: 5px;
height: 2px;
margin-top: -1px;
}
.noUi-marker-vertical.noUi-marker-sub {
width: 10px;
}
.noUi-marker-vertical.noUi-marker-large {
width: 15px;
}
.noUi-tooltip {
display: block;
position: absolute;
border: 1px solid #D9D9D9;
border-radius: 3px;
background: #fff;
color: #000;
padding: 5px;
text-align: center;
white-space: nowrap;
}
.noUi-horizontal .noUi-tooltip {
-webkit-transform: translate(-50%, 0);
transform: translate(-50%, 0);
left: 50%;
bottom: 120%;
}
.noUi-vertical .noUi-tooltip {
-webkit-transform: translate(0, -50%);
transform: translate(0, -50%);
top: 50%;
right: 120%;
}
.noUi-horizontal .noUi-origin > .noUi-tooltip {
-webkit-transform: translate(50%, 0);
transform: translate(50%, 0);
left: auto;
bottom: 10px;
}
.noUi-vertical .noUi-origin > .noUi-tooltip {
-webkit-transform: translate(0, -18px);
transform: translate(0, -18px);
top: auto;
right: 28px;
}

2254
www/assets/nouislider-15.5.0.js Normal file
View file

File diff suppressed because it is too large Load diff

1
www/assets/wNumb-1.2.0.min.js vendored Normal file
View file

@ -0,0 +1 @@
!function(e){"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?module.exports=e():window.wNumb=e()}(function(){"use strict";var o=["decimals","thousand","mark","prefix","suffix","encoder","decoder","negativeBefore","negative","edit","undo"];function w(e){return e.split("").reverse().join("")}function h(e,t){return e.substring(0,t.length)===t}function f(e,t,n){if((e[t]||e[n])&&e[t]===e[n])throw new Error(t)}function x(e){return"number"==typeof e&&isFinite(e)}function n(e,t,n,r,i,o,f,u,s,c,a,p){var d,l,h,g=p,v="",m="";return o&&(p=o(p)),!!x(p)&&(!1!==e&&0===parseFloat(p.toFixed(e))&&(p=0),p<0&&(d=!0,p=Math.abs(p)),!1!==e&&(p=function(e,t){return e=e.toString().split("e"),(+((e=(e=Math.round(+(e[0]+"e"+(e[1]?+e[1]+t:t)))).toString().split("e"))[0]+"e"+(e[1]?e[1]-t:-t))).toFixed(t)}(p,e)),-1!==(p=p.toString()).indexOf(".")?(h=(l=p.split("."))[0],n&&(v=n+l[1])):h=p,t&&(h=w((h=w(h).match(/.{1,3}/g)).join(w(t)))),d&&u&&(m+=u),r&&(m+=r),d&&s&&(m+=s),m+=h,m+=v,i&&(m+=i),c&&(m=c(m,g)),m)}function r(e,t,n,r,i,o,f,u,s,c,a,p){var d,l="";return a&&(p=a(p)),!(!p||"string"!=typeof p)&&(u&&h(p,u)&&(p=p.replace(u,""),d=!0),r&&h(p,r)&&(p=p.replace(r,"")),s&&h(p,s)&&(p=p.replace(s,""),d=!0),i&&function(e,t){return e.slice(-1*t.length)===t}(p,i)&&(p=p.slice(0,-1*i.length)),t&&(p=p.split(t).join("")),n&&(p=p.replace(n,".")),d&&(l+="-"),""!==(l=(l+=p).replace(/[^0-9\.\-.]/g,""))&&(l=Number(l),f&&(l=f(l)),!!x(l)&&l))}function i(e,t,n){var r,i=[];for(r=0;r<o.length;r+=1)i.push(e[o[r]]);return i.push(n),t.apply("",i)}return function e(t){if(!(this instanceof e))return new e(t);"object"==typeof t&&(t=function(e){var t,n,r,i={};for(void 0===e.suffix&&(e.suffix=e.postfix),t=0;t<o.length;t+=1)if(void 0===(r=e[n=o[t]]))"negative"!==n||i.negativeBefore?"mark"===n&&"."!==i.thousand?i[n]=".":i[n]=!1:i[n]="-";else if("decimals"===n){if(!(0<=r&&r<8))throw new Error(n);i[n]=r}else if("encoder"===n||"decoder"===n||"edit"===n||"undo"===n){if("function"!=typeof r)throw new Error(n);i[n]=r}else{if("string"!=typeof r)throw new Error(n);i[n]=r}return f(i,"mark","thousand"),f(i,"prefix","negative"),f(i,"prefix","negativeBefore"),i}(t),this.to=function(e){return i(t,n,e)},this.from=function(e){return i(t,r,e)})}});

2
www/draw.js
View file

@ -17,7 +17,7 @@ class Canvas {
this.wrapperEl.appendChild(this.filenameEl);
this.colors = ["red", "blue", "green"];
this.colors = ["black", "red", "blue", "green"];
this.resize();

21
www/play.html
View file

@ -36,6 +36,10 @@
stroke-linecap: round;
}
path.before_in {
opacity: .2;
}
#wrapper {
position: absolute;
top: 0;
@ -72,12 +76,29 @@
.playlist li:hover{
color: blue;
}
input[type='range']{
position: absolute;
z-index: 100;
bottom: 0;
left:0;
right: 0;width: 90%;
}
.scrubber{
position: absolute;
z-index: 100;
bottom: 0;
left:0;
right: 0;width: 90%;
}
</style>
<link rel="stylesheet" href="assets/nouislider-15.5.0.css">
</head>
<body>
<div id='interface'>
</div>
<script src="assets/nouislider-15.5.0.js"></script>
<script src="play.js"></script>
<script type='text/javascript'>
const player = new Player(document.getElementById("interface"));

323
www/play.js
View file

@ -1,11 +1,28 @@
class Player {
constructor(wrapperEl) {
this.wrapperEl = wrapperEl;
this.svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
this.wrapperEl.appendChild(this.svgEl);
this.scrubberElOld = document.createElement('input');
this.scrubberElOld.type = "range";
this.scrubberElOld.min = 0;
this.scrubberElOld.step = 0.01;
this.wrapperEl.appendChild(this.scrubberElOld);
this.scrubberEl = document.createElement('div');
this.scrubberEl.classList.add('scrubber')
this.wrapperEl.appendChild(this.scrubberEl);
this.scrubberElOld.addEventListener("input", (ev) => {
this.scrubTo(ev.target.value);
})
this.inPointPosition = null;
this.outPointPosition = null;
this.currentTime = 0;
this.isPlaying = false;
}
playlist(url) {
@ -15,31 +32,31 @@ class Player {
fetch(request)
.then(response => response.json())
.then(data => {
let playlist = this.wrapperEl.querySelector('.playlist');
if(!playlist) {
playlist = document.createElement('nav');
playlist.classList.add('playlist');
this.wrapperEl.appendChild(playlist)
}
else{
playlist.innerHTML = "";
}
.then(response => response.json())
.then(data => {
let playlist = this.wrapperEl.querySelector('.playlist');
if (!playlist) {
playlist = document.createElement('nav');
playlist.classList.add('playlist');
this.wrapperEl.appendChild(playlist)
}
else {
playlist.innerHTML = "";
}
const listEl = document.createElement("ul");
for(let fileUrl of data) {
const liEl = document.createElement("li");
liEl.innerText = fileUrl
liEl.addEventListener('click', (e) => {
this.play(fileUrl);
playlist.style.display = "none";
});
listEl.appendChild(liEl);
}
playlist.appendChild(listEl);
// do something with the data sent in the request
});
const listEl = document.createElement("ul");
for (let fileUrl of data) {
const liEl = document.createElement("li");
liEl.innerText = fileUrl
liEl.addEventListener('click', (e) => {
this.play(fileUrl);
playlist.style.display = "none";
});
listEl.appendChild(liEl);
}
playlist.appendChild(listEl);
// do something with the data sent in the request
});
}
play(file) {
@ -49,61 +66,245 @@ class Player {
fetch(request)
.then(response => response.json())
.then(data => {
this.playStrokes(data)
// do something with the data sent in the request
.then(response => response.json())
.then(data => {
this.playStrokes(data)
// do something with the data sent in the request
});
}
playStrokes(drawing) {
this.strokes = drawing.shape;
this.currentPathI = null;
this.currentPointI = null;
this.dimensions = drawing.dimensions;
this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
this.startTime = window.performance.now() - this.strokes[0].points[0][3];
this.playStrokePosition(0, 1);
this.duration = this.getDuration();
this.scrubberElOld.max = this.duration;
this.playTimout = null;
const slider = noUiSlider.create(this.scrubberEl, {
start: [this.currentTime, this.duration],
connect: true,
range: {
'min': 0,
'max': this.duration
}
});
slider.on("slide", (values, handle) => {
this.isPlaying = false;
// console.log(values, handle);
// both in and out need to have a value
this.inPointPosition = this.findPositionForTime(values[0]);
this.outPointPosition = this.findPositionForTime(values[1]);
if (handle === 0) {
// in point
if (
this.currentPathI < this.inPointPosition[0] ||
this.currentPointI < this.inPointPosition[1]) {
this.drawStrokePosition(
// this.inPointPosition[0],
// this.inPointPosition[1],
// always draw at out position, as to see the whole shape of the range
this.outPointPosition[0],
this.outPointPosition[1],
);
}
}
if (handle === 1) {
// out point
// this.outPointPosition = this.findPositionForTime(values[1]);
this.drawStrokePosition(
this.outPointPosition[0],
this.outPointPosition[1],
);
}
// this.inPointPosition = values;
// this.outPointPosition = vaalues[0];
// this.scrubTo()
// this.scrubTo(ev.target.value);
});
}
playStrokes(drawing){
this.strokes = drawing.shape;
this.currentPath = null;
this.dimensions = drawing.dimensions;
this.svgEl.setAttribute('viewBox', `0 0 ${this.dimensions[0]} ${this.dimensions[1]}`)
this.startTime = window.performance.now() - this.strokes[0].points[0][3];
this.playStroke(0,1);
getDuration() {
const points = this.strokes[this.strokes.length - 1].points;
return points[points.length - 1][3];
}
playStroke(path_i, point_i){
getStrokesForPathRange(in_point, out_point) {
// get paths for given range. Also, split path at in & out if necessary.
strokes = {};
for (let i = in_point[0]; i <= out_point[0]; i++) {
const path = this.strokes[i];
const in_i = (in_point[0] === i) ? in_point[1] : 0;
const out_i = (out_point[0] === i) ? out_point[1] : Math.inf;
const points = path.points.slice(in_i, out_i);
strokes[i] = points; // preserve indexes
}
return strokes;
}
// TODO: when drawing, have a group active & inactive.
// active is getPathRange(currentIn, currentOut)
// inactive is what comes before and after.
// then, playing the video is just running pathRanghe(0, playhead)
drawStrokePosition(path_i, point_i, show_all) {
if(typeof show_all === 'undefined')
show_all = false;
// check if anything is placed that is in the future from the current playhead
if (this.currentPathI !== null && this.currentPoint !== null) {
if (this.currentPathI > path_i) {
console.log('remove', path_i, ' -> ', this.currentPathI)
// remove everything that comes later
for (let i = path_i + 1; i <= this.currentPathI; i++) {
// console.log('remove', i);
const pathEl = this.svgEl.querySelector(`.path${i}`);
if (pathEl) {
pathEl.parentNode.removeChild(pathEl);
}
}
}
}
// an inpoint is set, so we're annotating
// make everything coming before translucent
if (this.inPointPosition !== null) {
const [inPath_i, inPoint_i] = this.inPointPosition;
// returns a static NodeList
const currentBeforeEls = this.svgEl.querySelectorAll(`.before_in`);
for (let currentBeforeEl of currentBeforeEls) {
currentBeforeEl.classList.remove('before_in');
}
for (let index = 0; index < inPath_i; index++) {
const pathEl = this.svgEl.querySelector(`.path${index}`);
if (pathEl) {
pathEl.classList.add('before_in');
}
}
}
this.currentPathI = path_i;
this.currentPointI = point_i;
const path = this.strokes[path_i];
// console.log(path);
let pathEl = this.svgEl.querySelector(`.path${path_i}`);
if(!pathEl){
if (!pathEl) {
pathEl = document.createElementNS('http://www.w3.org/2000/svg', 'path');
pathEl.style.stroke = path.color;
pathEl.classList.add('path'+path_i)
pathEl.classList.add('path' + path_i)
this.svgEl.appendChild(pathEl)
}
const d = this.strokes2D(path.points.slice(0, point_i));
pathEl.setAttribute('d', d);
let next_path, next_point,t;
if(path.points.length > point_i + 1){
next_path = path_i;
next_point = point_i + 1;
t = path.points[next_point][3];// - path.points[point_i][3];
// setTimeout(() => this.playStroke(next_path, next_point), dt);
} else if(this.strokes.length > path_i + 1) {
next_path = path_i + 1;
next_point = 1;
t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3];
// use starttime instead of diff, to prevent floating
} else {
console.log('done');
return
}
const dt = t - (window.performance.now() - this.startTime);
setTimeout(() => this.playStroke(next_path, next_point), dt);
const stroke = path.points.slice(0, point_i);
const d = this.strokes2D(stroke);
pathEl.setAttribute('d', d);
this.scrubberElOld.value = path.points[point_i][3];
this.currentTime = path.points[point_i][3];
}
playUntil(path_i){
getNextPosition(path_i, point_i) {
const path = this.strokes[path_i];
let next_path, next_point;
if (path.points.length > point_i + 1) {
next_path = path_i;
next_point = point_i + 1;
// setTimeout(() => this.playStroke(next_path, next_point), dt);
} else if (this.strokes.length > path_i + 1) {
next_path = path_i + 1;
next_point = 1;
// use starttime instead of diff, to prevent floating
} else {
return [null, null];
}
// when an outpoint is set, stop playing there
if(next_path > this.outPointPosition[0] || next_point > this.outPointPosition[1]){
return [null, null];
}
return [next_path, next_point];
}
playStrokePosition(path_i, point_i, allow_interrupt) {
if(allow_interrupt) {
if(!this.isPlaying) {
console.log('not playing because of interrupt');
return;
}
} else{
this.isPlaying = true;
}
this.drawStrokePosition(path_i, point_i);
const [next_path, next_point] = this.getNextPosition(path_i, point_i);
if (next_path === null) {
console.log('done playing');
return;
}
const t = this.strokes[next_path].points[next_point][3];// - path.points[point_i][3];
const dt = t - (window.performance.now() - this.startTime);
this.playTimout = setTimeout(() => this.playStrokePosition(next_path, next_point, true), dt);
}
playUntil(path_i) {
// for scrubber
}
scrubTo(ms) {
const [path_i, point_i] = this.findPositionForTime(ms);
// console.log(path_i, point_i);
clearTimeout(this.playTimout);
this.playStrokePosition(path_i, point_i);
// this.playHead = ms;
}
findPositionForTime(ms) {
ms = Math.min(Math.max(ms, 0), this.duration);
console.log('scrub to', ms)
let path_i = 0;
let point_i = 0;
this.strokes.every((stroke, index) => {
const startAt = stroke.points[0][3];
const endAt = stroke.points[stroke.points.length - 1][3];
if (startAt > ms) {
return false; // too far
}
if (endAt > ms) {
// we're getting close. Find the right point_i
path_i = index;
stroke.points.every((point, pi) => {
if (point[3] > ms) {
// too far
return false;
}
point_i = pi;
return true;
});
return false;
} else {
// in case nothings comes after, we store the last best option thus far
path_i = index;
point_i = stroke.points.length - 1;
return true;
}
});
return [path_i, point_i];
}
strokes2D(strokes) {
// strokes to a d attribute for a path