diff --git a/README.md b/README.md index ab67a4e..df5aa4f 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,10 @@ Create a hand drawn vector animation. poetry run python webserver.py ``` -`parse_offsets.py` can be used to pad the diagram in order to sync it with the audio. This is necessary eg. after a network failure. It works by adding a line with the required offset to the `.json_appendable`-file. \ No newline at end of file +`parse_offsets.py` can be used to pad the diagram in order to sync it with the audio. This is necessary eg. after a network failure. It works by adding a line with the required offset to the `.json_appendable`-file. + +## record + +On Linux: for now, set the pulse default input device to the monitor of the speakers. Then use wf-recorder: + +`wf-recorder -g"$(slurp)" -a -f recording.mp4` diff --git a/app/svganim/strokes.py b/app/svganim/strokes.py index f796c5a..8a05cc9 100644 --- a/app/svganim/strokes.py +++ b/app/svganim/strokes.py @@ -1,7 +1,10 @@ from __future__ import annotations +import asyncio +import copy import json from os import X_OK, PathLike import os +import subprocess from typing import Optional, Union import shelve from pydub import AudioSegment @@ -12,9 +15,11 @@ import logging logger = logging.getLogger('svganim.strokes') +Milliseconds = float +Seconds = float class Annotation: - def __init__(self, tag: str, drawing: Drawing, t_in: float, t_out: float) -> None: + def __init__(self, tag: str, drawing: Drawing, t_in: Milliseconds, t_out: Milliseconds) -> None: self.tag = tag self.t_in = t_in self.t_out = t_out @@ -29,10 +34,15 @@ class Annotation: def get_as_svg(self) -> str: return self.getAnimationSlice().get_as_svg() + + def getJsonUrl(self) -> str: + return self.drawing.get_url() + f"?t_in={self.t_in}&t_out={self.t_out}" Filename = Union[str, bytes, PathLike[str], PathLike[bytes]] +SliceId = [str, float, float] + class Drawing: def __init__(self, filename: Filename, metadata_dir: Filename, basedir: Filename) -> None: @@ -70,24 +80,26 @@ class Drawing: if 'file' not in md['audio']: return None - return AudioSlice(filename=os.path.join(self.basedir, md['audio']['file'][1:]), offset=md['audio']['offset']*1000) + return AudioSlice(filename=os.path.join(self.basedir, md['audio']['file'][1:]), drawing=self, offset=md['audio']['offset']*1000) def get_animation(self) -> AnimationSlice: # with open(self.eventfile, "r") as fp: strokes = [] + viewboxes = [] with open(self.eventfile, "r") as fp: events = json.loads("[" + fp.read() + "]") for i, event in enumerate(events): if i == 0: - # metadata on first line - pass + # metadata on first line, add as initial viewbox to slice + viewboxes.append(TimedViewbox(-float('Infinity'), 0, 0, event[1], event[2])) else: if type(event) is list: # ignore double metadatas, which appear when continuaing an existing drawing continue if event["event"] == "viewbox": - pass + viewboxes.extend([TimedViewbox( + b['t'], b['x'], b['y'], b['width'], b['height']) for b in event['viewboxes']]) if event["event"] == "stroke": # points = [] # for i in range(int(len(stroke) / 4)): @@ -100,7 +112,7 @@ class Drawing: for p in event["points"]], ) ) - return AnimationSlice(strokes, audioslice=self.get_audio()) + return AnimationSlice([self.id, None, None], strokes, viewboxes, audioslice=self.get_audio()) def get_metadata(self): canvas = self.get_canvas_metadata() @@ -127,6 +139,12 @@ class Viewbox: return f"{self.x} {self.y} {self.width} {self.height}" +class TimedViewbox(Viewbox): + def __init__(self, time: Milliseconds, x: float, y: float, width: float, height: float): + super().__init__(x, y, width, height) + self.t = time + + FrameIndex = tuple[int, int] @@ -134,36 +152,89 @@ class AnimationSlice: # either a whole drawing or the result of applying an annotation to a drawing (an excerpt) # TODO rename to AnimationSlice to include audio as well def __init__( - self, strokes: list[Stroke], t_in: float = 0, t_out: float = None, audioslice: AudioSlice = None + self, slice_id: SliceId, strokes: list[Stroke], viewboxes: list[TimedViewbox] = [], t_in: float = 0, t_out: float = None, audioslice: AudioSlice = None ) -> None: + self.id = slice_id self.strokes = strokes + self.viewboxes = viewboxes self.t_in = t_in self.t_out = t_out self.audio = audioslice # TODO: Audio - def get_bounding_box(self) -> Viewbox: + def asDict(self) -> dict: + """Can be used to json-ify the animation-slice + """ + + # conversion necessary for when no t_in is given + boxes = [v.__dict__ for v in self.viewboxes] + for box in boxes: + if box['t'] == -float('Infinity'): + box['t'] = 0 + + drawing = { + "file": self.getUrl(), + "time": "-", # creation date + # dimensions of drawing canvas + "dimensions": [self.viewboxes[0].width, self.viewboxes[0].height], + "shape": [s.asDict() for s in self.strokes], + "viewboxes": boxes, + "bounding_box": self.get_bounding_box().__dict__, + "audio": self.getAudioDict() if self.audio else None + } + return drawing + + def getAudioDict(self): + """quick and dirty to not use audio.asDict(), but it avoids passing all around sorts of data""" + return { + "file": '/files/' + self.getUrl('.mp3'), + "offset": 0 + # "offset": self.audio.offset / 1000 + } + + def getUrl(self, extension = '') -> str: + if not self.id[1] and not self.id[2]: + return self.id[0] + + return self.id[0] + f"{extension}?t_in={self.t_in}&t_out={self.t_out}" + + def get_bounding_box(self, stroke_thickness: float = 3.5) -> Viewbox: + """Stroke_thickness 3.5 == 1mm. If it should not be considered, just set it to 0. + """ + if len(self.strokes) == 0: + # empty set + return Viewbox(0,0,0,0) + min_x, max_x = float("inf"), float("-inf") min_y, max_y = float("inf"), float("-inf") for s in self.strokes: for p in s.points: - if p.x < min_x: - min_x = p.x - if p.x > max_x: - max_x = p.x - if p.y < min_y: - min_y = p.y - if p.y > max_y: - max_y = p.y + + x1 = p.x - stroke_thickness/2 + x2 = p.x + stroke_thickness/2 + y1 = p.y - stroke_thickness/2 + y2 = p.y + stroke_thickness/2 + if x1 < min_x: + min_x = x1 + if x2 > max_x: + max_x = x2 + if y1 < min_y: + min_y = y1 + if y2 > max_y: + max_y = y2 return Viewbox(min_x, min_y, max_x - min_x, max_y - min_y) - def getSlice(self, t_in: float, t_out: float) -> AnimationSlice: + def getSlice(self, t_in: Milliseconds, t_out: Milliseconds) -> AnimationSlice: + """slice the slice. T in ms""" frame_in = self.getIndexForInPoint(t_in) frame_out = self.getIndexForOutPoint(t_out) - strokes = self.getStrokeSlices(frame_in, frame_out) + strokes = self.getStrokeSlices(frame_in, frame_out, t_in) + # TODO shift t of points with t_in + viewboxes = self.getViewboxesSlice(t_in, t_out) + print(viewboxes[0]) audio = self.audio.getSlice(t_in, t_out) if self.audio else None - return AnimationSlice(strokes, t_in, t_out, audio) + return AnimationSlice([self.id[0], t_in, t_out], strokes, viewboxes, t_in, t_out, audio) def get_as_svg_dwg(self) -> svgwrite.Drawing: box = self.get_bounding_box() @@ -171,7 +242,8 @@ class AnimationSlice: dwg = svgwrite.Drawing(fn, size=(box.width, box.height)) dwg.viewbox(box.x, box.y, box.width, box.height) self.add_to_dwg(dwg) - dwg.defs.add(dwg.style("path{stroke-width:1mm;stroke-linecap: round;}")) + dwg.defs.add( + dwg.style("path{stroke-width:1mm;stroke-linecap: round;}")) return dwg def get_as_svg(self) -> str: @@ -186,8 +258,32 @@ class AnimationSlice: stroke.add_to_dwg(group) dwg.add(group) + def getViewboxesSlice(self, t_in: Milliseconds, t_out: Milliseconds) -> list[TimedViewbox]: + """Extract the viewboxes for in- and outpoints. + If there's one before inpoint, move that to the t_in, so that animation starts at the right position + the slice is offset by t_in ms + """ + viewboxes = [] # Add single empty element, so that we can use viewboxes[0] later + lastbox = None + for viewbox in self.viewboxes: + if viewbox.t > t_out: + break + + if viewbox.t <= t_in: + # make sure the first box is the last box from _before_ the slice + firstbox = TimedViewbox( + 0, viewbox.x, viewbox.y, viewbox.width, viewbox.height) + if not len(viewboxes): + viewboxes.append(firstbox) + else: + viewboxes[0] = firstbox + continue + + viewboxes.append(TimedViewbox(viewbox.t-t_in, viewbox.x, viewbox.y, viewbox.width, viewbox.height)) + return viewboxes + def getStrokeSlices( - self, index_in: FrameIndex, index_out: FrameIndex + self, index_in: FrameIndex, index_out: FrameIndex, t_offset: Seconds = 0 ) -> list[Stroke]: """Get list of Stroke/StrokeSlice based in in and out indexes Based on annotation.js getStrokesSliceForPathRange(in_point, out_point) @@ -204,10 +300,10 @@ class AnimationSlice: out_i = index_out[1] if index_out[0] == i else len( stroke.points) - 1 - slices.append(StrokeSlice(stroke, in_i, out_i)) + slices.append(StrokeSlice(stroke, in_i, out_i, t_offset)) return slices - def getIndexForInPoint(self, ms) -> FrameIndex: + def getIndexForInPoint(self, ms: Milliseconds) -> FrameIndex: """Get the frame index (path, point) based on the given time The In point version (so the first index after ms) Equal to annotations.js findPositionForTime(ms) @@ -235,14 +331,14 @@ class AnimationSlice: break # done :-) return (path_i, point_i) - def getIndexForOutPoint(self, ms) -> FrameIndex: + def getIndexForOutPoint(self, ms: Milliseconds) -> FrameIndex: """Get the frame index (path, point) based on the given time The Out point version (so the last index before ms) Equal to annotations.js findPositionForTime(ms) """ - return self.getIndexForTime( ms) + return self.getIndexForTime(ms) - def getIndexForTime(self, ms) -> FrameIndex: + def getIndexForTime(self, ms: Milliseconds) -> FrameIndex: """Get the frame index (path, point) based on the given time Equal to annotations.js findPositionForTime(ms) """ @@ -269,49 +365,114 @@ class AnimationSlice: point_i = len(stroke.points) - 1 return (path_i, point_i) +audiocache = {} class AudioSlice: - def __init__(self, filename: Filename, t_in: float = None, t_out: float = None, offset: float = None): + def __init__(self, filename: Filename, drawing: Drawing, t_in: Milliseconds = None, t_out: Milliseconds = None, offset: Milliseconds = None): self.filename = filename + self.drawing = drawing self.t_in = t_in # in ms self.t_out = t_out # in ms - self.offset = offset # in ms + self.offset = offset # in ms TODO: use from self.drawing metadata def getSlice(self, t_in: float, t_out: float) -> AnimationSlice: - return AudioSlice(self.filename, t_in, t_out, self.offset) + return AudioSlice(self.filename, self.drawing, t_in, t_out, self.offset) - def export(self, format="mp3"): + def asDict(self): + return { + "file": self.getUrl(), + # "offset": self.offset/1000 + } + + def getUrl(self): + fn = self.filename.replace("../files/audio", "/file/") + + params = [] + if self.t_in: + params.append(f"t_in={self.t_in}") + if self.t_out: + params.append(f"t_out={self.t_in}") + if len(params): + fn += "?" + "&".join(params) + return fn + + async def export(self, format="mp3"): """Returns file descriptor of tempfile""" # Opening file and extracting segment - song = AudioSegment.from_file(self.filename) - start = self.t_in - self.offset - end = self.t_out - self.offset + start = int(self.t_in - self.offset) # millisecond precision is enough + end = int(self.t_out - self.offset) # millisecond precision is enough - if start < 0 and end < 0: - extract = AudioSegment.silent( - duration=end-start, frame_rate=song.frame_rate) - else: - if start < 0: - preroll = AudioSegment.silent( - duration=start * -1, frame_rate=song.frame_rate) - start = 0 - else: - preroll = None - if end > len(song): - postroll = AudioSegment.silent( - duration=end - len(song), frame_rate=song.frame_rate) - end = len(song) - 1 - else: - postroll = None + # call ffmpeg directly, with given in and outpoint, so no unnecessary data is loaded, and no double conversion (e.g. ogg -> wav -> ogg ) is performed + out_f = io.BytesIO() - extract = song[start: end] - if preroll: - extract = preroll + extract - if postroll: - extract += postroll + # build converter command to export + conversion_command = [ + "ffmpeg", + '-ss', f"{start}ms", + '-to', f"{end}ms", + "-i", self.filename, # ss before input, so not whole file is loaded + ] - # Saving - return extract.export(None, format=format) + conversion_command.extend([ + "-f", format, '-', # to stdout + ]) + + # read stdin / write stdout + logger.info("ffmpeg start") + proc = await asyncio.create_subprocess_exec( + *conversion_command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL) + + p_out, p_err = await proc.communicate() + + logger.info("ffmpeg finished") + + if proc.returncode != 0: + raise Exception( + "Encoding failed. ffmpeg/avlib returned error code: {0}\n\nCommand:{1}".format( + p.returncode, conversion_command)) + + out_f.write(p_out) + + + out_f.seek(0) + return out_f + + # old way, use AudioSegment, easy but slow (reads whole ogg to wav, then export segment to ogg again) + # logger.info("loading audio") + # if self.filename in audiocache: + # song = audiocache[self.filename] + # else: + # song = AudioSegment.from_file(self.filename) + # audiocache[self.filename] = song + # logger.info("loaded audio") + + # if start < 0 and end < 0: + # extract = AudioSegment.silent( + # duration=end-start, frame_rate=song.frame_rate) + # else: + # if start < 0: + # preroll = AudioSegment.silent( + # duration=start * -1, frame_rate=song.frame_rate) + # start = 0 + # else: + # preroll = None + # if end > len(song): + # postroll = AudioSegment.silent( + # duration=end - len(song), frame_rate=song.frame_rate) + # end = len(song) - 1 + # else: + # postroll = None + + # extract = song[start: end] + # if preroll: + # extract = preroll + extract + # if postroll: + # extract += postroll + + # # Saving + # return extract.export(None, format=format) class AnnotationIndex: @@ -395,7 +556,7 @@ class AnnotationIndex: class Point: - def __init__(self, x: float, y: float, last: bool, t: float): + def __init__(self, x: float, y: float, last: bool, t: Seconds): self.x = float(x) self.y = float(y) # if y == 0 it can still be integer.... odd python self.last = last @@ -409,6 +570,9 @@ class Point: # TODO: change so that it actually scales to FIT dimensions return Point(self.x, self.y, self.last, self.t) + def asList(self) -> list: + return [self.x, self.y, 1 if self.last else 0, self.t] + Points = list[Point] SvgDrawing = Union[svgwrite.container.SVG, svgwrite.container.Group] @@ -419,25 +583,28 @@ class Stroke: self.color = color self.points = points + def asDict(self) -> dict: + return {"color": self.color, "points": [p.asList() for p in self.points]} + def add_to_dwg(self, dwg: SvgDrawing): path = svgwrite.path.Path(d=self.get_as_d()).stroke( self.color, 1).fill("none") dwg.add(path) - def get_bounding_box(self) -> Viewbox: - min_x, max_x = float("inf"), float("-inf") - min_y, max_y = float("inf"), float("-inf") + # def get_bounding_box(self) -> Viewbox: + # min_x, max_x = float("inf"), float("-inf") + # min_y, max_y = float("inf"), float("-inf") - for p in self.points: - if p.x < min_x: - min_x = p.x - if p.x > max_x: - max_x = p.x - if p.y < min_y: - min_y = p.y - if p.y > max_y: - max_y = p.y - return Viewbox(min_x, min_y, max_x - min_x, max_y - min_y) + # for p in self.points: + # if p.x < min_x: + # min_x = p.x + # if p.x > max_x: + # max_x = p.x + # if p.y < min_y: + # min_y = p.y + # if p.y > max_y: + # max_y = p.y + # return Viewbox(min_x, min_y, max_x - min_x, max_y - min_y) def get_as_d(self): d = "" @@ -466,17 +633,22 @@ class Stroke: class StrokeSlice(Stroke): - def __init__(self, stroke: Stroke, i_in: int = None, i_out: int = None) -> None: + def __init__(self, stroke: Stroke, i_in: int = None, i_out: int = None, t_offset: Seconds = 0) -> None: self.stroke = stroke self.i_in = 0 if i_in is None else i_in self.i_out = len(self.stroke.points) - 1 if i_out is None else i_out + # deepcopy points, because slices can be offset in time + self.points = copy.deepcopy(self.stroke.points[self.i_in: self.i_out + 1]) + for p in self.points: + p.t -= t_offset + def slice_id(self): return f"{self.i_in}-{self.i_out}" - @property - def points(self) -> Points: - return self.stroke.points[self.i_in: self.i_out + 1] + # @property + # def points(self) -> Points: + # return self.stroke.points[self.i_in: self.i_out + 1] @property def color(self) -> str: diff --git a/app/svganim/uimethods.py b/app/svganim/uimethods.py new file mode 100644 index 0000000..38acd95 --- /dev/null +++ b/app/svganim/uimethods.py @@ -0,0 +1,8 @@ +from hashlib import md5 + + +def annotation_hash(handler, input): + return md5(input.encode()).hexdigest() + +# def nmbr(handler, lst) -> int: +# leno \ No newline at end of file diff --git a/app/templates/index.html b/app/templates/index.html index 505b02d..1d447bd 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -2,9 +2,12 @@ Annotations + + + + + {% for tag in index.tags %} -

{{tag}}

- +
+ +

{{tag}} ({{len(index.tags[tag])}})

+ + +
{% end %}