commit 1702cce9d1857aa4d010106a5af9094e819c3ce1 Author: Ruben van de Ven Date: Mon Dec 16 12:19:48 2019 +0100 First version diff --git a/README.md b/README.md new file mode 100644 index 0000000..28792c0 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ + +Build array of images sorted by size: + +```bash +python zoom_animation.py --annotations ../../datasets/COCO/annotations/instances_train2017.json --output zoom --category_id 18 +``` + +Turn into png + +```bash +cd zoom/dog +for file in *.svg; do inkscape -z -f "${file}" -w 640 -e "../dog_png/${file}.png"; done +``` + +Turn png into mp4 + +```bash +cd ../dog_png +#ffmpeg -r 1 -i %d_*.png -pix_fmt yuv420p bloch2.mp4 +ffmpeg -f image2 -pattern_type glob -i '*.png' ../dog.mp4 + +``` diff --git a/coco.sql b/coco.sql new file mode 100644 index 0000000..841127f --- /dev/null +++ b/coco.sql @@ -0,0 +1,44 @@ +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS "segments" ( + "id" INTEGER UNIQUE, + "annotation_id" INTEGER, + "points" TEXT, + PRIMARY KEY("id") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "categories" ( + "id" INTEGER UNIQUE, + "supercategory" TEXT, + "name" TEXT UNIQUE, + PRIMARY KEY("id") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "images" ( + "id" INTEGER UNIQUE, + "flickr_url" TEXT, + "coco_url" TEXT, + "width" FLOAT, + "height" FLOAT, + "date_captured" DATETIME, + PRIMARY KEY("id","id") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "annotations" ( + "id" INTEGER UNIQUE, + "image_id" INTEGER, + "category_id" INTEGER, + "iscrowd" BOOL, + "area" FLOAT, + "bbox_top" FLOAT, + "bbox_left" FLOAT, + "bbox_width" FLOAT, + "bbox_height" FLOAT, + PRIMARY KEY("id") +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS "segments_annotation" ON "segments" ( + "annotation_id" +); +CREATE INDEX IF NOT EXISTS "annotations_image" ON "annotations" ( + "image_id" +); +CREATE INDEX IF NOT EXISTS "annotations_category" ON "annotations" ( + "category_id" +); +COMMIT; diff --git a/generate_lonely_segments.py b/generate_lonely_segments.py new file mode 100644 index 0000000..cd9c91b --- /dev/null +++ b/generate_lonely_segments.py @@ -0,0 +1,36 @@ +import pycocotools.coco +import argparse +import logging +import tqdm +import urllib.request +import os + +logger = logging.getLogger("coco") + +argParser = argparse.ArgumentParser(description='Find all images with single segments and generate svg') +argParser.add_argument( + '--annotations', + type=str, + default='../../datasets/COCO/annotations/instances_train2017.json' + ) +argParser.add_argument( + '--output', + type=str, + help='Output directory' + ) +args = argParser.parse_args() + + +logger.info(f"Load {args.annotations}") +coco = pycocotools.coco.COCO(args.annotations) + + +for img_id, annotations in tqdm.tqdm(coco.imgToAnns.items()): + if len(annotations) != 1: + continue + + annotation = annotations[0] # we have only one + category = coco.cats[annotation['category_id']]['name'] + fn = os.path.join(args.output, f"{category}_{img_id}.jpg") + if not os.path.exists(fn): + urllib.request.urlretrieve(coco.imgs[img_id]['coco_url'], fn) diff --git a/server.py b/server.py new file mode 100644 index 0000000..960823d --- /dev/null +++ b/server.py @@ -0,0 +1,125 @@ +import tornado.ioloop +import tornado.web +import tornado.websocket +import argparse +import logging +import coloredlogs +from coco.storage import COCOStorage +import json +from urllib.parse import urlparse + +logger = logging.getLogger('coco.server') + +class JsonEncoder(json.JSONEncoder): + def default(self, obj): + method = getattr(obj, "forJson", None) + if callable(method ): + return obj.forJson() + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + +class RestHandler(tornado.web.RequestHandler): + def initialize(self, storage: COCOStorage): + self.storage = storage + self.set_header("Content-Type", "application/json") + + def get(self, *params): + self.write(json.dumps(self.getData(*params), cls=JsonEncoder)) + +class CategoryHandler(RestHandler): + def getData(self): + return self.storage.getCategories() + +class AnnotationHandler(RestHandler): + def getData(self): + # get specific annotation + annotation_id = self.get_argument('id', None) + annotation_id = None if not annotation_id else int(annotation_id) + + # get by category id + category_id = self.get_argument('category', None) + category_id = None if not category_id else int(category_id) + + normalise = self.get_argument('normalise', False) + normalise = int(normalise) if normalise is not False else False + +# category_id = None if not category_id else int(category_id) + + logger.debug(f'Get annotation id: {annotation_id}, category: {category_id}, normalised: {normalise}') + + annotation = self.storage.getRandomAnnotation(annotation_id=annotation_id, category_id=category_id) + if normalise: + return annotation.getNormalised(normalise, normalise) + return annotation + + +class WebSocketHandler(tornado.websocket.WebSocketHandler): + CORS_ORIGINS = ['localhost', 'coco.local', 'r3.local'] + + def check_origin(self, origin): + parsed_origin = urlparse(origin) + # parsed_origin.netloc.lower() gives localhost:3333 + valid = parsed_origin.hostname in self.CORS_ORIGINS + return valid + + # the client connected + def open(self, p = None): + WebSocketHandler.connections.add(self) + logger.info("New client connected") + self.write_message("hello!") + + # the client sent the message + def on_message(self, message): + logger.debug(f"recieve: {message}") + + +class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler): + def set_extra_headers(self, path): + """For subclass to add extra headers to the response""" + if path[-5:] == '.html': + self.set_header("Access-Control-Allow-Origin", "*") + +def make_app(db_filename, debug): + storage = COCOStorage(db_filename) + + return tornado.web.Application([ + (r"/ws(.*)", WebSocketHandler), + (r"/categories.json", CategoryHandler, {'storage': storage}), + (r"/annotation.json", AnnotationHandler, {'storage': storage}), + (r"/(.*)", StaticFileWithHeaderHandler, + {"path": 'www', "default_filename": 'index.html'}), + ], debug=debug) + +if __name__ == "__main__": + argParser = argparse.ArgumentParser(description='Server for COCO web interface') + argParser.add_argument( + '--port', + '-P', + type=int, + default=8888, + help='Port to listen on' + ) + argParser.add_argument( + '--db', + type=str, + metavar='DATABASE', + required=True, + help='Database to serve from' + ) + argParser.add_argument( + '--verbose', + '-v', + action='store_true', + help='Increase log level' + ) + args = argParser.parse_args() + + loglevel = logging.DEBUG if args.verbose else logging.INFO + coloredlogs.install( + level=loglevel, + fmt="%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s" + ) + + app = make_app(args.db, debug=args.verbose ) + app.listen(args.port) + tornado.ioloop.IOLoop.current().start() \ No newline at end of file diff --git a/tools.py b/tools.py new file mode 100644 index 0000000..55e0f54 --- /dev/null +++ b/tools.py @@ -0,0 +1,106 @@ +import pycocotools.coco +import argparse +import logging +import os +import pprint +import sqlite3 + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("coco") + +argParser = argparse.ArgumentParser(description='Create shape SVG\'s') +argParser.add_argument( + '--annotations', + type=str, + default='../../datasets/COCO/annotations/instances_val2017.json' + ) +argParser.add_argument( + '--categories', + action='store_true', + help='Show categories' + ) +argParser.add_argument( + '--propagate', + type=str, + metavar='DATABASE', + help='Store data in sqlite db' + ) +args = argParser.parse_args() + + +logger.info(f"Load {args.annotations}") +coco = pycocotools.coco.COCO(args.annotations) + + +if args.categories: + cats = {} + for id, cat in coco.cats.items(): + if cat['supercategory'] not in cats: + cats[cat['supercategory']] = [] + cats[cat['supercategory']].append(cat) +# pp = pprint.PrettyPrinter(indent=4) + pprint.pprint(cats, sort_dicts=False) + +if args.propagate: + if not os.path.exists(args.propagate): + con = sqlite3.connect(args.propagate) + cur = con.cursor() + with open('coco.sql', 'r') as fp: + cur.executescript(fp.read()) + con.close() + + con = sqlite3.connect(args.propagate) + logger.info("Create categories") + cur = con.cursor() + cur.executemany('INSERT OR IGNORE INTO categories(id, supercategory, name) VALUES (:id, :supercategory, :name)', coco.cats.values()) + con.commit() + + logger.info("Images...") + cur.executemany(''' + INSERT OR IGNORE INTO images(id, flickr_url, coco_url, width, height, date_captured) + VALUES (:id, :flickr_url, :coco_url, :width, :height, :date_captured) + ''', coco.imgs.values()) + con.commit() + + logger.info("Annotations...") + + + def annotation_generator(): + for c in coco.anns.values(): + ann = c.copy() + ann['bbox_top'] = ann['bbox'][1] + ann['bbox_left'] = ann['bbox'][0] + ann['bbox_width'] = ann['bbox'][2] + ann['bbox_height'] = ann['bbox'][3] + yield ann + + cur.executemany(''' + INSERT OR IGNORE INTO annotations(id, image_id, category_id, iscrowd, area, bbox_top, bbox_left, bbox_width, bbox_height) + VALUES (:id, :image_id, :category_id, :iscrowd, :area, :bbox_top, :bbox_left, :bbox_width, :bbox_height) + ''', annotation_generator()) + con.commit() + + + logger.info("Segments...") + + def segment_generator(): + for ann in coco.anns.values(): + for i, seg in enumerate(ann['segmentation']): + yield { + 'id': ann['id']*10 + i, # create a uniqe segment id, supports max 10 segments per annotation + 'annotation_id': ann['id'], + 'points': str(seg)[1:-1], + } + + cur.executemany(''' + INSERT OR IGNORE INTO segments(id, annotation_id, points) + VALUES (:id, :annotation_id, :points) + ''', segment_generator()) + con.commit() + + + logger.info("Done...") + +# for id, cat in coco.cats.items(): +# cur = con.cursor() +# cur.execute \ No newline at end of file diff --git a/www/coco.js b/www/coco.js new file mode 100644 index 0000000..7e714c5 --- /dev/null +++ b/www/coco.js @@ -0,0 +1,234 @@ + +////////////////////////// +// use texture.js without d3 to create discerning patterns on the email blocks +///////////////////////////////////// + + +// some faked dom, see https://github.com/riccardoscalco/textures/issues/17 +function dom(name) { + this.name = name; + this.els = []; + this.attrs = {}; +} +dom.prototype.append = function(name) { + if (name === "defs") return this; + if (this.name === undefined) { + this.name = name; + return this; + } + var el = new dom(name); + this.els.push(el); + return el; +} +dom.prototype.attr = function(key, value) { + this.attrs[key] = value; + return this; +} +dom.prototype.toString = function() { + var attrs = []; + var k; + for (k in this.attrs) { + attrs += " " + k + "='" + this.attrs[k] + "'"; + } + if (this.els.length) { + return "<"+this.name+attrs+">"+this.els.map(function(el) { return el.toString()}).join('\n')+""; + } else { + return "<"+this.name+attrs+"/>" + } +} +//////////// end of faked dom /////////////////////////////// + +let categoryMap = {}; +let textureMap = {}; +function getColoredTexture(i, color) { + let j = i % 11; // update when adding items to switch: + switch(j) { + case 0: + return textures.lines().size(4).strokeWidth(1).stroke(color); + case 1: + return textures.circles().radius(2).size(5).fill('none').strokeWidth(1).stroke(color).complement(); + case 2: + return textures.lines().size(10).orientation("3/8").stroke(color); + case 3: + return textures.lines().heavier(4).thinner(.8).stroke(color); + case 4: + return textures.paths().d("hexagons").size(3).strokeWidth(1).stroke(color); + case 5: + return textures.lines().orientation("vertical", "horizontal").size(4).strokeWidth(1).shapeRendering("crispEdges").stroke(color); + case 6: + return textures.paths().d("hexagons").size(5).strokeWidth(2).stroke("rgba(0,0,0,0)").fill(color); + case 7: + return textures.circles().size(4).stroke(color); + case 8: + return textures.circles().thicker().complement().stroke(color); + case 9: + return textures.paths().d("caps").lighter().thicker().size(5).stroke(color); + case 10: + return textures.paths().d("hexagons").size(4).strokeWidth(2).stroke(color); +// textures.lines().size(4).strokeWidth(1).orientation("-3/8"), + } +}; + +const categoryColors = { + "person": "#f00", + "vehicle": "#0f0", + "outdoor": "#006", + "animal": "#ff0", + "food": "#0ff", + "furniture": "#f0f", + "indoor": "#fff", + "electronic": "#390", + "kitchen": "#930", + "accessory": "#f90", + "sports": "#f09", +} + +function getColorForSuperCategory(name) { + return categoryColors[name]; +} + +function getTextureForCategory(id) { + + let hash = id; + if(!textureMap.hasOwnProperty(hash)) { + let color = categoryColors[categoryMap[id]['supercategory']]; + textureMap[hash] = getColoredTexture(id, color); + } + + return textureMap[hash]; +} + + + +class CocoCanvas { + start(){ + this.catNavEl = document.getElementById('catNav'); + this.canvas = document.getElementById('svgCanvas'); + this.loadNav() + } + loadNav() { + let r = new Request('/categories.json'); + fetch(r) + .then(response => response.json()) + .then(categories => { + for(let cat of categories) { + categoryMap[cat['id']] = cat; + } + this.buildNav(categories); + }).catch(function(e){ + console.error(e); + }); + } + + buildNav(categories) { + let ulEl = crel('ul'); + for(let cat of categories) { + ulEl.appendChild( + crel('li', { + 'id': 'category-' + cat['id'], + 'on': { + 'click': (e) => { + this.requestAnnotation(cat['id']); + } + } + }, cat['name']) + ); + } + this.catNavEl.appendChild(ulEl); + + let defsEl = document.createElementNS("http://www.w3.org/2000/svg", 'defs'); + for(let cat of categories) { + let texture = getTextureForCategory(cat['id']); + let sel = new dom(); + texture(sel); + defsEl.innerHTML += sel.toString(); + } + this.canvas.appendChild(defsEl); + } + + requestAnnotation(category_id) { + let r = new Request(`/annotation.json?category=${category_id}&normalise=100`); + fetch(r) + .then(response => response.json()) + .then(annotation => { + this.addAnnotationAsShape(annotation); + }).catch(function(e){ + console.error(e); + });; + } + + pointsToD(points) { + let start = points.shift() + let d = `M${start[0].toPrecision(4)} ${start[1].toPrecision(4)} L `; + points = points.map((p) => `${p[0].toPrecision(4)} ${p[1].toPrecision(4)}`); + d += points.join(' '); + return d; + } + + getMousePosition(evt) { + // from http://www.petercollingridge.co.uk/tutorials/svg/interactive/dragging/ + let CTM = this.canvas.getScreenCTM(); + return { + x: (evt.clientX - CTM.e) / CTM.a, + y: (evt.clientY - CTM.f) / CTM.d + }; + } + + addAnnotationAsShape(annotation) { + console.log('Add annotation', annotation); + + let category = categoryMap[annotation['category_id']] + let texture = getTextureForCategory(category['id']); + + let x = 500 - annotation['bbox'][2]/2; + let y = 500 - annotation['bbox'][3]/2; + let annEl = crel(document.createElementNS("http://www.w3.org/2000/svg", 'g'), { + 'data-id': annotation['id'], + 'transform': `translate(${x}, ${y})`, + 'on': { + 'mousedown': function(downE) { + console.log(downE); + console.log(this) + let offset = this.getMousePosition(downE); +// offset.x -= parseFloat(downE.target.getAttributeNS(null, "x")); +// offset.y -= parseFloat(downE.target.getAttributeNS(null, "y")); + + // Get initial translation amount + console.log(annEl.transform.baseVal); + let transform = annEl.transform.baseVal.getItem(0); + offset.x -= transform.matrix.e; + offset.y -= transform.matrix.f; + + let moveEvent = (moveE) => { + let coord = this.getMousePosition(moveE); + transform.matrix.e = coord.x - offset.x; + transform.matrix.f = coord.y - offset.y; +// annEl.setAttributeNS(null, "x", coord.x - offset.x); +// annEl.setAttributeNS(null, "y", coord.y - offset.y); + }; + + + document.addEventListener('mousemove', moveEvent); + document.addEventListener('mouseup', (upE) => { + document.removeEventListener('mousemove', moveEvent); + }); + + }.bind(this) + } + }); + + for(let segment of annotation['segments']) { + + let pathEl = crel(document.createElementNS("http://www.w3.org/2000/svg", 'path'), { + 'fill': texture.url(), + 'd': this.pointsToD(segment) + }); + annEl.appendChild(pathEl); + } + console.log(annEl); + this.canvas.appendChild(annEl); + } +} + +let cc = new CocoCanvas(); +cc.start(); diff --git a/www/coco_example.js b/www/coco_example.js new file mode 100644 index 0000000..9400caa --- /dev/null +++ b/www/coco_example.js @@ -0,0 +1,127 @@ + +////////////////////////// +// use texture.js without d3 to create discerning patterns on the email blocks +///////////////////////////////////// + + +// some faked dom, see https://github.com/riccardoscalco/textures/issues/17 +function dom(name) { + this.name = name; + this.els = []; + this.attrs = {}; +} +dom.prototype.append = function(name) { + if (name === "defs") return this; + if (this.name === undefined) { + this.name = name; + return this; + } + var el = new dom(name); + this.els.push(el); + return el; +} +dom.prototype.attr = function(key, value) { + this.attrs[key] = value; + return this; +} +dom.prototype.toString = function() { + var attrs = []; + var k; + for (k in this.attrs) { + attrs += " " + k + "='" + this.attrs[k] + "'"; + } + if (this.els.length) { + return "<"+this.name+attrs+">"+this.els.map(function(el) { return el.toString()}).join('\n')+""; + } else { + return "<"+this.name+attrs+"/>" + } +} +//////////// end of faked dom /////////////////////////////// + +let textureMap = {}; +function getColoredTexture(i, color) { + let j = i % 11; // update when adding items to switch: + switch(j) { + case 0: + return textures.lines().size(4).strokeWidth(1).stroke(color); + case 1: + return textures.circles().radius(2).size(10).fill('none').strokeWidth(1).stroke(color).complement(); + case 2: + return textures.lines().size(10).orientation("3/8").stroke(color); + case 3: + return textures.lines().heavier(4).thinner(.8).stroke(color); + case 4: + return textures.paths().d("hexagons").size(7).strokeWidth(1).stroke(color); + case 5: + return textures.lines().orientation("vertical", "horizontal").size(4).strokeWidth(1).shapeRendering("crispEdges").stroke(color); + case 6: + return textures.paths().d("hexagons").size(5).strokeWidth(2).stroke("rgba(0,0,0,0)").fill(color); + case 7: + return textures.circles().size(4).stroke(color); + case 8: + return textures.circles().thicker().complement().stroke(color); + case 9: + return textures.paths().d("caps").lighter().thicker().stroke(color); + case 10: + return textures.paths().d("hexagons").size(4).strokeWidth(2).stroke(color); +// textures.lines().size(4).strokeWidth(1).orientation("-3/8"), + } +}; + +const categoryColors = { + "person": "#f00", + "vehicle": "#0f0", + "outdoor": "#006", + "animal": "#ff0", + "food": "#0ff", + "furniture": "#f0f", + "indoor": "#fff", + "electronic": "#390", + "kitchen": "#930", + "accessory": "#f90", + "sports": "#f09", +} + +function getColorForSuperCategory(name) { + return categoryColors[name]; +} + +function getTextureForCategory(name, color) { + let hash = name+'-'+color; + if(!textureMap.hasOwnProperty(hash)) { + let i = Object.keys(textureMap).length; + textureMap[hash] = getColoredTexture(i, color); + } + + return textureMap[hash]; +} + + +// turn HTMLCollection into Array, to prevent dynamic updating of the collection during loop +pathEls = Array.from(document.getElementsByTagName('path')); +for(pathEl of pathEls){ + let defsEls = pathEl.parentNode.getElementsByTagName('defs'); + let defsEl; + if(defsEls.length < 1) { + defsEl = document.createElement("DEFS"); + pathEl.parentNode.appendChild(defsEl); + } else { + defsEl = defsEls[0]; + } + + console.log(pathEl,pathEl.classList); + if(pathEl.classList.length != 2) continue; + let super_name = pathEl.classList.item(0).substr(6); + let cat_name = pathEl.classList.item(1).substr(4); + let color = getColorForSuperCategory(super_name) + let texture = getTextureForCategory(cat_name, color); + + if(!defsEl.parentNode.getElementById(texture.id())) { + let sel = new dom(); + texture(sel); + console.log(sel, sel.toString()) + defsEl.innerHTML += sel.toString(); + } + + pathEl.style.fill = texture.url(); +} \ No newline at end of file diff --git a/www/crel.min.js b/www/crel.min.js new file mode 100644 index 0000000..ee1eeb4 --- /dev/null +++ b/www/crel.min.js @@ -0,0 +1,47 @@ +!function(n,e){"object"==typeof exports?module.exports=e():"function"==typeof define&&define.amd?define(e):n.crel=e()}(this,function(){function n(a){var d,s=arguments,p=s[1],y=2,m=s.length,x=n[o];if(a=n[u](a)?a:c.createElement(a),m>1){if((!e(p,r)||n[f](p)||Array.isArray(p))&&(--y,p=null),m-y==1&&e(s[y],"string"))a.textContent=s[y];else for(;y + + + + + + + + + + + + \ No newline at end of file diff --git a/www/index.html b/www/index.html new file mode 100644 index 0000000..76650a4 --- /dev/null +++ b/www/index.html @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/www/svg-inject.min.js b/www/svg-inject.min.js new file mode 100644 index 0000000..1631485 --- /dev/null +++ b/www/svg-inject.min.js @@ -0,0 +1,10 @@ +!function(o,l){var r,a,s="createElement",g="getElementsByTagName",b="length",E="style",d="title",y="undefined",k="setAttribute",w="getAttribute",x=null,A="__svgInject",C="--inject-",S=new RegExp(C+"\\d+","g"),I="LOAD_FAIL",t="SVG_NOT_SUPPORTED",L="SVG_INVALID",v=["src","alt","onload","onerror"],j=l[s]("a"),G=typeof SVGRect!=y,f={useCache:!0,copyAttributes:!0,makeIdsUnique:!0},N={clipPath:["clip-path"],"color-profile":x,cursor:x,filter:x,linearGradient:["fill","stroke"],marker:["marker", +"marker-end","marker-mid","marker-start"],mask:x,pattern:["fill","stroke"],radialGradient:["fill","stroke"]},u=1,c=2,O=1;function T(e){return(r=r||new XMLSerializer).serializeToString(e)}function P(e,r){var t,n,i,o,a=C+O++,f=/url\("?#([a-zA-Z][\w:.-]*)"?\)/g,u=e.querySelectorAll("[id]"),c=r?[]:x,l={},s=[],d=!1;if(u[b]){for(i=0;i