From 7d11274be8149ef0024d72c30e4379a740dfaa3f Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Wed, 4 Mar 2020 22:24:49 +0100 Subject: [PATCH] Version with save option --- README.md | 30 ++++ coco.sql | 1 + plottingdata_coco.service | 9 ++ requirements.txt | 5 + server.py | 110 +++++++++++--- tools.py | 243 +++++++++++++++++++++++++++--- www/canvas.svg | 309 ++++++++++++++++++++++++++++++++++++++ www/canvas_patterns.json | 82 ++++++++++ www/coco.css | 155 +++++++++++++++++++ www/coco.js | 244 +++++++++++++++++++++++------- www/index.html | 52 ++----- www/saved.html | 40 +++++ www/saved/.gitignore | 2 + 13 files changed, 1150 insertions(+), 132 deletions(-) create mode 100644 plottingdata_coco.service create mode 100644 requirements.txt create mode 100644 www/canvas.svg create mode 100644 www/canvas_patterns.json create mode 100644 www/coco.css create mode 100644 www/saved.html create mode 100644 www/saved/.gitignore diff --git a/README.md b/README.md index 28792c0..e153615 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ +server.py +: Server for the web interface + +create_shapes.py +: Create an svg file per image, with classes on the shapes according to their classes + +zoom_animation.py +: Create svg frames for a category. Ordered by the area of the shapes. + +generate_lonely_segments.py +: Find and download the images with only one object in them. + +tools.py +: Turn a COCO json file (eg `instances_val2017.json`) into a database format (eg `coco_train.db`) + + +--- Build array of images sorted by size: @@ -20,3 +37,16 @@ cd ../dog_png ffmpeg -f image2 -pattern_type glob -i '*.png' ../dog.mp4 ``` + +# To run as server: +```bash +cp plottingdata_coco.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable plottingdata_coco.service +systemctl start plottingdata_coco.service +``` + + +``` +rsync . --exclude zoom --exclude venv --exclude archive -av here.rubenvandeven.com:/home/ruben/coco/ --exclude shapes --exclude lonely --exclude .git --exclude __pycache__ --info progress2 +``` diff --git a/coco.sql b/coco.sql index 841127f..affd752 100644 --- a/coco.sql +++ b/coco.sql @@ -30,6 +30,7 @@ CREATE TABLE IF NOT EXISTS "annotations" ( "bbox_left" FLOAT, "bbox_width" FLOAT, "bbox_height" FLOAT, + "zerkine_moments" TEXT DEFAULT NULL, PRIMARY KEY("id") ) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS "segments_annotation" ON "segments" ( diff --git a/plottingdata_coco.service b/plottingdata_coco.service new file mode 100644 index 0000000..41341be --- /dev/null +++ b/plottingdata_coco.service @@ -0,0 +1,9 @@ +[Unit] +Description=PlottingData COCO interface +[Service] +ExecStart=/home/ruben/coco/venv/bin/python /home/ruben/coco/server.py --db /home/ruben/coco/coco_train.db --port 8080 +WorkingDirectory=/home/ruben/coco +User=www-data +Restart=on-failure +[Install] +WantedBy=multi-user.target diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a41560c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +tornado +coloredlogs +pycocotools +numpy +mahotas diff --git a/server.py b/server.py index 960823d..c715b49 100644 --- a/server.py +++ b/server.py @@ -7,6 +7,9 @@ import coloredlogs from coco.storage import COCOStorage import json from urllib.parse import urlparse +import uuid +import os +import glob logger = logging.getLogger('coco.server') @@ -25,6 +28,9 @@ class RestHandler(tornado.web.RequestHandler): def get(self, *params): self.write(json.dumps(self.getData(*params), cls=JsonEncoder)) + + def post(self, *params): + self.write(json.dumps(self.getData(*params), cls=JsonEncoder)) class CategoryHandler(RestHandler): def getData(self): @@ -51,28 +57,93 @@ class AnnotationHandler(RestHandler): if normalise: return annotation.getNormalised(normalise, normalise) return annotation - - -class WebSocketHandler(tornado.websocket.WebSocketHandler): - CORS_ORIGINS = ['localhost', 'coco.local', 'r3.local'] - - def check_origin(self, origin): - parsed_origin = urlparse(origin) - # parsed_origin.netloc.lower() gives localhost:3333 - valid = parsed_origin.hostname in self.CORS_ORIGINS - return valid - # the client connected - def open(self, p = None): - WebSocketHandler.connections.add(self) - logger.info("New client connected") - self.write_message("hello!") - # the client sent the message - def on_message(self, message): - logger.debug(f"recieve: {message}") +class SaveHandler(RestHandler): + def getData(self): + """ + Save an SVG. Regenerate it on the server to prevent any maliscious input + """ + req = tornado.escape.json_decode(self.request.body) + scene = int(req['scene']) + annotations = [] + + with open('www/canvas_patterns.json') as fp: + patterns = json.load(fp) + svgGs = [] + + for annotation in req['annotations'][:100]: # max 200 annotations + annId = int(annotation['id']) + ann = self.storage.getAnnotationById(annId) + normalisedAnn = ann.getNormalised(100,100) + x = float(annotation['x']) + y = float(annotation['y']) + fill = patterns[str(ann.category_id)] + segments = [] + textX = normalisedAnn.bbox[2]+5 + textY = normalisedAnn.bbox[3] + + cat = self.storage.getCategory(ann.category_id) + image = self.storage.getImage(ann.image_id) + + for segment in normalisedAnn.segments: + d = segment.getD() + segments.append(f"""""") + svgGs.append(f""" + + + {"".join(segments)} + {cat['name']} + + """) + annotations.append({'id': annId, 'x': x, 'y': y}) + source = json.dumps({ + 'scene': scene, + 'annotations': annotations + }) + + with open('www/canvas.svg') as fp: + svgContent = fp.read() + svgContent = svgContent.replace('{source}', json.dumps(source))\ + .replace('', "".join(svgGs)+"") + + + saveId = uuid.uuid4().hex + '.svg' + filename = os.path.join('www/saved', saveId) + with open(filename, 'w') as fp: + fp.write(svgContent) + return {'submission':'/saved/'+saveId} + + + +class SavedHandler(tornado.web.RequestHandler): + def initialize(self, storage: COCOStorage): + self.storage = storage + + def get(self): + images = [] + files = glob.glob("www/saved/*.svg") + files.sort(key=lambda f: -1 * os.path.getmtime(f)) + + for filename in files[:100]: + with open(filename, 'r') as fp: + # remove first XML line: + contents = '\n'.join(fp.read().split('\n')[1:]) + images.append(contents) + + with open("www/saved.html") as fp: + template = fp.read() + + template = template.replace("{images}", ''.join(images)) + self.write(template) + + class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler): def set_extra_headers(self, path): """For subclass to add extra headers to the response""" @@ -83,9 +154,10 @@ def make_app(db_filename, debug): storage = COCOStorage(db_filename) return tornado.web.Application([ - (r"/ws(.*)", WebSocketHandler), (r"/categories.json", CategoryHandler, {'storage': storage}), (r"/annotation.json", AnnotationHandler, {'storage': storage}), + (r"/save", SaveHandler, {'storage': storage}), + (r"/saved", SavedHandler, {'storage': storage}), (r"/(.*)", StaticFileWithHeaderHandler, {"path": 'www', "default_filename": 'index.html'}), ], debug=debug) diff --git a/tools.py b/tools.py index 55e0f54..9e37f6a 100644 --- a/tools.py +++ b/tools.py @@ -4,6 +4,16 @@ import logging import os import pprint import sqlite3 +from coco.storage import COCOStorage, Annotation, Segment +import cv2 +import mahotas +import subprocess +import tqdm +import numpy as np +import ast +import svgwrite +from svgwrite.extensions import Inkscape +from xml.etree import ElementTree logging.basicConfig(level=logging.INFO) logger = logging.getLogger("coco") @@ -20,10 +30,34 @@ argParser.add_argument( help='Show categories' ) argParser.add_argument( - '--propagate', + '--db', type=str, metavar='DATABASE', - help='Store data in sqlite db' + help='SQLite db filename, will be created if not existing' + ) +argParser.add_argument( + '--propagate', + action='store_true', + help='Store annotation data in sqlite db' + ) +argParser.add_argument( + '--zerkine', + action='store_true', + help='Find and store annotation Zerkine moments for those that do not have it yet' + ) +argParser.add_argument( + '--similar', + type=int, + metavar="ANNOTATION_ID", + help='Find similar shapes for annotation' + ) +argParser.add_argument( + '--stickers', + type=str, + metavar="SVG_FILENAME", + help=""" + Create an SVG with sticker pages (afterwards convert to EPS: \"for f in *; do echo $f; inkscape -f $f --export-eps $f.eps; done\") + """ ) args = argParser.parse_args() @@ -40,31 +74,28 @@ if args.categories: cats[cat['supercategory']].append(cat) # pp = pprint.PrettyPrinter(indent=4) pprint.pprint(cats, sort_dicts=False) - + +storage = None +if args.db: + storage = COCOStorage(args.db) + con = storage.con + if args.propagate: - if not os.path.exists(args.propagate): - con = sqlite3.connect(args.propagate) - cur = con.cursor() - with open('coco.sql', 'r') as fp: - cur.executescript(fp.read()) - con.close() - - con = sqlite3.connect(args.propagate) logger.info("Create categories") cur = con.cursor() cur.executemany('INSERT OR IGNORE INTO categories(id, supercategory, name) VALUES (:id, :supercategory, :name)', coco.cats.values()) con.commit() - + logger.info("Images...") cur.executemany(''' INSERT OR IGNORE INTO images(id, flickr_url, coco_url, width, height, date_captured) VALUES (:id, :flickr_url, :coco_url, :width, :height, :date_captured) ''', coco.imgs.values()) con.commit() - + logger.info("Annotations...") - - + + def annotation_generator(): for c in coco.anns.values(): ann = c.copy() @@ -73,16 +104,16 @@ if args.propagate: ann['bbox_width'] = ann['bbox'][2] ann['bbox_height'] = ann['bbox'][3] yield ann - + cur.executemany(''' INSERT OR IGNORE INTO annotations(id, image_id, category_id, iscrowd, area, bbox_top, bbox_left, bbox_width, bbox_height) VALUES (:id, :image_id, :category_id, :iscrowd, :area, :bbox_top, :bbox_left, :bbox_width, :bbox_height) ''', annotation_generator()) con.commit() - - + + logger.info("Segments...") - + def segment_generator(): for ann in coco.anns.values(): for i, seg in enumerate(ann['segmentation']): @@ -91,16 +122,178 @@ if args.propagate: 'annotation_id': ann['id'], 'points': str(seg)[1:-1], } - + cur.executemany(''' INSERT OR IGNORE INTO segments(id, annotation_id, points) VALUES (:id, :annotation_id, :points) ''', segment_generator()) con.commit() - - + + logger.info("Done...") + +if args.zerkine: + nr = storage.countAnnotationsWithoutZerkine() + for i in tqdm.tqdm(range(nr)): + annotation = storage.getAnnotationWithoutZerkine() + normAnn = annotation.getNormalised(100, 100) + filenameRoot = '/tmp/tmp_ann_to_convert' + dwg = normAnn.asSvg(filenameRoot + '.svg', square=True, bg='black') + dwg.save() + # convert to rasterised + subprocess.call([ + 'inkscape', + '-f', filenameRoot + '.svg', + '-e', filenameRoot + '.png', + ], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + # read with opencv + image = cv2.imread(filenameRoot + '.png', cv2.IMREAD_GRAYSCALE) + moments = mahotas.features.zernike_moments(image, 21) + storage.storeZerkineForAnnotation(annotation, moments, delayCommit = True) + if not i % 100: + storage.con.commit() + storage.con.commit() + +if args.similar: + # todo find similar + annotation = storage.getAnnotationById(args.similar, withZerkine=True) + dwg = annotation.asSvg(f'tmp/source.svg', square=True) + dwg.save() -# for id, cat in coco.cats.items(): -# cur = con.cursor() -# cur.execute \ No newline at end of file + shapeA = np.array(annotation.segments[0].points) + + annMoments = np.array(annotation.zerkine_moment) + distances = [] + print(annotation) +# Stupid, this seems to have been superfluous +# zerkines = storage.getZerkines() +# for zerkine in tqdm.tqdm(zerkines): +# if annotation.id == zerkine['id']: +# continue +# +# diff = annMoments - np.array(Annotation.parseZerkineFromDB(zerkine['zerkine_moment'])) +# distance = np.linalg.norm(diff) +# distances.append((zerkine['id'], distance)) + anns = storage.getAllAnnotationPoints() + for ann in tqdm.tqdm(anns): + try: + shapeB = np.array(Segment.asCoordinates(ast.literal_eval('['+ann['points']+']'))) + # fourth param is require, but according to docs does nothing + distance= cv2.matchShapes(shapeA, shapeB, cv2.CONTOURS_MATCH_I2, 1) + distances.append((ann['id'], distance)) + except Exception as e: + logger.critical(f"Exception comparing {annotation.id} to {ann['id']}, points: {ann['points']}") + logger.exception(e) + + distances = sorted(distances, key=lambda d: d[1]) + + for i in range(10): + similarAnnotation = storage.getAnnotationById(distances[i][0]) + print(coco.cats[similarAnnotation.category_id]) + dwg = similarAnnotation.asSvg(f'tmp/result_{i}.svg', square=True) + dwg.save() + +if args.stickers: + grid = (3, 4) # items in the grid x,y + size = (105, 148) # in mm + sizeFactor = 5 # influences the size of the patterns + viewBoxSize = (size[0] * sizeFactor, size[1] * sizeFactor) + margin = 5 + gridSize = ( + int((viewBoxSize[0]-((grid[0]+1)*margin))/grid[0]), + int((viewBoxSize[1]-((grid[1]+1)*margin))/grid[1]) + ) + + # see also textures.xml + textureIds = ["#siqwx","#wjnbs","#pnfez","#ejtxy","#obabs","#hehoj","#mrwjs","#ryjbw","#rkkau","#vbjcl","#zzehx","#mumke","#brhhk","#gujvh","#hfgqa","#lrbsh","#bndby","#bfnxk","#ydler","#pnxdr","#htqlj","#nunnt","#tidaw","#tcdum","#kwwja","#hgdkl","#nvkwz","#uzdqb","#fgshk","#vknil","#yeenr","#mslkw","#eibaw","#meama","#akuvz","#khkpp","#ibnow","#wivvx","#svksy","#xhmew","#jmiqu","#gfcer","#iueil","#iufvt","#ugkud","#dchzd","#nejks","#dqseb","#yhrwm","#bmiet","#qovkk","#hxoiq","#jfguh","#kbpkl","#ikarj","#nucap","#qfsqn","#bboqt","#pxkjn","#lbnx","#nxkmp","#snojb","#oioil","#hvldz","#qpscp","#oborh","#crobu","#ydhwn","#geanf","#sdfeo","#cgtma","#rjfrc","#uhcys","#lrgem","#osiho","#etssd","#esxcs","#hczhr","#nnhxw","#wrlbu"] + + nr = 0 + total_nr = len(coco.cats) + for category_id, cat in coco.cats.items(): + nr+=1 + filename = os.path.join( + args.stickers, + f"{category_id}_{cat['supercategory']}_{cat['name']}.svg") + dwg = svgwrite.Drawing( + filename, + size=(f'{size[0]}mm', f'{size[1]}mm'), + viewBox=f"0 0 {viewBoxSize[0]} {viewBoxSize[1]}" + ) + + annotations = storage.getRandomAnnotations( + limit = grid[0]*grid[1], + category_id = category_id + ) + + inkscape = Inkscape(dwg) + contourG = inkscape.layer(label='Snijlijnen') + drawingG = inkscape.layer(label='Shapes') + + # dwg.add(svgwrite.container.Defs()) + dwg.add(drawingG) + dwg.add(contourG) + + + font_size = 10 + text = dwg.text( + f"{nr:02d}/{total_nr}", + insert=(margin, margin+font_size), font_size=font_size, fill='black' + ) + drawingG.add(text) + + text = dwg.text( + f"{category_id}. {cat['supercategory']} - {cat['name']}", + insert=(viewBoxSize[0]-margin, margin+font_size), font_size=font_size, fill='black', + style='text-anchor:end;') + drawingG.add(text) + + text = dwg.text( + f"Common Objects In Context", + insert=(margin, viewBoxSize[1]-margin), font_size=font_size, fill='black', + ) + drawingG.add(text) + + text = dwg.text( + f"Plotting Data", + insert=(viewBoxSize[0]-margin, viewBoxSize[1]-margin), font_size=font_size, fill='black', + style='text-anchor:end;') + drawingG.add(text) + + for i, annotation in enumerate(annotations): + normAnn = annotation.getNormalised(gridSize[0], gridSize[1]) + translation = normAnn.getTranslationToCenter() + # print(translation) + + pX = i%grid[0] + pY = int(i/grid[0]) + posX = pX*gridSize[0] + (pX+1)*margin - translation[0] + posY = pY*gridSize[1] + (pY+1)*margin - translation[1] + # print(i, posX, posY, gridSize) + + + positionG = svgwrite.container.Group(transform=f'translate({posX}, {posY})') + normAnn.writeToDrawing(positionG, stroke='#2FEE2F', stroke_width='1pt', fill_opacity="0") + contourG.add(positionG) + + position2G = svgwrite.container.Group(transform=f'translate({posX}, {posY})') + pattern_id = textureIds[category_id % len(textureIds)] + normAnn.writeToDrawing(position2G, fill=f'url({pattern_id})', stroke='blue', stroke_width='0') + drawingG.add(position2G) + + + + xml = dwg.get_xml() + with open('textures.xml', 'r') as fp: + textureTree = ElementTree.fromstring(fp.read()) + defsTree = xml.find('defs') + for pattern in textureTree: + defsTree.append(pattern) + xmlString = ElementTree.tostring(xml) + with open(filename, 'wb') as fp: + # print(xmlString) + fp.write(xmlString) + logger.info(f"Wrote to {filename}") + + \ No newline at end of file diff --git a/www/canvas.svg b/www/canvas.svg new file mode 100644 index 0000000..77c1dda --- /dev/null +++ b/www/canvas.svg @@ -0,0 +1,309 @@ + + + {source} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/www/canvas_patterns.json b/www/canvas_patterns.json new file mode 100644 index 0000000..e9700e3 --- /dev/null +++ b/www/canvas_patterns.json @@ -0,0 +1,82 @@ +{ +"1": "url(#tprke)", +"2": "url(#szylj)", +"3": "url(#ooumd)", +"4": "url(#fheyp)", +"5": "url(#ssscs)", +"6": "url(#htafw)", +"7": "url(#mwmpe)", +"8": "url(#ipzlm)", +"9": "url(#ryfgx)", +"10": "url(#gktcz)", +"11": "url(#nkuai)", +"13": "url(#tnder)", +"14": "url(#hfgks)", +"15": "url(#mrlhp)", +"16": "url(#cmsda)", +"17": "url(#wsewh)", +"18": "url(#htoli)", +"19": "url(#aoggk)", +"20": "url(#mfcrf)", +"21": "url(#igtrq)", +"22": "url(#wgyom)", +"23": "url(#qxsnm)", +"24": "url(#gmgmx)", +"25": "url(#qubbv)", +"27": "url(#ttene)", +"28": "url(#fazqm)", +"31": "url(#nfcwz)", +"32": "url(#rwmct)", +"33": "url(#iiitd)", +"34": "url(#eajyr)", +"35": "url(#aknuz)", +"36": "url(#vhjmw)", +"37": "url(#mrist)", +"38": "url(#mbbhg)", +"39": "url(#hfuqj)", +"40": "url(#jrwrh)", +"41": "url(#bncvb)", +"42": "url(#upgob)", +"43": "url(#krnlo)", +"44": "url(#sgkgd)", +"46": "url(#mfhbm)", +"47": "url(#skgis)", +"48": "url(#oswbs)", +"49": "url(#lsouj)", +"50": "url(#bvkgt)", +"51": "url(#fgivr)", +"52": "url(#yfcyp)", +"53": "url(#gmbhz)", +"54": "url(#yfydn)", +"55": "url(#gxqls)", +"56": "url(#cfbwj)", +"57": "url(#fsgnv)", +"58": "url(#vakgp)", +"59": "url(#oaprx)", +"60": "url(#fxicb)", +"61": "url(#hjala)", +"62": "url(#xjxrr)", +"63": "url(#yzmaq)", +"64": "url(#dguls)", +"65": "url(#hltax)", +"67": "url(#czaje)", +"70": "url(#julqj)", +"72": "url(#gic)", +"73": "url(#ohqxb)", +"74": "url(#bvwec)", +"75": "url(#jdlwh)", +"76": "url(#wwcdm)", +"77": "url(#wcyvp)", +"78": "url(#ljnqw)", +"79": "url(#xcdur)", +"80": "url(#nxkhz)", +"81": "url(#cpnap)", +"82": "url(#lhupq)", +"84": "url(#aaqtb)", +"85": "url(#lufrm)", +"86": "url(#azwdr)", +"87": "url(#yffat)", +"88": "url(#hslpy)", +"89": "url(#zsvlo)", +"90": "url(#muhub)" +} \ No newline at end of file diff --git a/www/coco.css b/www/coco.css new file mode 100644 index 0000000..7ecc364 --- /dev/null +++ b/www/coco.css @@ -0,0 +1,155 @@ +body{ + font-family:sans-serif; + background: darkblue; + margin:0; +} + +#svgCanvas{ + width:100vw; + height:100vh; +} +.catNav{ + position:absolute; + top:0; + color:white; + bottom:0; + overflow-y: auto; + padding: 10px; +} +#catNav { + left: 0; +} +#catNav2 { + right:0; +} + +.catNav ul{ + list-style:none; + padding:0px; + margin: 30px 0px; + border-width: 3px; +} +#catNav ul { + border-left-style: solid; +} +#catNav2 ul { + border-right-style: solid; +} + +.catNav li{ + padding:2px; +} +.catNav li:hover{ + cursor:pointer; + text-decoration:underline; +} +#catNav li{ + transform: rotate(-30deg); + transform-origin: top left; + padding-left: 5px; +} +#catNav2 li{ + transform: rotate(30deg); + text-align:right; + transform-origin: top right; + padding-right: 5px; +} + +g{ + cursor:grab; +} + +g:active{ + cursor:grabbing; +} + +svg text{ + fill:white; + font-size: 40pt; + + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.buttons{ + position:absolute; + top: 10px; + text-align:center; + width: 300px; + left: calc(50% - 150px); +} +.buttons.buttons-inline{ + position:static; + margin: 10px; + width:100%; +} +.buttons span, .buttons label{ + color:white; + cursor:pointer; +} +.buttons span:hover{ + text-decoration:underline; +} +.buttons #sceneLabel{ + display:block; +} + +body.hideImages svg image{ + display:none; +} +body.hideLabels svg text{ + display:none; +} + +.supercategory-person{ + border-color:rgb(250,8,0); +} +.supercategory-vehicle{ + border-color:rgb(128,255,0); +} +.supercategory-outdoor{ + border-color:rgb(0,255,255); +} +.supercategory-animal{ + border-color:rgb(255,0,255); +} +.supercategory-accessory{ + border-color:rgb(255,255,0); +} +.supercategory-sports{ + border-color:rgb(0,128,255); +} +.supercategory-kitchen{ + border-color:rgb(250,133,0); +} +.supercategory-furniture{ + border-color:rgb(187,0,250); +} +.supercategory-food{ + border-color:rgb(0,250,133); +} +.supercategory-electronic{ + border-color:rgb(250,217,0); +} +.supercategory-appliance{ + border-color:rgb(255,0,128); +} +.supercategory-indoor{ + border-color:rgb(133,0,250); +} + +#save{margin: 5px;} + +#saved{ + text-align: center; +} +#saved svg{ + width: 30vw; + height: 20vw; + margin: 10px; + border:solid 1px white; +} \ No newline at end of file diff --git a/www/coco.js b/www/coco.js index 7e714c5..9f90e5a 100644 --- a/www/coco.js +++ b/www/coco.js @@ -58,9 +58,9 @@ function getColoredTexture(i, color) { case 6: return textures.paths().d("hexagons").size(5).strokeWidth(2).stroke("rgba(0,0,0,0)").fill(color); case 7: - return textures.circles().size(4).stroke(color); + return textures.circles().size(4).fill(color); case 8: - return textures.circles().thicker().complement().stroke(color); + return textures.circles().thicker().complement().fill(color); case 9: return textures.paths().d("caps").lighter().thicker().size(5).stroke(color); case 10: @@ -70,17 +70,18 @@ function getColoredTexture(i, color) { }; const categoryColors = { - "person": "#f00", - "vehicle": "#0f0", - "outdoor": "#006", - "animal": "#ff0", - "food": "#0ff", - "furniture": "#f0f", - "indoor": "#fff", - "electronic": "#390", - "kitchen": "#930", - "accessory": "#f90", - "sports": "#f09", + "person": "#FA0800", + "vehicle": "#80FF00", + "outdoor": "#00FFFF", + "animal": "#FF00FF", + "accessory": "#FFFF00", + "food": "#00FA85", + "furniture": "#BB00FA", + "indoor": "#8500FA", + "electronic": "#FAD900", + "kitchen": "#FA8500", + "sports": "#0080FF", + "appliance": "#FF0080", } function getColorForSuperCategory(name) { @@ -88,13 +89,13 @@ function getColorForSuperCategory(name) { } function getTextureForCategory(id) { - + let hash = id; if(!textureMap.hasOwnProperty(hash)) { let color = categoryColors[categoryMap[id]['supercategory']]; textureMap[hash] = getColoredTexture(id, color); } - + return textureMap[hash]; } @@ -103,8 +104,21 @@ function getTextureForCategory(id) { class CocoCanvas { start(){ this.catNavEl = document.getElementById('catNav'); + this.catNav2El = document.getElementById('catNav2'); this.canvas = document.getElementById('svgCanvas'); + this.loadImagesBtnEl = document.getElementById("loadImages"); + this.loadLabelsBtnEl = document.getElementById("loadLabels"); + this.sceneSelectEl = document.getElementById('scene'); + this.savedBtnEl = document.getElementById('save'); this.loadNav() + this.annotations = []; + + this.loadImagesBtnEl.addEventListener('change', (e) => this.toggleImages(e)); + this.loadLabelsBtnEl.addEventListener('change', (e) => this.toggleLabels(e)); + this.savedBtnEl.addEventListener('click', function(e){ + if(this.savedBtnEl.disabled || this.annotations.length < 1) return; + this.save(); + }.bind(this)); } loadNav() { let r = new Request('/categories.json'); @@ -119,44 +133,67 @@ class CocoCanvas { console.error(e); }); } - + buildNav(categories) { - let ulEl = crel('ul'); + let lastSuperCat = null; + let supercategories = [] + let supercategoriesEls = {} + // create menu's per supercategory, divide these over left & right for(let cat of categories) { - ulEl.appendChild( - crel('li', { - 'id': 'category-' + cat['id'], - 'on': { - 'click': (e) => { - this.requestAnnotation(cat['id']); - } - } - }, cat['name']) - ); + if(supercategories.indexOf(cat['supercategory']) < 0) { + supercategories.push(cat['supercategory']); + } } - this.catNavEl.appendChild(ulEl); - - let defsEl = document.createElementNS("http://www.w3.org/2000/svg", 'defs'); + for(let supercat of supercategories) { + let ulEl = crel('ul', {'data-name': supercat, 'class':'supercategory-'+supercat}); + supercategoriesEls[supercat] = ulEl; + // first half left, other half right side of the page + if(supercategories.indexOf(supercat)/supercategories.length < .5) { + this.catNavEl.appendChild(ulEl); + } else { + this.catNav2El.appendChild(ulEl); + } + } + + // entries for the menus + for(let cat of categories) { +// let firstOfType = lastSuperCat != cat['supercategory'] ? ' first-of-super' : ''; +// lastSuperCat = cat['supercategory']; + supercategoriesEls[cat['supercategory']].appendChild(crel('li', { + 'id': 'category-' + cat['id'], +// 'class': 'supercategory-' + cat['supercategory'], + 'on': { + 'click': (e) => { + this.requestAnnotation(cat['id']); + } + } + }, cat['name'])); + + } + + + this.defsEl = document.createElementNS("http://www.w3.org/2000/svg", 'defs'); for(let cat of categories) { let texture = getTextureForCategory(cat['id']); let sel = new dom(); texture(sel); - defsEl.innerHTML += sel.toString(); + this.defsEl.innerHTML += sel.toString(); } - this.canvas.appendChild(defsEl); + this.canvas.appendChild(this.defsEl); } - + requestAnnotation(category_id) { let r = new Request(`/annotation.json?category=${category_id}&normalise=100`); - fetch(r) + return fetch(r) .then(response => response.json()) .then(annotation => { this.addAnnotationAsShape(annotation); }).catch(function(e){ console.error(e); - });; + }); + } - + pointsToD(points) { let start = points.shift() let d = `M${start[0].toPrecision(4)} ${start[1].toPrecision(4)} L `; @@ -164,7 +201,7 @@ class CocoCanvas { d += points.join(' '); return d; } - + getMousePosition(evt) { // from http://www.petercollingridge.co.uk/tutorials/svg/interactive/dragging/ let CTM = this.canvas.getScreenCTM(); @@ -173,13 +210,13 @@ class CocoCanvas { y: (evt.clientY - CTM.f) / CTM.d }; } - + addAnnotationAsShape(annotation) { console.log('Add annotation', annotation); - + let category = categoryMap[annotation['category_id']] let texture = getTextureForCategory(category['id']); - + let x = 500 - annotation['bbox'][2]/2; let y = 500 - annotation['bbox'][3]/2; let annEl = crel(document.createElementNS("http://www.w3.org/2000/svg", 'g'), { @@ -187,18 +224,19 @@ class CocoCanvas { 'transform': `translate(${x}, ${y})`, 'on': { 'mousedown': function(downE) { - console.log(downE); - console.log(this) + // after clicking the element should be the top element + // and the last svg element is drawn on top. + annEl.parentNode.appendChild(annEl); +// console.log(downE); +// console.log(this) let offset = this.getMousePosition(downE); -// offset.x -= parseFloat(downE.target.getAttributeNS(null, "x")); -// offset.y -= parseFloat(downE.target.getAttributeNS(null, "y")); // Get initial translation amount - console.log(annEl.transform.baseVal); +// console.log(annEl.transform.baseVal); let transform = annEl.transform.baseVal.getItem(0); offset.x -= transform.matrix.e; offset.y -= transform.matrix.f; - + let moveEvent = (moveE) => { let coord = this.getMousePosition(moveE); transform.matrix.e = coord.x - offset.x; @@ -206,29 +244,131 @@ class CocoCanvas { // annEl.setAttributeNS(null, "x", coord.x - offset.x); // annEl.setAttributeNS(null, "y", coord.y - offset.y); }; - - + + document.addEventListener('mousemove', moveEvent); document.addEventListener('mouseup', (upE) => { document.removeEventListener('mousemove', moveEvent); }); - + }.bind(this) } }); - + for(let segment of annotation['segments']) { - + let pathEl = crel(document.createElementNS("http://www.w3.org/2000/svg", 'path'), { 'fill': texture.url(), 'd': this.pointsToD(segment) }); annEl.appendChild(pathEl); } - console.log(annEl); + this.canvas.appendChild(annEl); + + // keep info on annotation elements in CocoCanvas object + annotation['element'] = annEl; + this.annotations.push(annotation); + this.savedBtnEl.disabled = false; + + // based on status of checkboxes, add label or image (bit of a later hack) + this.toggleImages(); + this.toggleLabels(); + } + + // convert annotation shapes on canvas to images, which are masked by the shape + convertToImages() { + console.log('convert'); + for(let annotation of this.annotations) { + console.log(annotation); + if(annotation['element'].getElementsByTagName('image').length) { + //already done + continue; + } + let scale = annotation['scale']; + + let bbox = annotation['is_normalised'] ? annotation['bbox_original'] : annotation['bbox']; + let imgEl = crel(document.createElementNS("http://www.w3.org/2000/svg", 'image'), { + 'href': annotation['image']['coco_url'], + 'width': annotation['image']['width']* scale, + 'height': annotation['image']['height']* scale, + 'x': bbox[0] * -1 * scale, + 'y': bbox[1] * -1 * scale, + }); + annotation['element'].prepend(imgEl ); + } + } + + addLabels() { + console.log('labels'); + for(let annotation of this.annotations) { + console.log(annotation); + if(annotation['element'].getElementsByTagName('text').length) { + //already done + continue; + } + + let textEl = crel(document.createElementNS("http://www.w3.org/2000/svg", 'text'), { + 'x': annotation['bbox'][2] + 5, + 'y': annotation['bbox'][3], + }, categoryMap[annotation['category_id']]['name']); + annotation['element'].append(textEl ); + } + } + + toggleImages(e) { + if(this.loadImagesBtnEl.checked) { + this.convertToImages(); + document.body.classList.remove('hideImages'); + } + else { + document.body.classList.add('hideImages'); + } + } + + + toggleLabels(e) { + if(this.loadLabelsBtnEl.checked) { + this.addLabels(); + document.body.classList.remove('hideLabels'); + } + else { + document.body.classList.add('hideLabels'); + } + } + + save() { + let scene = this.sceneSelectEl.value; + let annotations = []; + for (let ann of this.annotations) { + annotations.push({ + 'id': ann['id'], + 'x': ann.element.transform.baseVal.getItem(0).matrix.e, + 'y': ann.element.transform.baseVal.getItem(0).matrix.f + }); + } + + let data = JSON.stringify({ + 'scene': this.sceneSelectEl.value, + 'annotations': annotations + }); + let r = new Request('/save', {'method': 'POST', 'body': data}); + fetch(r) + .then(response => response.json()) + .then(submission => { +// alert("Something went wrong when saving the file"); + // todo redirect to submission + console.log('saved', submission); + window.location = window.location + 'saved'; + }).catch(function(e){ + alert("Something went wrong when saving the file"); + console.error(e); + }); } } let cc = new CocoCanvas(); cc.start(); + +//for testing only: +//cc.requestAnnotation(1).then((e) => cc.convertToImages()); diff --git a/www/index.html b/www/index.html index 76650a4..0030bb3 100644 --- a/www/index.html +++ b/www/index.html @@ -1,43 +1,12 @@ - + -