From a43437188b84534024ebb9428b79977763be0b05 Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Wed, 30 Oct 2019 15:19:32 +0100 Subject: [PATCH] Status page test --- Pipfile | 1 + Pipfile.lock | 26 +++++++++++++++------ sorteerhoed/central_management.py | 11 +++++++++ sorteerhoed/plotter.py | 16 +++++++++++++ sorteerhoed/webserver.py | 23 +++++++++++++++++-- www/generated/square.svg | 38 ------------------------------- www/status.html | 13 +++++++++++ 7 files changed, 81 insertions(+), 47 deletions(-) delete mode 100644 www/generated/square.svg diff --git a/Pipfile b/Pipfile index 493adbb..55338d7 100644 --- a/Pipfile +++ b/Pipfile @@ -12,6 +12,7 @@ SQLAlchemy = "*" httpagentparser = "*" geoip2 = "*" ink-extensions = "*" +python-magic = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 1cc476d..fc2bfe8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "860fb04e54e9877d9409d7e4853e80ab3d7c0bee9fe3a538d99a73f460dc363d" + "sha256": "df61b6d8cd9643defac48d6d6af45a3bd9d850ab284b04e7ff001b95f591d8a3" }, "pipfile-spec": 6, "requires": { @@ -21,18 +21,18 @@ }, "boto3": { "hashes": [ - "sha256:2edee79d0e78c08b6d14d4dd91c0e4b3438dd4c90c859f06a397268b1cac17b2", - "sha256:3cd2078144c10417eb04e4bb263ea8e50a21c4aceafb52db33e3fe71e73b48aa" + "sha256:03130f43e15f02af7040946f7d4ebbba410fba5ee210b48ca8ba8407d2960b87", + "sha256:0b294a58635ed882a3821854f00264e6b3e65f2114c85c64109ad2280c7d5608" ], "index": "pypi", - "version": "==1.10.0" + "version": "==1.10.5" }, "botocore": { "hashes": [ - "sha256:507b8f13583a64ec2c9c112ff6e3dd8b548060adc7e1f57f25fda9fa34c2dfdb", - "sha256:c4b2ffb0f6ed7169beb260485bf5a42ee72a0a02f49f48b0557ed5e32bcf9e79" + "sha256:13bf9d0d9b9cda278349c9ba0c35c96570795438d5bd29e5188e110b3f5a5183", + "sha256:cfce2c7d6a218d693ed6daa8a41040761e35d7a728d1cf43ae14c4132fb146f3" ], - "version": "==1.13.0" + "version": "==1.13.5" }, "certifi": { "hashes": [ @@ -113,6 +113,7 @@ "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4", "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc", "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1", + "sha256:1409b14bf83a7d729f92e2a7fbfe7ec929d4883ca071b06e95c539ceedb6497c", "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046", "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36", "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5", @@ -123,11 +124,14 @@ "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc", "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7", "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38", + "sha256:9277562f175d2334744ad297568677056861070399cec56ff06abbe2564d1232", "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5", "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832", + "sha256:ae88588d687bd476be588010cbbe551e9c2872b816f2da8f01f6f1fda74e1ef0", "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a", "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f", "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9", + "sha256:c7fccd08b14aa437fe096c71c645c0f9be0655a9b1a4b7cffc77bcb23b3d61d2", "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692", "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84", "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79", @@ -149,6 +153,14 @@ "markers": "python_version >= '2.7'", "version": "==2.8.0" }, + "python-magic": { + "hashes": [ + "sha256:f2674dcfad52ae6c49d4803fa027809540b130db1dec928cfbb9240316831375", + "sha256:f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5" + ], + "index": "pypi", + "version": "==0.4.15" + }, "pyyaml": { "hashes": [ "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", diff --git a/sorteerhoed/central_management.py b/sorteerhoed/central_management.py index 2ae06b6..18ea8d8 100644 --- a/sorteerhoed/central_management.py +++ b/sorteerhoed/central_management.py @@ -126,6 +126,8 @@ class CentralManagement(): pass elif signal.name == 'hit.scanned': # TODO: wrap up hit & make new HIT + self.currentHit.scanned_at = datetime.datetime.utcnow() + self.server.statusPage.set('state', self.currentHit.getStatus()) self.makeHit() elif signal.name == 'scan.start': self.isScanning.set() @@ -138,6 +140,11 @@ class CentralManagement(): for name, value in signal.params.items(): if name == 'hit_id': continue + if name == 'ip': + self.currentHit.turk_ip = value + if name == 'location': + self.currentHit.turk_country = value + self.logger.debug(f'Set status: {name} to {value}') self.server.statusPage.set(name, value) elif signal.name == 'server.open': @@ -189,6 +196,10 @@ class CentralManagement(): # {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"surveycode<\\/QuestionIdentifier>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'} self.logger.info(f'Set status progress to submitted') # TODO: validate the content of the submission by parsing signal.params['event']['Answer'] and comparing it with sqsHit.uuid + sqsHit.answer = signal.params['event']['Answer'] + if sqsHit.uuid not in sqsHit.answer: + self.logger.critical(f"Not a valid answer given?! {sqsHit.answer}") + sqsHit.submit_hit_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") self.store.saveHIT(sqsHit) diff --git a/sorteerhoed/plotter.py b/sorteerhoed/plotter.py index a6bc475..0e63533 100644 --- a/sorteerhoed/plotter.py +++ b/sorteerhoed/plotter.py @@ -6,6 +6,17 @@ from threading import Event from sorteerhoed.Signal import Signal import time +class PathSegment: + def __init__(self): + self.d = [] + + def add(self, i): +# self.d.append(i) + + def get(self, prop): + if prop =='d': + return self.d + class Plotter: def __init__(self, config, eventQ: Queue, runningEvent: Event): #TODO: scanningEvent -> CentralManagement.isScanning -> prevent plotter move during scan, failsafe @@ -31,6 +42,10 @@ class Plotter: def axiDrawCueListener(self): if self.config['dummy_plotter']: while self.isRunning.is_set(): + # TODO: queue that collects a part of the path data + # on a specific limit or on a specific time interval, do the plot + # also, changing ad.pen_raise() or ad.pen_lower() trigger a new segment + # Plot with ad.plan_trajectory() ?? plotterRan = False try: move = self.q.get(True, 1) @@ -48,6 +63,7 @@ class Plotter: self.ad = axidraw.AxiDraw() self.ad.interactive() +# self.ad.plot_path() connected = self.ad.connect() if not connected: diff --git a/sorteerhoed/webserver.py b/sorteerhoed/webserver.py index 21db072..837d5d4 100644 --- a/sorteerhoed/webserver.py +++ b/sorteerhoed/webserver.py @@ -5,7 +5,7 @@ import tornado.ioloop import tornado.web import tornado.websocket from urllib.parse import urlparse -import uuid +import magic from threading import Thread, Event from queue import Queue, Empty @@ -14,6 +14,7 @@ from sorteerhoed import HITStore from sorteerhoed.Signal import Signal import httpagentparser import geoip2.database +import queue logger = logging.getLogger("sorteerhoed").getChild("webserver") @@ -25,6 +26,14 @@ class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler): self.set_header("Access-Control-Allow-Origin", "*") if path[-4:] == '.svg': self.set_header("Content-Type", "image/svg+xml") + if path[-4:] == '.png': + # in testing, without scanner, images are saved as svg + mime = magic.from_file(os.path.join(self.root, path), mime=True) + print(mime) + if mime == 'image/svg+xml': + self.set_header("Content-Type", "image/svg+xml") + + class WebSocketHandler(tornado.websocket.WebSocketHandler): CORS_ORIGINS = ['localhost', '.mturk.com', 'here.rubenvandeven.com'] @@ -165,6 +174,7 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler): class StatusWebSocketHandler(tornado.websocket.WebSocketHandler): CORS_ORIGINS = ['localhost'] connections = set() + queue = queue.Queue() def initialize(self): pass @@ -193,6 +203,7 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler): @classmethod def update_for_all(cls, prop, value): + logger.debug(f"update for all {prop} {value}") for connection in cls.connections: connection.write_message(json.dumps({ 'property': prop, @@ -279,7 +290,11 @@ class StatusPage(): self.__dict__[name] =value logger.info(f"Update status: {name}: {value}") - StatusWebSocketHandler.update_for_all(name, value) + if Server.loop: + Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, name, value) + else: + logger.warn("Status: no server loop to call update command") + def set(self, name, value): return self.__setattr__(name, value) @@ -291,6 +306,8 @@ class Server: TODO: change to have the HIT_id as param to the page. Load hit from storage with previous image """ + loop = None + def __init__(self, config, eventQ: Queue, runningEvent: Event, plotterQ: Queue, store: HITStore): self.isRunning = runningEvent self.eventQ = eventQ @@ -306,6 +323,7 @@ class Server: self.store = store self.statusPage = StatusPage() + def start(self): if not os.path.exists('GeoLite2-Country.mmdb'): raise Exception("Please download the GeoLite2 Country database and place the 'GeoLite2-Country.mmdb' file in the project root.") @@ -330,6 +348,7 @@ class Server: ], debug=True, autoreload=False) application.listen(self.config['server']['port']) self.server_loop = tornado.ioloop.IOLoop.current() + Server.loop = self.server_loop if self.isRunning.is_set(): self.server_loop.start() finally: diff --git a/www/generated/square.svg b/www/generated/square.svg deleted file mode 100644 index b85c40f..0000000 --- a/www/generated/square.svg +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - image/svg+xml - - - - - - - - - diff --git a/www/status.html b/www/status.html index a5dddc3..80d82b7 100644 --- a/www/status.html +++ b/www/status.html @@ -5,5 +5,18 @@ Status! websocket at /status/ws +
    +
+ \ No newline at end of file