Status page test

Pipfile

@@ -12,6 +12,7 @@ SQLAlchemy = "*"
 httpagentparser = "*"
 geoip2 = "*"
 ink-extensions = "*"
+python-magic = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "860fb04e54e9877d9409d7e4853e80ab3d7c0bee9fe3a538d99a73f460dc363d"
+            "sha256": "df61b6d8cd9643defac48d6d6af45a3bd9d850ab284b04e7ff001b95f591d8a3"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -21,18 +21,18 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:2edee79d0e78c08b6d14d4dd91c0e4b3438dd4c90c859f06a397268b1cac17b2",
+                "sha256:03130f43e15f02af7040946f7d4ebbba410fba5ee210b48ca8ba8407d2960b87",
-                "sha256:3cd2078144c10417eb04e4bb263ea8e50a21c4aceafb52db33e3fe71e73b48aa"
+                "sha256:0b294a58635ed882a3821854f00264e6b3e65f2114c85c64109ad2280c7d5608"
             ],
             "index": "pypi",
-            "version": "==1.10.0"
+            "version": "==1.10.5"
         },
         "botocore": {
             "hashes": [
-                "sha256:507b8f13583a64ec2c9c112ff6e3dd8b548060adc7e1f57f25fda9fa34c2dfdb",
+                "sha256:13bf9d0d9b9cda278349c9ba0c35c96570795438d5bd29e5188e110b3f5a5183",
-                "sha256:c4b2ffb0f6ed7169beb260485bf5a42ee72a0a02f49f48b0557ed5e32bcf9e79"
+                "sha256:cfce2c7d6a218d693ed6daa8a41040761e35d7a728d1cf43ae14c4132fb146f3"
             ],
-            "version": "==1.13.0"
+            "version": "==1.13.5"
         },
         "certifi": {
             "hashes": [
@@ -113,6 +113,7 @@
                 "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4",
                 "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc",
                 "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1",
+                "sha256:1409b14bf83a7d729f92e2a7fbfe7ec929d4883ca071b06e95c539ceedb6497c",
                 "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046",
                 "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36",
                 "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5",
@@ -123,11 +124,14 @@
                 "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc",
                 "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7",
                 "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38",
+                "sha256:9277562f175d2334744ad297568677056861070399cec56ff06abbe2564d1232",
                 "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5",
                 "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832",
+                "sha256:ae88588d687bd476be588010cbbe551e9c2872b816f2da8f01f6f1fda74e1ef0",
                 "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a",
                 "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f",
                 "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9",
+                "sha256:c7fccd08b14aa437fe096c71c645c0f9be0655a9b1a4b7cffc77bcb23b3d61d2",
                 "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692",
                 "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84",
                 "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79",
@@ -149,6 +153,14 @@
             "markers": "python_version >= '2.7'",
             "version": "==2.8.0"
         },
+        "python-magic": {
+            "hashes": [
+                "sha256:f2674dcfad52ae6c49d4803fa027809540b130db1dec928cfbb9240316831375",
+                "sha256:f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5"
+            ],
+            "index": "pypi",
+            "version": "==0.4.15"
+        },
         "pyyaml": {
             "hashes": [
                 "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",

sorteerhoed/central_management.py

@@ -126,6 +126,8 @@ class CentralManagement():
                     pass
                 elif signal.name == 'hit.scanned':
                     # TODO: wrap up hit & make new HIT
+                    self.currentHit.scanned_at = datetime.datetime.utcnow()
+                    self.server.statusPage.set('state', self.currentHit.getStatus())
                     self.makeHit()
                 elif signal.name == 'scan.start':
                     self.isScanning.set()
@@ -138,6 +140,11 @@ class CentralManagement():
                     for name, value in signal.params.items():
                         if name == 'hit_id':
                             continue
+                        if name == 'ip':
+                            self.currentHit.turk_ip = value
+                        if name == 'location':
+                            self.currentHit.turk_country = value
+                        
                         self.logger.debug(f'Set status: {name} to {value}')
                         self.server.statusPage.set(name, value)
                 elif signal.name == 'server.open':
@@ -189,6 +196,10 @@ class CentralManagement():
                         # {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"<?xml version=\\"1.0\\" encoding=\\"ASCII\\"?><QuestionFormAnswers xmlns=\\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\\"><Answer><QuestionIdentifier>surveycode<\\/QuestionIdentifier><FreeText>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'}
                         self.logger.info(f'Set status progress to submitted')
                         # TODO: validate the content of the submission by parsing signal.params['event']['Answer'] and comparing it with sqsHit.uuid
+                        sqsHit.answer = signal.params['event']['Answer']
+                        if sqsHit.uuid not in sqsHit.answer:
+                            self.logger.critical(f"Not a valid answer given?! {sqsHit.answer}")
+                        
                         sqsHit.submit_hit_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
                         
                     self.store.saveHIT(sqsHit)

sorteerhoed/plotter.py

@@ -6,6 +6,17 @@ from threading import Event
 from sorteerhoed.Signal import Signal
 import time
 
+class PathSegment:
+    def __init__(self):
+        self.d = []
+    
+    def add(self, i):
+#         self.d.append(i)
+    
+    def get(self, prop):
+        if prop =='d':
+            return self.d
+
 class Plotter:
     def __init__(self, config, eventQ: Queue, runningEvent: Event):
         #TODO: scanningEvent -> CentralManagement.isScanning -> prevent plotter move during scan, failsafe
@@ -31,6 +42,10 @@ class Plotter:
     def axiDrawCueListener(self):
         if self.config['dummy_plotter']:
             while self.isRunning.is_set():
+                # TODO: queue that collects a part of the path data
+                # on a specific limit or on a specific time interval, do the plot
+                # also, changing ad.pen_raise() or ad.pen_lower() trigger a new segment
+                # Plot with ad.plan_trajectory() ??
                 plotterRan = False
                 try:
                     move = self.q.get(True, 1)
@@ -48,6 +63,7 @@ class Plotter:
             self.ad = axidraw.AxiDraw()
             
             self.ad.interactive()
+#             self.ad.plot_path()
             
             connected = self.ad.connect()
             if not connected:

sorteerhoed/webserver.py

@@ -5,7 +5,7 @@ import tornado.ioloop
 import tornado.web
 import tornado.websocket
 from urllib.parse import urlparse
-import uuid
+import magic
 
 from threading import Thread, Event
 from queue import Queue, Empty
@@ -14,6 +14,7 @@ from sorteerhoed import HITStore
 from sorteerhoed.Signal import Signal
 import httpagentparser
 import geoip2.database
+import queue
 
 
 logger = logging.getLogger("sorteerhoed").getChild("webserver")
@@ -25,6 +26,14 @@ class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler):
             self.set_header("Access-Control-Allow-Origin", "*")
         if path[-4:] == '.svg':
             self.set_header("Content-Type", "image/svg+xml")
+        if path[-4:] == '.png':
+            # in testing, without scanner, images are saved as svg
+            mime = magic.from_file(os.path.join(self.root, path), mime=True)
+            print(mime)
+            if mime == 'image/svg+xml':
+                self.set_header("Content-Type", "image/svg+xml")
+                
+            
 
 class WebSocketHandler(tornado.websocket.WebSocketHandler):
     CORS_ORIGINS = ['localhost', '.mturk.com', 'here.rubenvandeven.com']
@@ -165,6 +174,7 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
 class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
     CORS_ORIGINS = ['localhost']
     connections = set()
+    queue = queue.Queue()
     
     def initialize(self):
         pass
@@ -193,6 +203,7 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
         
     @classmethod
     def update_for_all(cls, prop, value):
+        logger.debug(f"update for all {prop} {value}")
         for connection in cls.connections:
             connection.write_message(json.dumps({
                 'property': prop,
@@ -279,7 +290,11 @@ class StatusPage():
         
         self.__dict__[name] =value
         logger.info(f"Update status: {name}: {value}")
-        StatusWebSocketHandler.update_for_all(name, value)
+        if Server.loop:
+            Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, name, value)
+        else:
+            logger.warn("Status: no server loop to call update command")
+        
         
     def set(self, name, value):
         return self.__setattr__(name, value)
@@ -291,6 +306,8 @@ class Server:
     
     TODO: change to have the HIT_id as param to the page. Load hit from storage with previous image
     """
+    loop = None
+    
     def __init__(self, config, eventQ: Queue, runningEvent: Event, plotterQ: Queue, store: HITStore):
         self.isRunning = runningEvent
         self.eventQ = eventQ
@@ -306,6 +323,7 @@ class Server:
         self.store = store
         self.statusPage = StatusPage()
         
+        
     def start(self):
         if not os.path.exists('GeoLite2-Country.mmdb'):
             raise Exception("Please download the GeoLite2 Country database and place the 'GeoLite2-Country.mmdb' file in the project root.")
@@ -330,6 +348,7 @@ class Server:
             ], debug=True, autoreload=False)
             application.listen(self.config['server']['port'])
             self.server_loop = tornado.ioloop.IOLoop.current()
+            Server.loop = self.server_loop
             if self.isRunning.is_set():
                 self.server_loop.start()
         finally:

www/generated/square.svg

@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   id="svg8"
-   version="1.1"
-   viewBox="0 0 210 210"
-   height="210mm"
-   width="210mm">
-  <defs
-     id="defs2" />
-  <metadata
-     id="metadata5">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title></dc:title>
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <g
-     transform="translate(0,-87)"
-     id="layer1">
-    <rect
-       y="138.32738"
-       x="51.327381"
-       height="107.34524"
-       width="107.34524"
-       id="rect815"
-       style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1" />
-  </g>
-</svg>

www/status.html

@@ -5,5 +5,18 @@
 <body>
 Status!
 websocket at /status/ws
+<ol id='msgs'>
+</ol>
+<script>
+let url = window.location.origin.replace('http', 'ws') +'/status/ws';
+let el = document.getElementById("msgs")
+let socket = new WebSocket(url);
+
+socket.addEventListener('message', function(e){
+	let liEl = document.createElement('li');
+	liEl.innerHTML = e.data;
+	el.appendChild(liEl);
+});
+</script>
 </body>
 </html>