r
/
guest_worker
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored to work with separate assignment table

This commit is contained in:
Ruben van de Ven 2020-01-13 13:49:59 +01:00
parent ac04fb6082
commit d3bf3d47ea
4 changed files with 345 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
sorteerhoed/HITStore.py
View File

@ -9,9 +9,6 @@ import datetime
from contextlib import contextmanager from contextlib import contextmanager
import uuid import uuid
import os import os
import coloredlogs
import argparse
from sqlalchemy.sql.functions import func
mainLogger = logging.getLogger("sorteerhoed") mainLogger = logging.getLogger("sorteerhoed")
logger = mainLogger.getChild("store") logger = mainLogger.getChild("store")
@ -43,6 +40,7 @@ class HIT(Base):
created_at = Column(DateTime, default=datetime.datetime.utcnow) created_at = Column(DateTime, default=datetime.datetime.utcnow)
updated_at = Column(DateTime, default=datetime.datetime.utcnow) updated_at = Column(DateTime, default=datetime.datetime.utcnow)
scanned_at = Column(DateTime, default=None) scanned_at = Column(DateTime, default=None)
plotted_at = Column(DateTime, default=None)
deleted_at = Column(DateTime, default=None) deleted_at = Column(DateTime, default=None)
assignments = relationship("Assignment", back_populates="hit", order_by="Assignment.created_at") assignments = relationship("Assignment", back_populates="hit", order_by="Assignment.created_at")
fee = Column(Float(precision=2), default=None) fee = Column(Float(precision=2), default=None)
@ -53,6 +51,9 @@ class HIT(Base):
def getImagePath(self): def getImagePath(self):
return os.path.join('scanimation/interfaces/frames', f"{self.id:06d}.jpg") return os.path.join('scanimation/interfaces/frames', f"{self.id:06d}.jpg")
def getImageUrl(self):
return os.path.join('/frames', f"{self.id:06d}.jpg")
def getSvgImageUrl(self): def getSvgImageUrl(self):
return f"scans/{self.id:06d}.svg" return f"scans/{self.id:06d}.svg"
@ -68,23 +69,44 @@ class HIT(Base):
def getAssignmentById(self, assignmentId): def getAssignmentById(self, assignmentId):
for a in self.assignments: for a in self.assignments:
if a.assignment_id == assignmentId: if a.assignment_id == assignmentId:
return return a
return None
def getStatus(self): def getStatus(self):
assignment = self.getLastAssignment()
if self.deleted_at:
return "deleted"
if not self.hit_id:
return "creating"
if not assignment:
return "awaiting worker"
if self.scanned_at: if self.scanned_at:
return "completed" return "scanned"
if self.submit_hit_at: return assignment.getStatus()
return "submission confirmed"
if self.submit_page_at: def toDict(self) -> dict:
return "submitted by worker" values = {c.name: getattr(self, c.name) for c in self.__table__.columns}
if self.open_page_at: assignment = self.getLastAssignment()
return "working" values['assignment'] = assignment.toDict() if assignment else None
if self.accept_time: values['state'] = self.getStatus()
return "accepted by worker" values['scan_image'] = self.getImageUrl() if self.scanned_at else None
# on abandon: values['svg_image'] = self.getSvgImageUrl() if self.isSubmitted() else None
if self.worker_id: return values
return "abandoned by worker"
return "awaiting worker" def delete(self):
self.deleted_at = datetime.datetime.utcnow()
def isSubmitted(self) -> bool:
a = self.getLastAssignment()
if not a:
return False
return bool(a.submit_page_at)
def isConfirmed(self) -> bool:
a = self.getLastAssignment()
if not a:
return False
return bool(a.confirmed_at)
class Assignment(Base): class Assignment(Base):
__tablename__ = 'assignments' __tablename__ = 'assignments'
@ -99,7 +121,7 @@ class Assignment(Base):
assignment_id = Column(String(255), default = None) assignment_id = Column(String(255), default = None)
worker_id = Column(String(255), default = None) worker_id = Column(String(255), default = None)
accept_at = Column(DateTime, default=None) accept_at = Column(DateTime, default=None) # accept time acccording to SQS
# open_page_at = Column(DateTime, default=None) # open_page_at = Column(DateTime, default=None)
submit_page_at = Column(DateTime, default=None) # Submit the page submit_page_at = Column(DateTime, default=None) # Submit the page
confirmed_at = Column(DateTime, default=None) # validate with UUID when getting Message from Amazon confirmed_at = Column(DateTime, default=None) # validate with UUID when getting Message from Amazon
@ -109,6 +131,24 @@ class Assignment(Base):
answer = Column(String(255), default=None) answer = Column(String(255), default=None)
turk_ip = Column(String(255), default=None) turk_ip = Column(String(255), default=None)
turk_country = Column(String(255), default=None) turk_country = Column(String(255), default=None)
turk_os = Column(String(255), default=None)
turk_browser = Column(String(255), default=None)
def getStatus(self):
if self.rejected_at:
return "rejected"
if self.abandoned_at:
return "abandoned"
if not self.submit_page_at:
return "working"
if not self.confirmed_at:
return "submitted"
return "confirmed"
def toDict(self) -> dict:
values = {c.name: getattr(self, c.name) for c in self.__table__.columns}
return values
class Store: class Store:
def __init__(self, db_filename, logLevel=0): def __init__(self, db_filename, logLevel=0):
@ -122,6 +162,19 @@ class Store:
self.session = self.Session() self.session = self.Session()
self.currentHit = None # mirrors Centralmanagmenet, stored here so we can quickly access it from webserver classes self.currentHit = None # mirrors Centralmanagmenet, stored here so we can quickly access it from webserver classes
self.updateHooks = []
def registerUpdateHook(self, hook):
if hook not in self.updateHooks:
logger.info(f"Register update hook: {hook}")
self.updateHooks.append(hook)
def triggerUpdateHooks(self, hit = None):
for hook in self.updateHooks:
if callable(hook): # it's a method
hook(hit)
else: # assume it's an object
hook.update(hit)
@contextmanager @contextmanager
def getSession(self): def getSession(self):
@ -133,8 +186,8 @@ class Store:
self.session.rollback() self.session.rollback()
raise raise
def getHits(self, session): def getHits(self):
return self.session.query(Source).order_by(HIT.created_at.desc()) return self.session.query(HIT).order_by(HIT.created_at.desc())
def getHitById(self, hitId): def getHitById(self, hitId):
return self.session.query(HIT).\ return self.session.query(HIT).\
@ -147,8 +200,19 @@ class Store:
def getLastSubmittedHit(self): def getLastSubmittedHit(self):
return self.session.query(HIT).\ return self.session.query(HIT).\
filter(HIT.submit_page_at!=None).\ join(Assignment).\
order_by(HIT.submit_page_at.desc()).first() filter(Assignment.submit_page_at!=None).\
order_by(HIT.created_at.desc()).first()
def getNewestHits(self, n = 2) -> list:
hits = list(
self.session.query(HIT).\
filter(HIT.deleted_at==None).\
order_by(HIT.created_at.desc()).limit(2)
)
# select DESC, because we want latest, then reverse list to get in right order
hits.reverse()
return hits
def createHIT(self) -> HIT: def createHIT(self) -> HIT:
with self.getSession() as s: with self.getSession() as s:
@ -157,40 +221,57 @@ class Store:
s.flush() s.flush()
s.refresh(hit) s.refresh(hit)
logger.info(f"Created HIT {hit.id}") logger.info(f"Created HIT {hit.id}")
self.triggerUpdateHooks(hit)
return hit return hit
def newAssignment(self, hit: HIT) -> Assignment: def newAssignment(self, hit: HIT, assignmentId) -> Assignment:
# TODO: reset() central management if has pending lastAssignment()
with self.getSession() as s: with self.getSession() as s:
assignment = Assignment() assignment = Assignment()
assignment.assignment_id = assignmentId
hit.assignments.append(assignment) hit.assignments.append(assignment)
s.add(assignment) s.add(assignment)
s.flush() s.flush()
s.refresh(hit) s.refresh(hit)
logger.info(f"Created Assignment {assignment.id}") logger.info(f"Created Assignment {assignment.id}")
self.triggerUpdateHooks(hit)
return assignment return assignment
def saveHIT(self, hit): def saveHIT(self, hit):
with self.getSession() as s: with self.getSession() as s:
logger.info(f"Updating hit! {hit.id}") logger.info(f"Updating hit! {hit.id}")
# s.flush() # s.flush()
self.triggerUpdateHooks(hit)
def addHIT(self, hit: HIT): def saveAssignment(self, assignment):
with self.getSession() as s: with self.getSession() as s:
s.add(hit) logger.info(f"Updating assignment! {assignment.id}")
s.flush() # s.flush()
s.refresh(hit) self.triggerUpdateHooks(assignment.hit)
logger.info(f"Added {hit.id}")
# def addHIT(self, hit: HIT):
# with self.getSession() as s:
# s.add(hit)
# s.flush()
# s.refresh(hit)
# logger.info(f"Added {hit.id}")
def getAvgDurationOfPreviousNHits(self, n) -> int: def getAvgDurationOfPreviousNHits(self, n) -> int:
latest_hits = self.session.query(HIT).\ latest_assignments = self.session.query(Assignment).\
filter(HIT.submit_hit_at!=None).\ filter(Assignment.created_at!=None).\
filter(HIT.accept_time!=None).\ filter(Assignment.submit_page_at!=None).\
order_by(HIT.submit_hit_at.desc()).limit(n) order_by(Assignment.created_at.desc()).limit(n)
durations = [] durations = []
for hit in latest_hits:
durations.append((hit.submit_hit_at - hit.accept_time).total_seconds()) for assignment in latest_assignments:
durations.append((assignment.submit_page_at - assignment.created_at).total_seconds())
if not len(durations): if not len(durations):
return int(2.5*60) return int(2.5*60) # default to 2.5 minutes
return int(sum(durations) / len(durations)) return int(sum(durations) / len(durations))
def getEstimatedHitDuration(self): def getEstimatedHitDuration(self):
@ -203,15 +284,3 @@ class Store:
return self.session.query(HIT).\ return self.session.query(HIT).\
filter(HIT.submit_hit_at != None).\ filter(HIT.submit_hit_at != None).\
order_by(HIT.submit_hit_at.desc()).limit(n) order_by(HIT.submit_hit_at.desc()).limit(n)
# def rmSource(self, id: int):
# with self.getSession() as session:
# source = session.query(Source).get(id)
# if not source:
# logging.warning(f"Source nr {id} not found")
# else:
# logging.info(f"Deleting source {source.id}: {source.url}")
# session.delete(source)
#
# def getRandomNewsItem(self, session) -> NewsItem:
# return session.query(NewsItem).order_by(func.random()).limit(1).first()

243
sorteerhoed/central_management.py
View File

@ -17,10 +17,14 @@ from PIL import Image
import datetime import datetime
from shutil import copyfile from shutil import copyfile
import colorsys import colorsys
import tqdm
class Level(object): class Level(object):
# Level effect adapted from https://stackoverflow.com/a/3125421 """
Level image effect adapted from https://stackoverflow.com/a/3125421
"""
def __init__(self, minv, maxv, gamma): def __init__(self, minv, maxv, gamma):
self.minv= minv/255.0 self.minv= minv/255.0
self.maxv= maxv/255.0 self.maxv= maxv/255.0
@ -81,6 +85,7 @@ class CentralManagement():
self.isScanning = threading.Event() self.isScanning = threading.Event()
self.scanLock = threading.Lock() self.scanLock = threading.Lock()
self.notPaused = threading.Event() self.notPaused = threading.Event()
self.lightStatus = 0
def loadConfig(self, filename, args): def loadConfig(self, filename, args):
@ -94,6 +99,7 @@ class CentralManagement():
'hit_store.db' 'hit_store.db'
) )
self.store = HITStore.Store(varDb, logLevel=logging.DEBUG if self.debug else logging.INFO) self.store = HITStore.Store(varDb, logLevel=logging.DEBUG if self.debug else logging.INFO)
self.store.registerUpdateHook(self.updateLightHook) # change light based on status
self.logger.debug(f"Loaded configuration: {self.config}") self.logger.debug(f"Loaded configuration: {self.config}")
@ -142,6 +148,10 @@ class CentralManagement():
ExpireAt=datetime.datetime.fromisoformat('2015-01-01') ExpireAt=datetime.datetime.fromisoformat('2015-01-01')
) )
self.mturk.delete_hit(HITId=pending_hit['HITId']) self.mturk.delete_hit(HITId=pending_hit['HITId'])
staleHit = self.store.getHitByRemoteId(pending_hit['HITId'])
staleHit.delete()
self.store.saveHIT(staleHit)
self.sqs = SqsListener(self.config, self.eventQueue, self.isRunning) self.sqs = SqsListener(self.config, self.eventQueue, self.isRunning)
sqsThread = threading.Thread(target=self.sqs.start, name='sqs') sqsThread = threading.Thread(target=self.sqs.start, name='sqs')
@ -160,8 +170,6 @@ class CentralManagement():
# event listener: # event listener:
dispatcherThread = threading.Thread(target=self.eventListener, name='dispatcher') dispatcherThread = threading.Thread(target=self.eventListener, name='dispatcher')
dispatcherThread.start() dispatcherThread.start()
#
#
self.eventQueue.put(Signal('start', {'ding':'test'})) self.eventQueue.put(Signal('start', {'ding':'test'}))
@ -177,8 +185,9 @@ class CentralManagement():
self.expireCurrentHit() self.expireCurrentHit()
def expireCurrentHit(self): def expireCurrentHit(self):
if self.currentHit and self.currentHit.hit_id: # hit pending if self.currentHit and not self.currentHit.isConfirmed():
self.logger.warn(f"Delete hit: {self.currentHit.hit_id}") if self.currentHit.hit_id: # hit pending at Amazon
self.logger.warn(f"Expire hit: {self.currentHit.hit_id}")
self.mturk.update_expiration_for_hit( self.mturk.update_expiration_for_hit(
HITId=self.currentHit.hit_id, HITId=self.currentHit.hit_id,
ExpireAt=datetime.datetime.fromisoformat('2015-01-01') ExpireAt=datetime.datetime.fromisoformat('2015-01-01')
@ -187,7 +196,12 @@ class CentralManagement():
self.mturk.delete_hit(HITId=self.currentHit.hit_id) self.mturk.delete_hit(HITId=self.currentHit.hit_id)
except Exception as e: except Exception as e:
self.logger.exception(e) self.logger.exception(e)
if not self.currentHit.isSubmitted():
self.currentHit.delete()
self.store.saveHIT(self.currentHit)
def eventListener(self): def eventListener(self):
while self.isRunning.is_set(): while self.isRunning.is_set():
try: try:
@ -206,27 +220,27 @@ class CentralManagement():
- Plotter complete - Plotter complete
- -
""" """
#TODO: make level debug()
self.logger.info(f"SIGNAL: {signal}") self.logger.info(f"SIGNAL: {signal}")
if signal.name == 'start': if signal.name == 'start':
self.makeHit() self.makeHit()
self.lastHitTime = datetime.datetime.now() self.lastHitTime = datetime.datetime.now()
elif signal.name == 'hit.scan': elif signal.name == 'hit.scan':
# start a scan
if signal.params['id'] != self.currentHit.id: if signal.params['id'] != self.currentHit.id:
self.logger.info(f"Hit.scanned had wrong id: {signal}") self.logger.info(f"Hit.scan had wrong id: {signal}")
continue continue
self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='scanning')) # self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='scanning'))
elif signal.name == 'hit.scanned': elif signal.name == 'hit.scanned':
# TODO: wrap up hit & make new HIT # TODO: wrap up hit & make new HIT
if signal.params['id'] != self.currentHit.id: if signal.params['hit_id'] != self.currentHit.id:
self.logger.info(f"Hit.scanned had wrong id: {signal}") self.logger.info(f"Hit.scanned had wrong id: {signal}")
continue continue
self.currentHit.scanned_at = datetime.datetime.utcnow() self.currentHit.scanned_at = datetime.datetime.utcnow()
time_diff = datetime.datetime.now() - self.lastHitTime time_diff = datetime.datetime.now() - self.lastHitTime
to_wait = 10 - time_diff.total_seconds() to_wait = 10 - time_diff.total_seconds()
self.statusPageQueue.add(dict(hit_id=self.currentHit.id, state='scan')) # self.statusPageQueue.add(dict(hit_id=self.currentHit.id, state='scan'))
if to_wait > 0: if to_wait > 0:
self.logger.warn(f"Sleep until next hit: {to_wait}s") self.logger.warn(f"Sleep until next hit: {to_wait}s")
@ -237,10 +251,11 @@ class CentralManagement():
self.makeHit() self.makeHit()
self.lastHitTime = datetime.datetime.now() self.lastHitTime = datetime.datetime.now()
elif signal.name == 'hit.creating': elif signal.name == 'hit.creating':
self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='create_hit')) # self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='create_hit'))
pass
elif signal.name == 'hit.created': elif signal.name == 'hit.created':
self.statusPageQueue.add(dict(hit_id=signal.params['id'], remote_id=signal.params['remote_id'], state='hit')) # self.statusPageQueue.add(dict(hit_id=signal.params['id'], remote_id=signal.params['remote_id'], state='hit'))
pass
elif signal.name == 'scan.start': elif signal.name == 'scan.start':
pass pass
elif signal.name == 'scan.finished': elif signal.name == 'scan.finished':
@ -251,38 +266,41 @@ class CentralManagement():
# Create new assignment # Create new assignment
if signal.params['hit_id'] != self.currentHit.id: if signal.params['hit_id'] != self.currentHit.id:
continue continue
assignment = self.currentHit.getAssignmentById(signal.params['assignment_id'])
assignment = self.store.newAssignment(self.currentHit)
assignment.assignment_id = signal.params['assignment_id']
self.store.saveAssignment(assignment)
self.statusPageQueue.add(dict(hit_id=self.currentHit.id, assignment_id=assignment.assignment_id, state='assignment'))
elif signal.name == 'assignment.info': elif signal.name == 'assignment.info':
assignment = self.currentHit.getAssignmentById(signal.params['assignment_id']) assignment = self.currentHit.getAssignmentById(signal.params['assignment_id'])
if not assignment: if not assignment:
self.logger.warning(f"assignment.info assignment.id not for current hit assignments: {signal}") self.logger.warning(f"assignment.info assignment.id not for current hit assignments: {signal}")
change = False
for name, value in signal.params.items(): for name, value in signal.params.items():
if name == 'ip': if name == 'ip':
assignment.turk_ip = value assignment.turk_ip = value
if name == 'location': if name == 'location':
assignment.turk_country = value assignment.turk_country = value
if name == 'os':
assignment.turk_os = value
if name == 'browser':
assignment.turk_browser = value
change = True
self.logger.debug(f'Set assignment: {name} to {value}') self.logger.debug(f'Set assignment: {name} to {value}')
self.server.statusPage.set(name, value)
if change:
self.store.saveAssignment(assignment)
elif signal.name == 'server.open': elif signal.name == 'server.open':
self.currentHit.open_page_at = datetime.datetime.utcnow() self.currentHit.open_page_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit) self.store.saveHIT(self.currentHit)
self.setLight(True) self.setLight(True)
self.server.statusPage.set('state', self.currentHit.getStatus()) elif signal.name == 'assignment.submit':
self.server.statusPage.set('hit_opened', self.currentHit.open_page_at) a = self.currentHit.getLastAssignment()
elif signal.name == 'server.submit': if a.assignment_id != signal.params['assignment_id']:
self.currentHit.submit_page_at = datetime.datetime.utcnow() self.logger.critical(f"Submit of invalid assignment_id: {signal}")
self.store.saveHIT(self.currentHit)
a.submit_page_at = datetime.datetime.utcnow()
self.store.saveAssignment(a)
self.plotter.park() self.plotter.park()
self.server.statusPage.set('hit_opened', self.currentHit.open_page_at)
# park always triggers a plotter.finished after being processed # park always triggers a plotter.finished after being processed
elif signal.name[:4] == 'sqs.': elif signal.name[:4] == 'sqs.':
@ -293,49 +311,37 @@ class CentralManagement():
else: else:
sqsHit = self.currentHit sqsHit = self.currentHit
updateStatus = True updateStatus = True
sqsAssignment = sqsHit.getAssignmentById(signal.params['event']['AssignmentId'])
if not sqsAssignment:
self.logger.critical(f"Invalid assignmentId given for hit: {signal.params['event']}")
continue
if signal.name == 'sqs.AssignmentAccepted': if signal.name == 'sqs.AssignmentAccepted':
self.logger.info(f'Set status progress to accepted') self.logger.info(f'Set status progress to accepted')
sqsHit.accept_time = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsAssignment.accept_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
sqsHit.worker_id = signal.params['event']['WorkerId'] sqsAssignment.worker_id = signal.params['event']['WorkerId']
if updateStatus:
self.server.statusPage.set('worker_id', sqsHit.worker_id)
# {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAccepted', 'EventTimestamp': '2019-10-23T20:16:10Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} # {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAccepted', 'EventTimestamp': '2019-10-23T20:16:10Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
elif signal.name == 'sqs.AssignmentAbandoned': elif signal.name == 'sqs.AssignmentAbandoned':
self.logger.info(f'Set status progress to abandoned') self.logger.info(f'Set status progress to abandoned')
#{'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAbandoned', 'EventTimestamp': '2019-10-23T20:23:06Z', 'HITId': '3JHB4BPSFKKFQ263K4EFULI3LC79QJ', 'AssignmentId': '3U088ZLJVL450PB6MJZUIIUCB6VW0Y', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} #{'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAbandoned', 'EventTimestamp': '2019-10-23T20:23:06Z', 'HITId': '3JHB4BPSFKKFQ263K4EFULI3LC79QJ', 'AssignmentId': '3U088ZLJVL450PB6MJZUIIUCB6VW0Y', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
sqsHit.accept_time = None sqsAssignment.abandoned_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
sqsHit.open_page_at = None # if updateStatus:
if self.currentHit.id == sqsHit.id: # self.setLight(False)
if not sqsHit.submit_page_at:
self.reset()
else:
sqsHit.submit_hit_at = datetime.datetime.utcnow() # fake submit
if updateStatus:
self.setLight(False)
self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Accepted task without working on it.') self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Accepted task without working on it.')
elif signal.name == 'sqs.AssignmentReturned': elif signal.name == 'sqs.AssignmentReturned':
self.logger.info(f'Set status progress to returned') self.logger.info(f'Set status progress to returned')
sqsHit.accept_time = None sqsAssignment.returned_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
sqsHit.open_page_at = None
if self.currentHit.id == sqsHit.id:
if not sqsHit.submit_page_at:
self.reset()
else:
sqsHit.submit_hit_at = datetime.datetime.utcnow() # fake submit
if updateStatus:
self.setLight(False)
# {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentReturned', 'EventTimestamp': '2019-10-23T20:16:47Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} # {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentReturned', 'EventTimestamp': '2019-10-23T20:16:47Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
elif signal.name == 'sqs.AssignmentSubmitted': elif signal.name == 'sqs.AssignmentSubmitted':
# {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"<?xml version=\\"1.0\\" encoding=\\"ASCII\\"?><QuestionFormAnswers xmlns=\\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\\"><Answer><QuestionIdentifier>surveycode<\\/QuestionIdentifier><FreeText>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'} # {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"<?xml version=\\"1.0\\" encoding=\\"ASCII\\"?><QuestionFormAnswers xmlns=\\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\\"><Answer><QuestionIdentifier>surveycode<\\/QuestionIdentifier><FreeText>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'}
self.logger.info(f'Set status progress to submitted') self.logger.info(f'Set status progress to submitted')
# TODO: validate the content of the submission by parsing signal.params['event']['Answer'] and comparing it with sqsHit.uuid sqsAssignment.answer = signal.params['event']['Answer']
sqsHit.answer = signal.params['event']['Answer'] if sqsAssignment.uuid not in sqsAssignment.answer:
if sqsHit.uuid not in sqsHit.answer: self.logger.critical(f"Not a valid answer given?! {sqsAssignment.answer}")
self.logger.critical(f"Not a valid answer given?! {sqsHit.answer}")
if not sqsHit.submit_page_at: if not sqsAssignment.submit_page_at:
# page not submitted, hit is. Nevertheless, create new hit. # page not submitted, hit is. Nevertheless, create new hit.
try: try:
self.mturk.reject_assignment(AssignmentId=signal.params['event']['AssignmentId'], RequesterFeedback='Did not do the assignment') self.mturk.reject_assignment(AssignmentId=signal.params['event']['AssignmentId'], RequesterFeedback='Did not do the assignment')
@ -343,7 +349,8 @@ class CentralManagement():
self.logger.exception(e) self.logger.exception(e)
self.makeHit() self.makeHit()
else: else:
sqsHit.submit_hit_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsAssignment.confirmed_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
# block de worker na succesvolle submit, om dubbele workers te voorkomen # block de worker na succesvolle submit, om dubbele workers te voorkomen
# TODO: Disabled after worker mail, use quals instead # TODO: Disabled after worker mail, use quals instead
#self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Every worker can only work once on the taks.') #self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Every worker can only work once on the taks.')
@ -351,21 +358,19 @@ class CentralManagement():
self.store.saveHIT(sqsHit) self.store.saveHIT(sqsHit)
if updateStatus:
self.logger.warning(f'update status: {sqsHit.getStatus()}')
# TODO: have HITStore/HIT take care of this by emitting a signal
# only update status if it is the currentHit
self.server.statusPage.set('state', sqsHit.getStatus())
else:
self.logger.warning('DO NOT update status')
elif signal.name == 'plotter.finished': elif signal.name == 'plotter.finished':
if self.currentHit and self.currentHit.submit_page_at: # is _always_ triggered after submit due to plotter.park()
self.setLight(False) if self.currentHit and self.currentHit.isSubmitted():
self.currentHit.plotted_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit)
self.logger.info("Start scan thread")
scan = threading.Thread(target=self.scanImage, name='scan') scan = threading.Thread(target=self.scanImage, name='scan')
scan.start() scan.start()
self.server.statusPage.set('hit_submitted', self.currentHit.submit_page_at) elif signal.name == 'plotter.parked':
self.server.statusPage.set('state', self.currentHit.getStatus()) # should this have the code from plotter.finished?
pass
else: else:
self.logger.critical(f"Unknown signal: {signal.name}") self.logger.critical(f"Unknown signal: {signal.name}")
except Exception as e: except Exception as e:
@ -373,11 +378,10 @@ class CentralManagement():
self.logger.exception(e) self.logger.exception(e)
def makeHit(self): def makeHit(self):
self.expireCurrentHit() # expire hit if it is there self.expireCurrentHit() # expire hit if it is there
self.eventQueue.put(Signal('hit.creating', {'id': self.currentHit.id if self.currentHit else 'start'})) self.eventQueue.put(Signal('hit.creating', {'id': self.currentHit.id if self.currentHit else 'start'}))
self.server.statusPage.reset()
self.reloadConfig() # reload new config values if they are set self.reloadConfig() # reload new config values if they are set
# self.notPaused.wait() # self.notPaused.wait()
@ -387,7 +391,6 @@ class CentralManagement():
self.logger.info(f"Make HIT {self.currentHit.id}") self.logger.info(f"Make HIT {self.currentHit.id}")
# question = open(self.config['amazon']['task_xml'], mode='r').read().replace("{HIT_NR}",str(self.currentHit.id))
question = '''<?xml version="1.0" encoding="UTF-8"?> question = '''<?xml version="1.0" encoding="UTF-8"?>
<ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"> <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">
<ExternalURL>https://guest.rubenvandeven.com:8888/draw?id={HIT_NR}</ExternalURL> <ExternalURL>https://guest.rubenvandeven.com:8888/draw?id={HIT_NR}</ExternalURL>
@ -422,7 +425,6 @@ class CentralManagement():
self.currentHit.hit_id = new_hit['HIT']['HITId'] self.currentHit.hit_id = new_hit['HIT']['HITId']
self.store.saveHIT(self.currentHit) self.store.saveHIT(self.currentHit)
# TODO: have HITStore/HIT take care of this by emitting a signal
# self.server.statusPage.set('hit_id', new_hit['HIT']['HITId']) # self.server.statusPage.set('hit_id', new_hit['HIT']['HITId'])
# self.server.statusPage.set('hit_created', self.currentHit.created_at) # self.server.statusPage.set('hit_created', self.currentHit.created_at)
# self.server.statusPage.set('fee', f"${self.currentHit.fee:.2f}") # self.server.statusPage.set('fee', f"${self.currentHit.fee:.2f}")
@ -489,47 +491,62 @@ class CentralManagement():
""" """
Run scanimage on scaner and returns a string with the filename Run scanimage on scaner and returns a string with the filename
""" """
cmd = [
'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff',
'--resolution=100', # lower res, faster (more powerful) scan & wipe
'-l','25' #y axis, margin from top of the scanner, hence increasing this, moves the scanned image upwards
,'-t','22', # x axis, margin from left side scanner (seen from the outside)
'-x',str(181),
'-y',str(245)
]
self.logger.info(f"{cmd}")
filename = self.currentHit.getImagePath()
with self.scanLock: with self.scanLock:
if self.config['dummy_plotter']:
self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.start'))
self.logger.warning("Fake scanner for a few seconds")
for i in tqdm.tqdm(range(5)):
time.sleep(1)
self.eventQueue.put(Signal('hit.scanned', {'hit_id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.finished'))
return
cmd = [
'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff',
'--resolution=100', # lower res, faster (more powerful) scan & wipe
'-l','25' #y axis, margin from top of the scanner, hence increasing this, moves the scanned image upwards
,'-t','22', # x axis, margin from left side scanner (seen from the outside)
'-x',str(181),
'-y',str(245)
]
self.logger.info(f"{cmd}")
filename = self.currentHit.getImagePath()
self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id})) self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.start')) self.eventQueue.put(Signal('scan.start'))
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# opens connection to scanner, but only starts scanning when output becomes ready: # opens connection to scanner, but only starts scanning when output becomes ready:
o, e = proc.communicate(80) o, e = proc.communicate(80)
if e: if e:
self.logger.critical(f"Scanner caused: {e.decode()}") self.logger.critical(f"Scanner caused: {e.decode()}")
#TODO: should clear self.isRunning.clear() ? # Should this clear self.isRunning.clear() ?
try: try:
f = io.BytesIO(o) f = io.BytesIO(o)
img = Image.open(f) img = Image.open(f)
img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM) img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM)
tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma']) tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma'])
tunedImg.save(filename) tunedImg.save(filename)
except Exception as e: except Exception as e:
self.logger.critical("Cannot create image from scan. Did scanner work?") self.logger.critical("Cannot create image from scan. Did scanner work?")
self.logger.exception(e) self.logger.exception(e)
# TODO: create copyfile('www/basic.svg', filename)
copyfile('www/basic.svg', filename)
time.sleep(5) # sleep a few seconds for scanner to return to start position
time.sleep(5) # sleep a few seconds for scanner to return to start position
self.eventQueue.put(Signal('hit.scanned', {'hit_id':self.currentHit.id}))
self.eventQueue.put(Signal('hit.scanned', {'id':self.currentHit.id})) self.eventQueue.put(Signal('scan.finished'))
self.eventQueue.put(Signal('scan.finished'))
def setLight(self, on): def setLight(self, on):
value = 1 if on else 0 value = 1 if on else 0
if self.lightStatus == value:
return
self.lightStatus = value
cmd = [ cmd = [
'usbrelay', f'HURTM_1={value}' 'usbrelay', f'HURTM_1={value}'
] ]
@ -537,3 +554,17 @@ class CentralManagement():
code = subprocess.call(cmd) code = subprocess.call(cmd)
if code > 0: if code > 0:
self.logger.warning(f"Error on light change: {code}") self.logger.warning(f"Error on light change: {code}")
def updateLightHook(self, hit = None):
# ignore hit attribute, which comes from the HITstore
self.setLight(self.getLightStatus())
def getLightStatus(self) -> bool:
if not self.currentHit:
return False
a = self.currentHit.getLastAssignment()
if not a:
return False
if self.currentHit.plotted_at: # wait till plotter is done
return False
return True

182
sorteerhoed/webserver.py
View File

@ -21,6 +21,14 @@ import html
logger = logging.getLogger("sorteerhoed").getChild("webserver") logger = logging.getLogger("sorteerhoed").getChild("webserver")
class DateTimeEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime.datetime):
return o.isoformat(timespec='seconds')
return super().default(self, o)
class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler): class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler):
def set_extra_headers(self, path): def set_extra_headers(self, path):
"""For subclass to add extra headers to the response""" """For subclass to add extra headers to the response"""
@ -36,7 +44,6 @@ class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler):
self.set_header("Content-Type", "image/svg+xml") self.set_header("Content-Type", "image/svg+xml")
class WebSocketHandler(tornado.websocket.WebSocketHandler): class WebSocketHandler(tornado.websocket.WebSocketHandler):
""" """
Websocket from the workers Websocket from the workers
@ -66,12 +73,15 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.hit = self.store.currentHit self.hit = self.store.currentHit
self.assignment_id = str(self.get_query_argument('assignmentId'))
self.assignment_id = int(self.get_query_argument('assignment_id')) self.assignment = self.hit.getLastAssignment()
if self.assignment.assignment_id != self.assignment_id:
raise Exception(f"Opening websocket for invalid assignment {self.assignment_id}")
self.timeout = datetime.datetime.now() + datetime.timedelta(seconds=self.store.getHitTimeout()) self.timeout = datetime.datetime.now() + datetime.timedelta(seconds=self.store.getHitTimeout())
if self.hit.submit_hit_at: if self.hit.isSubmitted():
raise Exception("Opening websocket for already submitted hit") raise Exception("Opening websocket for already submitted hit")
#logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}") #logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}")
@ -85,6 +95,7 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
if self.assignment_id != self.hit.getLastAssignment().assignment_id: if self.assignment_id != self.hit.getLastAssignment().assignment_id:
logger.critical(f"Skip message for non-last assignment {message}") logger.critical(f"Skip message for non-last assignment {message}")
return
if datetime.datetime.now() > self.timeout: if datetime.datetime.now() > self.timeout:
logger.critical("Close websocket after timeout (abandon?)") logger.critical("Close websocket after timeout (abandon?)")
@ -93,7 +104,6 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
try: try:
msg = json.loads(message) msg = json.loads(message)
# TODO: sanitize input: min/max, limit strokes
if msg['action'] == 'move': if msg['action'] == 'move':
# TODO: min/max input # TODO: min/max input
point = [float(msg['direction'][0]),float(msg['direction'][1]), bool(msg['mouse'])] point = [float(msg['direction'][0]),float(msg['direction'][1]), bool(msg['mouse'])]
@ -127,8 +137,8 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.write_message(json.dumps({ self.write_message(json.dumps({
'action': 'submitted', 'action': 'submitted',
'msg': f"Submission ok, please copy this token to your HIT at Mechanical Turk: {self.hit.uuid}", 'msg': f"Submission ok, please copy this token to your HIT at Mechanical Turk: {self.assignment.uuid}",
'code': str(self.hit.uuid) 'code': str(self.assignment.uuid)
})) }))
self.close() self.close()
@ -136,8 +146,9 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
# not used, implicit in move? # not used, implicit in move?
pass pass
elif msg['action'] == 'info': elif msg['action'] == 'info':
self.eventQ.put(Signal('hit.info', dict( self.eventQ.put(Signal('assignment.info', dict(
hit_id=self.hit.id, hit_id=self.hit.id,
assignment_id=self.assignment_id,
resolution=msg['resolution'], resolution=msg['resolution'],
browser=msg['browser'] browser=msg['browser']
))) )))
@ -159,32 +170,35 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
if len(self.strokes) < 1: if len(self.strokes) < 1:
return False return False
self.eventQ.put(Signal("server.submit", dict(hit_id = self.hit.id))) self.eventQ.put(Signal("assignment.submit", dict(
hit_id = self.hit.id,
assignment_id=self.assignment_id)))
if self.config['dummy_plotter']: # deprecated: now done at scanner method:
d = strokes2D(self.strokes) # if self.config['dummy_plotter']:
svg = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?> # d = strokes2D(self.strokes)
<svg viewBox="0 0 600 600" # svg = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
xmlns:dc="http://purl.org/dc/elements/1.1/" # <svg viewBox="0 0 600 600"
xmlns:cc="http://creativecommons.org/ns#" # xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" # xmlns:cc="http://creativecommons.org/ns#"
xmlns:svg="http://www.w3.org/2000/svg" # xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://www.w3.org/2000/svg" # xmlns:svg="http://www.w3.org/2000/svg"
version="1.1" # xmlns="http://www.w3.org/2000/svg"
> # version="1.1"
<path d="{d}" style="stroke:black;stroke-width:2;fill:none;" /> # >
</svg> # <path d="{d}" style="stroke:black;stroke-width:2;fill:none;" />
""" # </svg>
# """
filename = self.hit.getImagePath() #
logger.info(f"Write to {filename}") # filename = self.hit.getImagePath()
with open(filename, 'w') as fp: # logger.info(f"Write to {filename}")
fp.write(svg) # with open(filename, 'w') as fp:
# fp.write(svg)
# we fake a hit.scanned event # we fake a hit.scanned event
self.eventQ.put(Signal('hit.scanned', {'hit_id':self.hit.id})) # self.eventQ.put(Signal('hit.scanned', {'hit_id':self.hit.id}))
return self.hit.uuid return self.assignment.uuid
@classmethod @classmethod
def rmConnection(cls, client): def rmConnection(cls, client):
@ -196,11 +210,9 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
class StatusWebSocketHandler(tornado.websocket.WebSocketHandler): class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
CORS_ORIGINS = ['localhost'] CORS_ORIGINS = ['localhost']
connections = set() connections = set()
queue = queue.Queue()
def initialize(self, statusPage): def initialize(self, statusPage):
self.statusPage = statusPage self.statusPage = statusPage
pass
def check_origin(self, origin): def check_origin(self, origin):
parsed_origin = urlparse(origin) parsed_origin = urlparse(origin)
@ -209,13 +221,9 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
return valid return valid
# the client connected # the client connected
def open(self, p = None): def open(self):
self.__class__.connections.add(self) self.__class__.connections.add(self)
for prop, value in self.statusPage.__dict__.items(): self.write_message(json.dumps(self.statusPage.fetch(), cls=DateTimeEncoder))
self.write_message(json.dumps({
'property': prop,
'value': value.isoformat(timespec='seconds') if type(value) is datetime.datetime else value
}))
# client disconnected # client disconnected
@ -230,13 +238,10 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
cls.connections.remove(client) cls.connections.remove(client)
@classmethod @classmethod
def update_for_all(cls, prop, value): def update_for_all(cls, data):
logger.debug(f"update for all {prop} {value}") logger.debug(f"update for all {data}")
for connection in cls.connections: for connection in cls.connections:
connection.write_message(json.dumps({ connection.write_message(json.dumps(data, cls=DateTimeEncoder))
'property': prop,
'value': value.isoformat(timespec='seconds') if type(value) is datetime.datetime else value
}))
def strokes2D(strokes): def strokes2D(strokes):
# strokes to a d attribute for a path # strokes to a d attribute for a path
@ -283,7 +288,7 @@ class DrawPageHandler(tornado.web.RequestHandler):
except Exception: except Exception:
self.write("HIT not found") self.write("HIT not found")
else: else:
if hit.submit_page_at: if hit.isSubmitted():
self.write("HIT already submitted") self.write("HIT already submitted")
return return
@ -294,6 +299,15 @@ class DrawPageHandler(tornado.web.RequestHandler):
previewOnly = False previewOnly = False
if assignmentId == 'ASSIGNMENT_ID_NOT_AVAILABLE': if assignmentId == 'ASSIGNMENT_ID_NOT_AVAILABLE':
previewOnly = True previewOnly = True
if len(assignmentId) and not previewOnly:
# process/create assignment
assignment = self.store.currentHit.getAssignmentById(assignmentId)
if not assignment:
# new assignment
logger.warning(f"Create new assignment {assignmentId}")
assignment = self.store.newAssignment(self.store.currentHit, assignmentId)
self.store.saveAssignment(assignment)
previous_hit = self.store.getLastSubmittedHit() previous_hit = self.store.getLastSubmittedHit()
if not previous_hit: if not previous_hit:
@ -330,7 +344,8 @@ class DrawPageHandler(tornado.web.RequestHandler):
self.eventQ.put(Signal('hit.assignment', dict( self.eventQ.put(Signal('hit.assignment', dict(
hit_id=hit.id, ip=ip, assignment_id=assignmentId hit_id=hit.id, ip=ip, assignment_id=assignmentId
))) )))
# self.eventQ.put(Signal('hit.info', dict(hit_id=hit.id, ip=ip)))
self.eventQ.put(Signal('assignment.info', dict(assignment_id=assignmentId, ip=ip)))
try: try:
geoip = self.geoip_reader.country(ip) geoip = self.geoip_reader.country(ip)
@ -383,50 +398,35 @@ class StatusPage():
""" """
Properties for on the status page, which are send over websockets the moment Properties for on the status page, which are send over websockets the moment
they are altered. they are altered.
""" """
def __init__(self):
self.reset() def __init__(self, store: HITStore):
self.store = store
def reset(self): self.store.registerUpdateHook(self)
logger.info("Resetting status")
self.hit_id = None def update(self, hit = None):
self.worker_id = None """
self.ip = None Send the given HIT formatted to the websocket clients
self.location = None
self.browser = None If no hit is given, load the last 2 items
self.os = None """
self.resolution = None if hit:
self.state = None data = [hit.toDict()]
self.fee = None else:
self.hit_created = None hits = self.store.getNewestHits(2)
self.hit_opened = None data = [hit.toDict() for hit in hits]
self.hit_submitted = None
def clearAssignment(self):
logger.info("Resetting hit assignment")
self.worker_id = None
self.ip = None
self.location = None
self.browser = None
self.os = None
self.resolution = None
self.hit_created = None
def __setattr__(self, name, value):
if name in self.__dict__ and self.__dict__[name] == value:
logger.debug(f"Ignore setting status of {name}: it already is set to {value}")
return
self.__dict__[name] =value
logger.info(f"Update status: {name}: {value}")
if Server.loop: if Server.loop:
Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, name, value) Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, data)
else: else:
logger.warn("Status: no server loop to call update command") logger.warn("Status: no server loop to call update command")
def fetch(self):
def set(self, name, value): """
return self.__setattr__(name, value) Fetch latest, used on connection of status page
"""
hits = self.store.getNewestHits(2)
return [hit.toDict() for hit in hits]
@ -434,8 +434,6 @@ class Server:
""" """
Server for HIT -> plotter events Server for HIT -> plotter events
As well as for the Status interface As well as for the Status interface
TODO: change to have the HIT_id as param to the page. Load hit from storage with previous image
""" """
loop = None loop = None
@ -452,7 +450,7 @@ class Server:
self.server_loop = None self.server_loop = None
self.store = store self.store = store
self.statusPage = StatusPage() self.statusPage = StatusPage(store)
def start(self): def start(self):
@ -489,6 +487,8 @@ class Server:
store = self.store, store = self.store,
path=self.web_root, path=self.web_root,
)), )),
(r"/frames/(.*)", StaticFileWithHeaderHandler,
{"path": 'scanimation/interfaces/frames'}),
(r"/(.*)", StaticFileWithHeaderHandler, (r"/(.*)", StaticFileWithHeaderHandler,
{"path": self.web_root}), {"path": self.web_root}),
], debug=True, autoreload=False) ], debug=True, autoreload=False)

2
www/assignment.js
View File

@ -51,7 +51,7 @@ let draw = function(e) {
strokeEl.setAttribute('d', d); strokeEl.setAttribute('d', d);
} }
console.log([pos['x'], pos['y']], isDrawing); //console.log([pos['x'], pos['y']], isDrawing);
socket.send(JSON.stringify({ socket.send(JSON.stringify({
'action': 'move', 'action': 'move',
'direction': [pos['x'], pos['y']], 'direction': [pos['x'], pos['y']],