guest_worker/sorteerhoed/central_management.py

import logging
import yaml
from sorteerhoed import HITStore
import os
import subprocess
import boto3
import threading
from queue import Queue
from sorteerhoed.plotter import Plotter
import queue
from sorteerhoed.sqs import SqsListener
from sorteerhoed.webserver import Server
import time


class CentralManagement():
    """
    Central management reads config and controls process flow
    
    The HIT Store is the archive of hits
    mturk thread communicates with mturk
    server thread is tornado communicating with the turkers and with the status interface on the installation
    Plotter thread reads plotter queue and sends it to there
    Scanner is for now a simpe imagescan command
    SQS: thread that listens for updates from Amazon
    """
    def __init__(self, debug_mode):
        self.logger = logging.getLogger("sorteerhoed").getChild('management')
        self.debug = debug_mode
        self.currentHit = None
        
        self.eventQueue = Queue()
        self.isRunning = threading.Event()

        
    def loadConfig(self, filename):
        with open(filename, 'r') as fp:
            self.logger.debug('Load config from {}'.format(filename))
            self.config = yaml.safe_load(fp)
        
        varDb = os.path.join(
#             self.config['storage_dir'],
            'hit_store.db'
            )            
        self.store = HITStore.Store(varDb, logLevel=logging.DEBUG if self.debug else logging.INFO)
        
        self.logger.debug(f"Loaded configuration: {self.config}")
#         self.amazon = 
#         self.server


#         self.panopticon = Panopticon(self, self.config, self.voiceStorage)
    def start(self):
        self.isRunning.set()
        
        try:
            
            # M-turk connection
            MTURK_SANDBOX = 'https://mturk-requester-sandbox.us-east-1.amazonaws.com'
            # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/mturk.html#MTurk.Client
            self.mturk = boto3.client('mturk',
               aws_access_key_id = self.config['amazon']['user_id'],
               aws_secret_access_key = self.config['amazon']['user_secret'],
               region_name='us-east-1',
               endpoint_url = MTURK_SANDBOX
            )
            
            self.logger.info(f"Mechanical turk: {self.mturk.get_account_balance()}")
            
            
            
            self.sqs = SqsListener(self.config, self.eventQueue, self.isRunning)
            sqsThread = threading.Thread(target=self.sqs.start)
            sqsThread.start()
            
            # the plotter itself
            self.plotter = Plotter(self.config, self.eventQueue, self.isRunning)
            plotterThread = threading.Thread(target=self.plotter.start)
            plotterThread.start()
            
            # webserver for turks and status
            self.server = Server(self.config, self.eventQueue, self.isRunning, self.plotter.q)
            serverThread = threading.Thread(target=self.server.start)
            serverThread.start()
            
            # event listener:
            dispatcherThread = threading.Thread(target=self.eventListener)
            dispatcherThread.start()
            
            self.makeHit()
            
        finally:
            self.isRunning.clear()
            self.server.stop()
            
    
    def eventListener(self):
        while self.isRunning.is_set():
            try:
                signal = self.eventQueue.get(True, 1)
            except queue.Empty:
                pass
#                 self.logger.log(5, "Empty queue.")
            else:
                """
                Possible events:
                - SQS events: accept/abandoned/returned/submitted
                - webserver events: open, draw, submit
                - scan complete
                - HIT created
                - Plotter complete
                - 
                """
                print(signal)
                # handle singals/events:
                # TODO: next steps
                # TODO: update status
                
    
    def makeHit(self):
        self.currentHit = HITStore.HIT()
        self.store.addHIT(self.currentHit)
        
        self.logger(f"Make HIT {self.currentHit.id}")
        
        question = open(self.config['amazon']['task_xml'], mode='r').read().replace("{HIT_NR}",self.currentHit.id)
        new_hit = self.mturk.create_hit(
            Title = 'Trace the drawn line',
            Description = 'Draw a line over the sketched line in the image',
            Keywords = 'polygons, trace, draw',
            Reward = '0.15', # TODO: make variable
            MaxAssignments = 1,
            LifetimeInSeconds = self.config['hit_lifetime'],
            AssignmentDurationInSeconds = self.config['hit_assignment_duration'],
            AutoApprovalDelayInSeconds = self.config['hit_autoapprove_delay'],
            Question = question,
        )
         
        self.logger.info("Created hit:", new_hit)
        self.logger.info("https://workersandbox.mturk.com/mturk/preview?groupId=" + new_hit['HIT']['HITGroupId'])
        
        self.currentHit.hit_id = new_hit['HIT']['HITId']
        
        print(self.currentHit)
        
    #     mturk.send_test_event_notification()
        if self.config['amazon']['sqs_url']:
            notification_info= self.mturk.update_notification_settings(
                HITTypeId=new_hit['HIT']['HITTypeId'],
                Notification = {
                    'Destination' : self.config['amazon']['sqs_url'],
                    'Transport': 'SQS',
                    'Version': '2014-08-15',
                    'EventTypes': [
                            'AssignmentAccepted',
                            'AssignmentAbandoned',
                            'AssignmentReturned',
                            'AssignmentSubmitted',
#                             'AssignmentRejected',
        #                     'AssignmentApproved',
#                             'HITCreated',
        #                     'HITExpired',
        #                     'HITReviewable',
        #                     'HITExtended',
        #                     'HITDisposed',
        #                     'Ping',
                    ]
                },
                Active=True
            )
            self.logger.debug(notification_info)
    
    def scanImage(self) -> str:
        """
        Run scanimage on scaner and returns a string with the filename
        """
        cmd = [
            'sudo', 'scanimage'
        ]
        filename = ""
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        o, e = proc.communicate(60)
        
        exec