104 lines
3.2 KiB
Python
104 lines
3.2 KiB
Python
|
import logging
|
||
|
|
from sqlalchemy.ext.declarative import declarative_base
|
||
|
|
from sqlalchemy import Column, Integer, String, DateTime
|
||
|
|
from sqlalchemy.orm import relationship
|
||
|
|
from sqlalchemy.sql.schema import ForeignKey, Sequence
|
||
|
|
from sqlalchemy.engine import create_engine
|
||
|
|
from sqlalchemy.orm.session import sessionmaker
|
||
|
|
import datetime
|
||
|
|
from contextlib import contextmanager
|
||
|
|
import uuid
|
||
|
|
import os
|
||
|
|
import coloredlogs
|
||
|
|
import argparse
|
||
|
|
from sqlalchemy.sql.functions import func
|
||
|
|
|
||
|
|
mainLogger = logging.getLogger("sorteerhoed")
|
||
|
|
logger = mainLogger.getChild("store")
|
||
|
|
|
||
|
|
Base = declarative_base()
|
||
|
|
|
||
|
|
"""
|
||
|
|
HIT lifetime:
|
||
|
|
|
||
|
|
|
||
|
|
created
|
||
|
|
accepted
|
||
|
|
(returned!)
|
||
|
|
working
|
||
|
|
awaiting amazon confirmation (submitted on page)
|
||
|
|
submitted
|
||
|
|
|
||
|
|
Actions:
|
||
|
|
creating Hit (creating hit with scanned image)
|
||
|
|
|
||
|
|
Scanning
|
||
|
|
|
||
|
|
"""
|
||
|
|
|
||
|
|
class HIT(Base):
|
||
|
|
__tablename__ = 'hits'
|
||
|
|
id = Column(Integer, Sequence('hit_id'), primary_key=True) # our sequential hit id
|
||
|
|
hit_id = Column(String(255)) # amazon's hit id
|
||
|
|
created_at = Column(DateTime, default=datetime.datetime.now())
|
||
|
|
updated_at = Column(DateTime, default=datetime.datetime.now())
|
||
|
|
uniqid = Column(String(32), default=uuid.uuid4().hex)
|
||
|
|
assignment_id = Column(String(255), default = None)
|
||
|
|
worker_id = Column(String(255), default = None)
|
||
|
|
accept_time = Column(DateTime, default=None)
|
||
|
|
open_page_at = Column(DateTime, default=None)
|
||
|
|
submit_page_at = Column(DateTime, default=None)
|
||
|
|
submit_hit_at = Column(DateTime, default=None)
|
||
|
|
answer = Column(String(255), default=None)
|
||
|
|
turk_ip = Column(String(255), default=None)
|
||
|
|
turk_country = Column(String(255), default=None)
|
||
|
|
turk_screen_width = Column(Integer, default = None)
|
||
|
|
turk_screen_height = Column(Integer, default = None)
|
||
|
|
|
||
|
|
|
||
|
|
class Store:
|
||
|
|
def __init__(self, db_filename, logLevel=0):
|
||
|
|
path = os.path.abspath(db_filename)
|
||
|
|
if logLevel <= logging.DEBUG:
|
||
|
|
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
|
||
|
|
|
||
|
|
self.engine = create_engine('sqlite:///'+path, echo=False)
|
||
|
|
Base.metadata.create_all(self.engine)
|
||
|
|
self.Session = sessionmaker(bind=self.engine)
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def getSession(self):
|
||
|
|
"""Provide a transactional scope around a series of operations."""
|
||
|
|
session = self.Session()
|
||
|
|
try:
|
||
|
|
yield session
|
||
|
|
session.commit()
|
||
|
|
except:
|
||
|
|
session.rollback()
|
||
|
|
raise
|
||
|
|
finally:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
def getHits(self, session):
|
||
|
|
return session.query(Source).order_by(HIT.created_at.desc())
|
||
|
|
|
||
|
|
def addHIT(self, hit: HIT):
|
||
|
|
with self.getSession() as s:
|
||
|
|
s.add(hit)
|
||
|
|
s.flush()
|
||
|
|
s.refresh(hit)
|
||
|
|
logging.info(f"Added {hit.id}")
|
||
|
|
|
||
|
|
# def rmSource(self, id: int):
|
||
|
|
# with self.getSession() as session:
|
||
|
|
# source = session.query(Source).get(id)
|
||
|
|
# if not source:
|
||
|
|
# logging.warning(f"Source nr {id} not found")
|
||
|
|
# else:
|
||
|
|
# logging.info(f"Deleting source {source.id}: {source.url}")
|
||
|
|
# session.delete(source)
|
||
|
|
#
|
||
|
|
# def getRandomNewsItem(self, session) -> NewsItem:
|
||
|
|
# return session.query(NewsItem).order_by(func.random()).limit(1).first()
|
||
|
|
|