Compare commits

...

50 commits

Author SHA1 Message Date
mt
75e0100144 backup script and minor config changes 2022-10-24 10:59:19 +02:00
Ruben van de Ven
34b7a1c280 Fix country conversion for unknown 2021-10-20 10:17:37 +02:00
Ruben van de Ven
752abb0cc6 small cp changes 2021-10-20 10:03:45 +02:00
Ruben van de Ven
caf7b59497 dynamic domain 2021-10-17 10:39:37 +02:00
Ruben van de Ven
6b861a3a65 fix opions bug 2021-10-17 10:37:08 +02:00
Ruben van de Ven
d4372083c5 fix opions bug 2021-10-17 10:35:50 +02:00
Ruben van de Ven
072db30611 Sorteerhoed service 2021-10-17 10:27:53 +02:00
Ruben van de Ven
7cc0ee36fa Too new python 2021-10-17 10:25:42 +02:00
Ruben van de Ven
58df48cd91 Prep as service, with buttons to start 2021-10-17 10:24:02 +02:00
Ruben van de Ven
ac0176b228 Scanimation service 2021-10-17 10:20:14 +02:00
Ruben van de Ven
e1150dba24 Toch weer 4kant... voor nu 2020-09-08 14:58:59 +02:00
merijn van moll
2db13ed87f nieuwe start-afbeelding 2020-08-31 14:36:57 +02:00
Ruben van de Ven
d342572a23 Save quality 2020-01-23 18:23:21 +01:00
mt
ba33edbc3d Insitu changes 2020-01-23 13:24:53 +01:00
Ruben van de Ven
a8b981eb67 Disable block 2020-01-22 21:10:22 +01:00
Ruben van de Ven
9b007fabff Change text 2020-01-22 21:04:53 +01:00
Ruben van de Ven
c49d254fe8 cursor fixed? 2020-01-22 20:42:55 +01:00
Ruben van de Ven
9137006c60 Fix return/reject terminology 2020-01-22 20:40:08 +01:00
Ruben van de Ven
bbd5c20d3b Worker id 2020-01-22 20:04:33 +01:00
Ruben van de Ven
a9f82a5feb WIP. fix for ambiguous assignment id 2020-01-22 19:58:33 +01:00
Ruben van de Ven
ac96bbff64 Merge branch 'master' of git.rubenvandeven.com:r/guest_worker 2020-01-22 19:07:20 +01:00
Ruben van de Ven
cd1ede252f Wip. Fix for unambiguous assignment id 2020-01-22 19:07:07 +01:00
mt
2d537e93f8 Merge branch 'master' of git.rubenvandeven.com:r/guest_worker 2020-01-22 18:39:01 +01:00
mt
94878d497e Aligning items 2020-01-22 18:38:59 +01:00
Ruben van de Ven
74f6ff55a3 Disable light on abandon 2020-01-22 18:38:33 +01:00
mt
097b3ee5e4 Merge branch 'master' of git.rubenvandeven.com:r/guest_worker 2020-01-22 18:31:12 +01:00
mt
eb66f137ac position 2020-01-22 18:31:09 +01:00
Ruben van de Ven
e570a2e9f7 Fixes for worker specs 2020-01-22 18:30:06 +01:00
Ruben van de Ven
eea090d265 Merge branch 'master' of git.rubenvandeven.com:r/guest_worker 2020-01-22 18:15:56 +01:00
Ruben van de Ven
802b93fa95 Hopefully better handling of abandons 2020-01-22 18:15:47 +01:00
merijn van moll
573a8e57bc pijl iets anders 2020-01-22 17:40:17 +01:00
merijn van moll
cb64077d95 Merge branch 'master' of https://git.rubenvandeven.com/r/guest_worker 2020-01-22 17:32:21 +01:00
merijn van moll
65edb5d6c9 paddings 2020-01-22 17:31:58 +01:00
mt
de12ac714a Changes to backend 2020-01-22 17:05:38 +01:00
merijn van moll
1bc8d09df6 Merge branch 'master' of https://git.rubenvandeven.com/r/guest_worker 2020-01-22 17:01:11 +01:00
Ruben van de Ven
da36fbf6fe worker_id filter for credit roll 2020-01-22 16:58:07 +01:00
Ruben van de Ven
b448d0ddcd Merge branch 'master' of git.rubenvandeven.com:r/guest_worker 2020-01-22 16:18:29 +01:00
Ruben van de Ven
534d471e6d Fix worker specs 2020-01-22 16:18:09 +01:00
merijn van moll
3494f490a5 test weggehaald 2020-01-22 16:16:28 +01:00
merijn van moll
087542321a Merge branch 'master' of https://git.rubenvandeven.com/r/guest_worker 2020-01-22 16:12:03 +01:00
merijn van moll
086c54fca0 fontmap 2020-01-22 16:11:28 +01:00
Ruben van de Ven
795a4c59ec Merge branch 'master' of git.rubenvandeven.com:r/guest_worker 2020-01-22 16:07:31 +01:00
Ruben van de Ven
c153ff55f9 Specs formatting 2020-01-22 16:07:04 +01:00
Ruben van de Ven
c97ac4b05d Distance traveled in worker specs 2020-01-22 16:04:13 +01:00
merijn van moll
4990a87f33 credits typografie 2020-01-22 16:00:21 +01:00
Ruben van de Ven
54bb2787bc Backend as credit fades 2020-01-22 14:36:52 +01:00
merijn van moll
08f2e4fd3e worker_specs kleine aanpassingen 2020-01-22 12:08:57 +01:00
merijn van moll
81c2c5ec80 interface typography 2020-01-21 16:09:24 +01:00
Ruben van de Ven
b2626244a4 New HIT status page 2020-01-13 16:13:42 +01:00
Ruben van de Ven
d3bf3d47ea Refactored to work with separate assignment table 2020-01-13 13:49:59 +01:00
37 changed files with 1629 additions and 518 deletions

4
.gitignore vendored
View file

@ -5,6 +5,8 @@ node_modules
.project .project
config.local.yml config.local.yml
hit_store.db hit_store.db
backup/*
www/scans/*.svg www/scans/*.svg
#scanimation/interfa #scanimation/interfa
GeoLite2-Country.mmdb

View file

@ -17,6 +17,8 @@ Pillow = "*"
tqdm = "*" tqdm = "*"
serial = "*" serial = "*"
pyserial = "*" pyserial = "*"
country-converter = "*"
svgpathtools = "*"
[dev-packages] [dev-packages]

119
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "3c0954c7917f567561faffcf18031aa0718c5144698d0de7022dd9ad9d49b1c4" "sha256": "d1d2c21498d8c771cdd3233e304f260e7fe6020dfb6a1529286e85e8ead0f34f"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -21,18 +21,18 @@
}, },
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:98fdd6fa754e17c0d9e87fbb464c43c7e72aaa6b4f78b418eba47b7d15deffee", "sha256:5222edc5b20d5c6ab7440fc4f89f987ead05be37ff5cc5359a3b9148d9b5a51e",
"sha256:fdaae8edd63ae114107d375862069d17de23e00489a65169b8141ddee6bdf78b" "sha256:bd3337cfc15613b0091fa567dc3065d94df88e5837ba1adbb1e35b91db728a66"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.10.48" "version": "==1.11.7"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:29370f50af7870661609fbfbc4ed01ef2fd531b87b98729700526d1a4b3a2f89", "sha256:9a17d36ee43f1398c7db3cb29aa2216de94bcb60f058b1c645d71e72a330ddf8",
"sha256:7f60edf33c6f5b7c1c9b9377267bdc56495f52704607f713d4c3bd1d82a08334" "sha256:e4b82b1a7389f3d16732eb839240c9d3e42470100d5a71415ea2a0a35b911b23"
], ],
"version": "==1.13.48" "version": "==1.14.7"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
@ -56,6 +56,13 @@
"index": "pypi", "index": "pypi",
"version": "==10.0" "version": "==10.0"
}, },
"country-converter": {
"hashes": [
"sha256:bc01ba2592b77a78b4f3e6f76600ca27852d71d1512cf1f320fecbcaaea3c6f9"
],
"index": "pypi",
"version": "==0.6.7"
},
"docutils": { "docutils": {
"hashes": [ "hashes": [
"sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0",
@ -159,6 +166,56 @@
], ],
"version": "==1.5.2" "version": "==1.5.2"
}, },
"numpy": {
"hashes": [
"sha256:1786a08236f2c92ae0e70423c45e1e62788ed33028f94ca99c4df03f5be6b3c6",
"sha256:17aa7a81fe7599a10f2b7d95856dc5cf84a4eefa45bc96123cbbc3ebc568994e",
"sha256:20b26aaa5b3da029942cdcce719b363dbe58696ad182aff0e5dcb1687ec946dc",
"sha256:2d75908ab3ced4223ccba595b48e538afa5ecc37405923d1fea6906d7c3a50bc",
"sha256:39d2c685af15d3ce682c99ce5925cc66efc824652e10990d2462dfe9b8918c6a",
"sha256:56bc8ded6fcd9adea90f65377438f9fea8c05fcf7c5ba766bef258d0da1554aa",
"sha256:590355aeade1a2eaba17617c19edccb7db8d78760175256e3cf94590a1a964f3",
"sha256:70a840a26f4e61defa7bdf811d7498a284ced303dfbc35acb7be12a39b2aa121",
"sha256:77c3bfe65d8560487052ad55c6998a04b654c2fbc36d546aef2b2e511e760971",
"sha256:9537eecf179f566fd1c160a2e912ca0b8e02d773af0a7a1120ad4f7507cd0d26",
"sha256:9acdf933c1fd263c513a2df3dceecea6f3ff4419d80bf238510976bf9bcb26cd",
"sha256:ae0975f42ab1f28364dcda3dde3cf6c1ddab3e1d4b2909da0cb0191fa9ca0480",
"sha256:b3af02ecc999c8003e538e60c89a2b37646b39b688d4e44d7373e11c2debabec",
"sha256:b6ff59cee96b454516e47e7721098e6ceebef435e3e21ac2d6c3b8b02628eb77",
"sha256:b765ed3930b92812aa698a455847141869ef755a87e099fddd4ccf9d81fffb57",
"sha256:c98c5ffd7d41611407a1103ae11c8b634ad6a43606eca3e2a5a269e5d6e8eb07",
"sha256:cf7eb6b1025d3e169989416b1adcd676624c2dbed9e3bcb7137f51bfc8cc2572",
"sha256:d92350c22b150c1cae7ebb0ee8b5670cc84848f6359cf6b5d8f86617098a9b73",
"sha256:e422c3152921cece8b6a2fb6b0b4d73b6579bd20ae075e7d15143e711f3ca2ca",
"sha256:e840f552a509e3380b0f0ec977e8124d0dc34dc0e68289ca28f4d7c1d0d79474",
"sha256:f3d0a94ad151870978fb93538e95411c83899c9dc63e6fb65542f769568ecfa5"
],
"version": "==1.18.1"
},
"pandas": {
"hashes": [
"sha256:00dff3a8e337f5ed7ad295d98a31821d3d0fe7792da82d78d7fd79b89c03ea9d",
"sha256:22361b1597c8c2ffd697aa9bf85423afa9e1fcfa6b1ea821054a244d5f24d75e",
"sha256:255920e63850dc512ce356233081098554d641ba99c3767dde9e9f35630f994b",
"sha256:26382aab9c119735908d94d2c5c08020a4a0a82969b7e5eefb92f902b3b30ad7",
"sha256:33970f4cacdd9a0ddb8f21e151bfb9f178afb7c36eb7c25b9094c02876f385c2",
"sha256:4545467a637e0e1393f7d05d61dace89689ad6d6f66f267f86fff737b702cce9",
"sha256:52da74df8a9c9a103af0a72c9d5fdc8e0183a90884278db7f386b5692a2220a4",
"sha256:61741f5aeb252f39c3031d11405305b6d10ce663c53bc3112705d7ad66c013d0",
"sha256:6a3ac2c87e4e32a969921d1428525f09462770c349147aa8e9ab95f88c71ec71",
"sha256:7458c48e3d15b8aaa7d575be60e1e4dd70348efcd9376656b72fecd55c59a4c3",
"sha256:78bf638993219311377ce9836b3dc05f627a666d0dbc8cec37c0ff3c9ada673b",
"sha256:8153705d6545fd9eb6dd2bc79301bff08825d2e2f716d5dced48daafc2d0b81f",
"sha256:975c461accd14e89d71772e89108a050fa824c0b87a67d34cedf245f6681fc17",
"sha256:9962957a27bfb70ab64103d0a7b42fa59c642fb4ed4cb75d0227b7bb9228535d",
"sha256:adc3d3a3f9e59a38d923e90e20c4922fc62d1e5a03d083440468c6d8f3f1ae0a",
"sha256:bbe3eb765a0b1e578833d243e2814b60c825b7fdbf4cdfe8e8aae8a08ed56ecf",
"sha256:df8864824b1fe488cf778c3650ee59c3a0d8f42e53707de167ba6b4f7d35f133",
"sha256:e45055c30a608076e31a9fcd780a956ed3b1fa20db61561b8d88b79259f526f7",
"sha256:ee50c2142cdcf41995655d499a157d0a812fce55c97d9aad13bc1eef837ed36c"
],
"version": "==0.25.3"
},
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be", "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be",
@ -187,6 +244,13 @@
"index": "pypi", "index": "pypi",
"version": "==7.0.0" "version": "==7.0.0"
}, },
"pyparsing": {
"hashes": [
"sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
"sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
],
"version": "==2.4.6"
},
"pyserial": { "pyserial": {
"hashes": [ "hashes": [
"sha256:6e2d401fdee0eab996cf734e67773a0143b932772ca8b42451440cfed942c627", "sha256:6e2d401fdee0eab996cf734e67773a0143b932772ca8b42451440cfed942c627",
@ -200,7 +264,6 @@
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
], ],
"markers": "python_version >= '2.7'",
"version": "==2.8.1" "version": "==2.8.1"
}, },
"python-magic": { "python-magic": {
@ -211,6 +274,13 @@
"index": "pypi", "index": "pypi",
"version": "==0.4.15" "version": "==0.4.15"
}, },
"pytz": {
"hashes": [
"sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
"sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
],
"version": "==2019.3"
},
"pyyaml": { "pyyaml": {
"hashes": [ "hashes": [
"sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6",
@ -237,10 +307,10 @@
}, },
"s3transfer": { "s3transfer": {
"hashes": [ "hashes": [
"sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", "sha256:248dffd2de2dfb870c507b412fc22ed37cd3255293e293c395158e7c55fbe5f9",
"sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" "sha256:80ed96731b3bd77395cd6197246069092015e1124164b2c152c8f741a823dd04"
], ],
"version": "==0.2.1" "version": "==0.3.1"
}, },
"serial": { "serial": {
"hashes": [ "hashes": [
@ -252,10 +322,10 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
], ],
"version": "==1.13.0" "version": "==1.14.0"
}, },
"sqlalchemy": { "sqlalchemy": {
"hashes": [ "hashes": [
@ -264,6 +334,21 @@
"index": "pypi", "index": "pypi",
"version": "==1.3.12" "version": "==1.3.12"
}, },
"svgpathtools": {
"hashes": [
"sha256:7f7bdafe2c03b312178460104705e1d554d8cf36c898bec41bdce9fed3504746",
"sha256:e4b3784ae41b725fbce6a33a8981210967b16d0b557cb5d98c0ed0c81f0f89b9"
],
"index": "pypi",
"version": "==1.3.3"
},
"svgwrite": {
"hashes": [
"sha256:11e47749b159ed7004721e11d380b4642a26154b8cb2f7b0102fea9c71a3dfa1",
"sha256:50fec23dc3fd49103808f0d672124f8c573ec5899da5686df734f856b8d3b737"
],
"version": "==1.3.1"
},
"tornado": { "tornado": {
"hashes": [ "hashes": [
"sha256:349884248c36801afa19e342a77cc4458caca694b0eda633f5878e458a44cb2c", "sha256:349884248c36801afa19e342a77cc4458caca694b0eda633f5878e458a44cb2c",
@ -287,10 +372,10 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
"sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
], ],
"version": "==1.25.7" "version": "==1.25.8"
} }
}, },
"develop": {} "develop": {}

36
backup_and_reset.sh Executable file
View file

@ -0,0 +1,36 @@
#!/bin/bash
BACKUP_DIR="backup/$1"
if [ -d "$BACKUP_DIR" ]; then
>&2 echo "$BACKUP_DIR already exists. (usage: ./backup_and_reset.sh BACKUP_NAME)"
exit 1
fi
read -p "If you continue current config will be copied to '$BACKUP_DIR'. Continue? [y/n]" -r
echo # (optional) move to a new line
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
echo "Cancelling"
exit 1
fi
mkdir "$BACKUP_DIR"
cp www/scans "$BACKUP_DIR/" -Rv
cp hit_store.db "$BACKUP_DIR/"
cp scanimation/interfaces/frames "$BACKUP_DIR/" -Rv
read -p "Reset or keep original files. WARNING, type yes for reset [y/n]" -r
echo # (optional) move to a new line
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
echo "Exit without resetting."
exit 1
fi
# reset
rm hit_store.db
rm scanimation/interfaces/frames/*.jpg
rm www/scans/*.svg
echo "Done resetting files."

14
scanimation.service Normal file
View file

@ -0,0 +1,14 @@
[Unit]
Description=GuestWorker Scanimation server
After=network.target
[Service]
WorkingDirectory=/home/mt/guest_worker/scanimation
ExecStart=/usr/bin/node run.js
User=mt
Restart=Always
RestartSec=10
[Install]
WantedBy=multi-user.target

View file

@ -31,6 +31,7 @@ let LocalServer = (config) => {
socket.on('animationInit', function(){ socket.on('animationInit', function(){
fs.readdir(config.frames_folder, (err, files) => { fs.readdir(config.frames_folder, (err, files) => {
files = files.filter(file => file.endsWith('.jpg'))
socket.emit('frameData', {frames: files}) socket.emit('frameData', {frames: files})
console.log('starting animation') console.log('starting animation')
}) })
@ -38,6 +39,7 @@ let LocalServer = (config) => {
watch(config.frames_folder, function(e, name){ watch(config.frames_folder, function(e, name){
fs.readdir(config.frames_folder, (err, files) => { fs.readdir(config.frames_folder, (err, files) => {
files = files.filter(file => file.endsWith('.jpg'))
socket.emit('frameData', {frames: files}) socket.emit('frameData', {frames: files})
console.log(`frames changed => ${name}`) console.log(`frames changed => ${name}`)
}) })

View file

@ -9,7 +9,7 @@
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
"integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
"requires": { "requires": {
"mime-types": "~2.1.24", "mime-types": "2.1.24",
"negotiator": "0.6.2" "negotiator": "0.6.2"
} }
}, },
@ -67,15 +67,15 @@
"integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
"requires": { "requires": {
"bytes": "3.1.0", "bytes": "3.1.0",
"content-type": "~1.0.4", "content-type": "1.0.4",
"debug": "2.6.9", "debug": "2.6.9",
"depd": "~1.1.2", "depd": "1.1.2",
"http-errors": "1.7.2", "http-errors": "1.7.2",
"iconv-lite": "0.4.24", "iconv-lite": "0.4.24",
"on-finished": "~2.3.0", "on-finished": "2.3.0",
"qs": "6.7.0", "qs": "6.7.0",
"raw-body": "2.4.0", "raw-body": "2.4.0",
"type-is": "~1.6.17" "type-is": "1.6.18"
}, },
"dependencies": { "dependencies": {
"debug": { "debug": {
@ -146,7 +146,7 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": { "requires": {
"ms": "^2.1.1" "ms": "2.1.2"
} }
}, },
"depd": { "depd": {
@ -174,12 +174,12 @@
"resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.4.0.tgz", "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.4.0.tgz",
"integrity": "sha512-XCyYVWzcHnK5cMz7G4VTu2W7zJS7SM1QkcelghyIk/FmobWBtXE7fwhBusEKvCSqc3bMh8fNFMlUkCKTFRxH2w==", "integrity": "sha512-XCyYVWzcHnK5cMz7G4VTu2W7zJS7SM1QkcelghyIk/FmobWBtXE7fwhBusEKvCSqc3bMh8fNFMlUkCKTFRxH2w==",
"requires": { "requires": {
"accepts": "~1.3.4", "accepts": "1.3.7",
"base64id": "2.0.0", "base64id": "2.0.0",
"cookie": "0.3.1", "cookie": "0.3.1",
"debug": "~4.1.0", "debug": "4.1.1",
"engine.io-parser": "~2.2.0", "engine.io-parser": "2.2.0",
"ws": "^7.1.2" "ws": "7.1.2"
} }
}, },
"engine.io-client": { "engine.io-client": {
@ -189,14 +189,14 @@
"requires": { "requires": {
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"component-inherit": "0.0.3", "component-inherit": "0.0.3",
"debug": "~4.1.0", "debug": "4.1.1",
"engine.io-parser": "~2.2.0", "engine.io-parser": "2.2.0",
"has-cors": "1.1.0", "has-cors": "1.1.0",
"indexof": "0.0.1", "indexof": "0.0.1",
"parseqs": "0.0.5", "parseqs": "0.0.5",
"parseuri": "0.0.5", "parseuri": "0.0.5",
"ws": "~6.1.0", "ws": "6.1.4",
"xmlhttprequest-ssl": "~1.5.4", "xmlhttprequest-ssl": "1.5.5",
"yeast": "0.1.2" "yeast": "0.1.2"
}, },
"dependencies": { "dependencies": {
@ -205,7 +205,7 @@
"resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz", "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
"integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==", "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
"requires": { "requires": {
"async-limiter": "~1.0.0" "async-limiter": "1.0.1"
} }
} }
} }
@ -216,10 +216,10 @@
"integrity": "sha512-6I3qD9iUxotsC5HEMuuGsKA0cXerGz+4uGcXQEkfBidgKf0amsjrrtwcbwK/nzpZBxclXlV7gGl9dgWvu4LF6w==", "integrity": "sha512-6I3qD9iUxotsC5HEMuuGsKA0cXerGz+4uGcXQEkfBidgKf0amsjrrtwcbwK/nzpZBxclXlV7gGl9dgWvu4LF6w==",
"requires": { "requires": {
"after": "0.8.2", "after": "0.8.2",
"arraybuffer.slice": "~0.0.7", "arraybuffer.slice": "0.0.7",
"base64-arraybuffer": "0.1.5", "base64-arraybuffer": "0.1.5",
"blob": "0.0.5", "blob": "0.0.5",
"has-binary2": "~1.0.2" "has-binary2": "1.0.3"
} }
}, },
"escape-html": { "escape-html": {
@ -237,36 +237,36 @@
"resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
"integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
"requires": { "requires": {
"accepts": "~1.3.7", "accepts": "1.3.7",
"array-flatten": "1.1.1", "array-flatten": "1.1.1",
"body-parser": "1.19.0", "body-parser": "1.19.0",
"content-disposition": "0.5.3", "content-disposition": "0.5.3",
"content-type": "~1.0.4", "content-type": "1.0.4",
"cookie": "0.4.0", "cookie": "0.4.0",
"cookie-signature": "1.0.6", "cookie-signature": "1.0.6",
"debug": "2.6.9", "debug": "2.6.9",
"depd": "~1.1.2", "depd": "1.1.2",
"encodeurl": "~1.0.2", "encodeurl": "1.0.2",
"escape-html": "~1.0.3", "escape-html": "1.0.3",
"etag": "~1.8.1", "etag": "1.8.1",
"finalhandler": "~1.1.2", "finalhandler": "1.1.2",
"fresh": "0.5.2", "fresh": "0.5.2",
"merge-descriptors": "1.0.1", "merge-descriptors": "1.0.1",
"methods": "~1.1.2", "methods": "1.1.2",
"on-finished": "~2.3.0", "on-finished": "2.3.0",
"parseurl": "~1.3.3", "parseurl": "1.3.3",
"path-to-regexp": "0.1.7", "path-to-regexp": "0.1.7",
"proxy-addr": "~2.0.5", "proxy-addr": "2.0.5",
"qs": "6.7.0", "qs": "6.7.0",
"range-parser": "~1.2.1", "range-parser": "1.2.1",
"safe-buffer": "5.1.2", "safe-buffer": "5.1.2",
"send": "0.17.1", "send": "0.17.1",
"serve-static": "1.14.1", "serve-static": "1.14.1",
"setprototypeof": "1.1.1", "setprototypeof": "1.1.1",
"statuses": "~1.5.0", "statuses": "1.5.0",
"type-is": "~1.6.18", "type-is": "1.6.18",
"utils-merge": "1.0.1", "utils-merge": "1.0.1",
"vary": "~1.1.2" "vary": "1.1.2"
}, },
"dependencies": { "dependencies": {
"cookie": { "cookie": {
@ -295,12 +295,12 @@
"integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
"requires": { "requires": {
"debug": "2.6.9", "debug": "2.6.9",
"encodeurl": "~1.0.2", "encodeurl": "1.0.2",
"escape-html": "~1.0.3", "escape-html": "1.0.3",
"on-finished": "~2.3.0", "on-finished": "2.3.0",
"parseurl": "~1.3.3", "parseurl": "1.3.3",
"statuses": "~1.5.0", "statuses": "1.5.0",
"unpipe": "~1.0.0" "unpipe": "1.0.0"
}, },
"dependencies": { "dependencies": {
"debug": { "debug": {
@ -346,10 +346,10 @@
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
"integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
"requires": { "requires": {
"depd": "~1.1.2", "depd": "1.1.2",
"inherits": "2.0.3", "inherits": "2.0.3",
"setprototypeof": "1.1.1", "setprototypeof": "1.1.1",
"statuses": ">= 1.5.0 < 2", "statuses": "1.5.0",
"toidentifier": "1.0.0" "toidentifier": "1.0.0"
} }
}, },
@ -358,7 +358,7 @@
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": { "requires": {
"safer-buffer": ">= 2.1.2 < 3" "safer-buffer": "2.1.2"
} }
}, },
"indexof": { "indexof": {
@ -447,7 +447,7 @@
"resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz", "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz",
"integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=", "integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=",
"requires": { "requires": {
"better-assert": "~1.0.0" "better-assert": "1.0.2"
} }
}, },
"parseuri": { "parseuri": {
@ -455,7 +455,7 @@
"resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz", "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz",
"integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=", "integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=",
"requires": { "requires": {
"better-assert": "~1.0.0" "better-assert": "1.0.2"
} }
}, },
"parseurl": { "parseurl": {
@ -473,7 +473,7 @@
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
"integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==", "integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==",
"requires": { "requires": {
"forwarded": "~0.1.2", "forwarded": "0.1.2",
"ipaddr.js": "1.9.0" "ipaddr.js": "1.9.0"
} }
}, },
@ -519,18 +519,18 @@
"integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
"requires": { "requires": {
"debug": "2.6.9", "debug": "2.6.9",
"depd": "~1.1.2", "depd": "1.1.2",
"destroy": "~1.0.4", "destroy": "1.0.4",
"encodeurl": "~1.0.2", "encodeurl": "1.0.2",
"escape-html": "~1.0.3", "escape-html": "1.0.3",
"etag": "~1.8.1", "etag": "1.8.1",
"fresh": "0.5.2", "fresh": "0.5.2",
"http-errors": "~1.7.2", "http-errors": "1.7.2",
"mime": "1.6.0", "mime": "1.6.0",
"ms": "2.1.1", "ms": "2.1.1",
"on-finished": "~2.3.0", "on-finished": "2.3.0",
"range-parser": "~1.2.1", "range-parser": "1.2.1",
"statuses": "~1.5.0" "statuses": "1.5.0"
}, },
"dependencies": { "dependencies": {
"debug": { "debug": {
@ -560,9 +560,9 @@
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
"integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
"requires": { "requires": {
"encodeurl": "~1.0.2", "encodeurl": "1.0.2",
"escape-html": "~1.0.3", "escape-html": "1.0.3",
"parseurl": "~1.3.3", "parseurl": "1.3.3",
"send": "0.17.1" "send": "0.17.1"
} }
}, },
@ -576,12 +576,12 @@
"resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.3.0.tgz", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.3.0.tgz",
"integrity": "sha512-2A892lrj0GcgR/9Qk81EaY2gYhCBxurV0PfmmESO6p27QPrUK1J3zdns+5QPqvUYK2q657nSj0guoIil9+7eFg==", "integrity": "sha512-2A892lrj0GcgR/9Qk81EaY2gYhCBxurV0PfmmESO6p27QPrUK1J3zdns+5QPqvUYK2q657nSj0guoIil9+7eFg==",
"requires": { "requires": {
"debug": "~4.1.0", "debug": "4.1.1",
"engine.io": "~3.4.0", "engine.io": "3.4.0",
"has-binary2": "~1.0.2", "has-binary2": "1.0.3",
"socket.io-adapter": "~1.1.0", "socket.io-adapter": "1.1.1",
"socket.io-client": "2.3.0", "socket.io-client": "2.3.0",
"socket.io-parser": "~3.4.0" "socket.io-parser": "3.4.0"
} }
}, },
"socket.io-adapter": { "socket.io-adapter": {
@ -598,15 +598,15 @@
"base64-arraybuffer": "0.1.5", "base64-arraybuffer": "0.1.5",
"component-bind": "1.0.0", "component-bind": "1.0.0",
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"debug": "~4.1.0", "debug": "4.1.1",
"engine.io-client": "~3.4.0", "engine.io-client": "3.4.0",
"has-binary2": "~1.0.2", "has-binary2": "1.0.3",
"has-cors": "1.1.0", "has-cors": "1.1.0",
"indexof": "0.0.1", "indexof": "0.0.1",
"object-component": "0.0.3", "object-component": "0.0.3",
"parseqs": "0.0.5", "parseqs": "0.0.5",
"parseuri": "0.0.5", "parseuri": "0.0.5",
"socket.io-parser": "~3.3.0", "socket.io-parser": "3.3.0",
"to-array": "0.1.4" "to-array": "0.1.4"
}, },
"dependencies": { "dependencies": {
@ -621,7 +621,7 @@
"integrity": "sha512-hczmV6bDgdaEbVqhAeVMM/jfUfzuEZHsQg6eOmLgJht6G3mPKMxYm75w2+qhAQZ+4X+1+ATZ+QFKeOZD5riHng==", "integrity": "sha512-hczmV6bDgdaEbVqhAeVMM/jfUfzuEZHsQg6eOmLgJht6G3mPKMxYm75w2+qhAQZ+4X+1+ATZ+QFKeOZD5riHng==",
"requires": { "requires": {
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"debug": "~3.1.0", "debug": "3.1.0",
"isarray": "2.0.1" "isarray": "2.0.1"
}, },
"dependencies": { "dependencies": {
@ -643,7 +643,7 @@
"integrity": "sha512-/G/VOI+3DBp0+DJKW4KesGnQkQPFmUCbA/oO2QGT6CWxU7hLGWqU3tyuzeSK/dqcyeHsQg1vTe9jiZI8GU9SCQ==", "integrity": "sha512-/G/VOI+3DBp0+DJKW4KesGnQkQPFmUCbA/oO2QGT6CWxU7hLGWqU3tyuzeSK/dqcyeHsQg1vTe9jiZI8GU9SCQ==",
"requires": { "requires": {
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"debug": "~4.1.0", "debug": "4.1.1",
"isarray": "2.0.1" "isarray": "2.0.1"
} }
}, },
@ -668,7 +668,7 @@
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
"requires": { "requires": {
"media-typer": "0.3.0", "media-typer": "0.3.0",
"mime-types": "~2.1.24" "mime-types": "2.1.24"
} }
}, },
"unpipe": { "unpipe": {
@ -691,7 +691,7 @@
"resolved": "https://registry.npmjs.org/ws/-/ws-7.1.2.tgz", "resolved": "https://registry.npmjs.org/ws/-/ws-7.1.2.tgz",
"integrity": "sha512-gftXq3XI81cJCgkUiAVixA0raD9IVmXqsylCrjRygw4+UOOGzPoxnQ6r/CnVL9i+mDncJo94tSkyrtuuQVBmrg==", "integrity": "sha512-gftXq3XI81cJCgkUiAVixA0raD9IVmXqsylCrjRygw4+UOOGzPoxnQ6r/CnVL9i+mDncJo94tSkyrtuuQVBmrg==",
"requires": { "requires": {
"async-limiter": "^1.0.0" "async-limiter": "1.0.1"
} }
}, },
"xmlhttprequest-ssl": { "xmlhttprequest-ssl": {

View file

@ -21,6 +21,11 @@ if __name__ == '__main__':
action='store_true', action='store_true',
help='Skip attempt to connect to plotter' help='Skip attempt to connect to plotter'
) )
argParser.add_argument(
'--autostart',
action='store_true',
help='Don\'t require a visit to the control panel to start the first hit. Usefull when you know plotter & scanner are already set up'
)
argParser.add_argument( argParser.add_argument(
'--for-real', '--for-real',
action='store_true', action='store_true',
@ -39,7 +44,7 @@ if __name__ == '__main__':
coloredlogs.install( coloredlogs.install(
level=loglevel, level=loglevel,
# default: "%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s" # default: "%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s"
fmt="%(asctime)s %(hostname)s %(name)s[%(process)d,%(threadName)s] %(levelname)s %(message)s" fmt="%(asctime)s %(hostname)s %(name)s[%(process)d,%(threadName)s,%(pathname)s:%(lineno)d] %(levelname)s %(message)s"
) )
# File logging # File logging

14
sorteerhoed.service Normal file
View file

@ -0,0 +1,14 @@
[Unit]
Description=GuestWorker Main server
After=network.target
[Service]
WorkingDirectory=/home/mt/guest_worker
ExecStart=/usr/bin/pipenv run python sorteerhoed.py --config config.local.yml
User=mt
Restart=Always
RestartSec=10
[Install]
WantedBy=multi-user.target

View file

@ -4,19 +4,19 @@ from sqlalchemy import Column, Integer, String, DateTime, Float
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from sqlalchemy.sql.schema import ForeignKey, Sequence from sqlalchemy.sql.schema import ForeignKey, Sequence
from sqlalchemy.engine import create_engine from sqlalchemy.engine import create_engine
from sqlalchemy.orm.session import sessionmaker from sqlalchemy.orm.session import sessionmaker, object_session
import datetime import datetime
from contextlib import contextmanager from contextlib import contextmanager
import uuid import uuid
import os import os
import coloredlogs import country_converter
import argparse from svgpathtools import svg2paths
from sqlalchemy.sql.functions import func
mainLogger = logging.getLogger("sorteerhoed") mainLogger = logging.getLogger("sorteerhoed")
logger = mainLogger.getChild("store") logger = mainLogger.getChild("store")
Base = declarative_base() Base = declarative_base()
cc = country_converter.CountryConverter()
""" """
HIT lifetime: HIT lifetime:
@ -43,6 +43,7 @@ class HIT(Base):
created_at = Column(DateTime, default=datetime.datetime.utcnow) created_at = Column(DateTime, default=datetime.datetime.utcnow)
updated_at = Column(DateTime, default=datetime.datetime.utcnow) updated_at = Column(DateTime, default=datetime.datetime.utcnow)
scanned_at = Column(DateTime, default=None) scanned_at = Column(DateTime, default=None)
plotted_at = Column(DateTime, default=None)
deleted_at = Column(DateTime, default=None) deleted_at = Column(DateTime, default=None)
assignments = relationship("Assignment", back_populates="hit", order_by="Assignment.created_at") assignments = relationship("Assignment", back_populates="hit", order_by="Assignment.created_at")
fee = Column(Float(precision=2), default=None) fee = Column(Float(precision=2), default=None)
@ -54,11 +55,15 @@ class HIT(Base):
def getImagePath(self): def getImagePath(self):
return os.path.join('scanimation/interfaces/frames', f"{self.id:06d}.jpg") return os.path.join('scanimation/interfaces/frames', f"{self.id:06d}.jpg")
def getImageUrl(self):
return os.path.join('/frames', f"{self.id:06d}.jpg")
def getSvgImageUrl(self): def getSvgImageUrl(self):
return f"scans/{self.id:06d}.svg" return f"/scans/{self.id:06d}.svg"
def getSvgImagePath(self): def getSvgImagePath(self):
return os.path.join('www', self.getSvgImageUrl()) # os.path.join on svgImageUrl leads to invalid absolute url
return os.path.join(f'www/scans/{self.id:06d}.svg')
def getLastAssignment(self): def getLastAssignment(self):
if not len(self.assignments): if not len(self.assignments):
@ -68,23 +73,73 @@ class HIT(Base):
def getAssignmentById(self, assignmentId): def getAssignmentById(self, assignmentId):
for a in self.assignments: for a in self.assignments:
if a.assignment_id == assignmentId: if a.assignment_id == assignmentId:
return return a
return None
def getStatus(self): def getStatus(self):
assignment = self.getLastAssignment()
if self.deleted_at:
return "deleted"
if not self.hit_id:
return "creating"
if not assignment:
return "awaiting worker"
if self.scanned_at: if self.scanned_at:
return "completed" return "scanned"
if self.submit_hit_at: return assignment.getStatus()
return "submission confirmed"
if self.submit_page_at: def toDict(self) -> dict:
return "submitted by worker" values = {c.name: getattr(self, c.name) for c in self.__table__.columns}
if self.open_page_at: assignment = self.getLastAssignment()
return "working" values['assignment'] = assignment.toDict() if assignment else None
if self.accept_time: values['state'] = self.getStatus()
return "accepted by worker" values['scan_image'] = self.getImageUrl() if self.scanned_at else None
# on abandon: values['svg_image'] = self.getSvgImageUrl() if self.isSubmitted() else None
if self.worker_id: values['preceding_assignments'] = [a.toShortDict() for a in self.getBasedOnAssignments()]
return "abandoned by worker" values['preceding_assignments'].append({
return "awaiting worker" 'worker_id': 'Ruben van de Ven & Merijn van Moll',
'turk_country': 'the Netherlands',
'turk_country_code': 'NL'
})
if not values['svg_image'] or not os.path.exists(self.getSvgImagePath()):
values['path_length'] = None
else:
try:
paths, _ = svg2paths(self.getSvgImagePath())
values['path_length'] = round(paths[0].length())
except:
values['path_length'] = None
return values
def delete(self):
self.deleted_at = datetime.datetime.utcnow()
def isSubmitted(self) -> bool:
a = self.getLastAssignment()
if not a:
return False
return bool(a.submit_page_at)
def isConfirmed(self) -> bool:
a = self.getLastAssignment()
if not a:
return False
return bool(a.confirmed_at)
def getBasedOnAssignments(self):
"""
Get preceding assignments, one per worker, excluding the one who did this HIT
"""
assignment = self.getLastAssignment()
session = object_session(self)
q = session.query(Assignment).\
filter(Assignment.submit_page_at < self.created_at).\
filter(Assignment.worker_id != None).\
group_by(Assignment.worker_id).\
order_by(Assignment.created_at.desc())
if assignment and assignment.worker_id:
q = q.filter(Assignment.worker_id != assignment.worker_id)
return q
class Assignment(Base): class Assignment(Base):
__tablename__ = 'assignments' __tablename__ = 'assignments'
@ -99,7 +154,7 @@ class Assignment(Base):
assignment_id = Column(String(255), default = None) assignment_id = Column(String(255), default = None)
worker_id = Column(String(255), default = None) worker_id = Column(String(255), default = None)
accept_at = Column(DateTime, default=None) accept_at = Column(DateTime, default=None) # accept time acccording to SQS
# open_page_at = Column(DateTime, default=None) # open_page_at = Column(DateTime, default=None)
submit_page_at = Column(DateTime, default=None) # Submit the page submit_page_at = Column(DateTime, default=None) # Submit the page
confirmed_at = Column(DateTime, default=None) # validate with UUID when getting Message from Amazon confirmed_at = Column(DateTime, default=None) # validate with UUID when getting Message from Amazon
@ -109,6 +164,52 @@ class Assignment(Base):
answer = Column(String(255), default=None) answer = Column(String(255), default=None)
turk_ip = Column(String(255), default=None) turk_ip = Column(String(255), default=None)
turk_country = Column(String(255), default=None) turk_country = Column(String(255), default=None)
turk_os = Column(String(255), default=None)
turk_browser = Column(String(255), default=None)
def getStatus(self):
if self.rejected_at:
return "rejected"
if self.abandoned_at:
return "abandoned"
if not self.submit_page_at:
return "working"
if not self.confirmed_at:
return "submitted"
return "confirmed"
def toDict(self) -> dict:
values = {c.name: getattr(self, c.name) for c in self.__table__.columns}
if self.turk_country:
if self.turk_country == 'Unknown':
values['turk_country_code'] = '--'
else:
values['turk_country_code'] = cc.convert([self.turk_country], to='ISO2')
else:
values['turk_country_code'] = None
return values
def toShortDict(self) -> dict:
values = {
'worker_id': self.worker_id,
'turk_country': self.turk_country
}
if self.turk_country:
if self.turk_country == 'Unknown':
values['turk_country_code'] = '--'
else:
values['turk_country_code'] = cc.convert([self.turk_country], to='ISO2')
else:
values['turk_country_code'] = None
return values
def getOriginalAssignmentId(self):
"""
Initial assumption: assignmentId would be unique for each worker. But it is not.
Hence assignment_id now gets the value assigmentId_workerId (confusing to say the least!)
Sometimes we want to recover the assignment id from this combination
"""
return self.assignment_id.split('_')[0]
class Store: class Store:
def __init__(self, db_filename, logLevel=0): def __init__(self, db_filename, logLevel=0):
@ -116,12 +217,35 @@ class Store:
if logLevel <= logging.DEBUG: if logLevel <= logging.DEBUG:
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
needsInitialization = not os.path.exists(path)
self.engine = create_engine('sqlite:///'+path, echo=False, connect_args={'check_same_thread': False}) self.engine = create_engine('sqlite:///'+path, echo=False, connect_args={'check_same_thread': False})
Base.metadata.create_all(self.engine) Base.metadata.create_all(self.engine)
self.Session = sessionmaker(bind=self.engine) self.Session = sessionmaker(bind=self.engine)
self.session = self.Session() self.session = self.Session()
self.currentHit = None # mirrors Centralmanagmenet, stored here so we can quickly access it from webserver classes self.currentHit = None # mirrors Centralmanagmenet, stored here so we can quickly access it from webserver classes
self.updateHooks = []
# if needsInitialization:
# self.insertInitialContent()
#
# def insertInitialContent(self):
# hit = self.createHIT()
# assignment = self.newAssignment(hit, 'initial')
#
def registerUpdateHook(self, hook):
if hook not in self.updateHooks:
logger.info(f"Register update hook: {hook}")
self.updateHooks.append(hook)
def triggerUpdateHooks(self, hit = None):
for hook in self.updateHooks:
if callable(hook): # it's a method
hook(hit)
else: # assume it's an object
hook.update(hit)
@contextmanager @contextmanager
def getSession(self): def getSession(self):
@ -133,8 +257,8 @@ class Store:
self.session.rollback() self.session.rollback()
raise raise
def getHits(self, session): def getHits(self):
return self.session.query(Source).order_by(HIT.created_at.desc()) return self.session.query(HIT).order_by(HIT.created_at.desc())
def getHitById(self, hitId): def getHitById(self, hitId):
return self.session.query(HIT).\ return self.session.query(HIT).\
@ -147,8 +271,20 @@ class Store:
def getLastSubmittedHit(self): def getLastSubmittedHit(self):
return self.session.query(HIT).\ return self.session.query(HIT).\
filter(HIT.submit_page_at!=None).\ join(Assignment).\
order_by(HIT.submit_page_at.desc()).first() filter(Assignment.submit_page_at!=None).\
order_by(HIT.created_at.desc()).first()
def getNewestHits(self, n = 2) -> list:
q = self.session.query(HIT).\
filter(HIT.deleted_at==None).\
order_by(HIT.created_at.desc())
if n is not None:
q = q.limit(n)
hits = list(q)
# select DESC, because we want latest, then reverse list to get in right order
hits.reverse()
return hits
def createHIT(self) -> HIT: def createHIT(self) -> HIT:
with self.getSession() as s: with self.getSession() as s:
@ -157,40 +293,57 @@ class Store:
s.flush() s.flush()
s.refresh(hit) s.refresh(hit)
logger.info(f"Created HIT {hit.id}") logger.info(f"Created HIT {hit.id}")
self.triggerUpdateHooks(hit)
return hit return hit
def newAssignment(self, hit: HIT) -> Assignment: def newAssignment(self, hit: HIT, assignmentId) -> Assignment:
# TODO: reset() central management if has pending lastAssignment()
with self.getSession() as s: with self.getSession() as s:
assignment = Assignment() assignment = Assignment()
assignment.assignment_id = assignmentId
hit.assignments.append(assignment) hit.assignments.append(assignment)
s.add(assignment) s.add(assignment)
s.flush() s.flush()
s.refresh(hit) s.refresh(hit)
logger.info(f"Created Assignment {assignment.id}") logger.info(f"Created Assignment {assignment.id}")
self.triggerUpdateHooks(hit)
return assignment return assignment
def saveHIT(self, hit): def saveHIT(self, hit):
with self.getSession() as s: with self.getSession() as s:
logger.info(f"Updating hit! {hit.id}") logger.info(f"Updating hit! {hit.id}")
# s.flush() # s.flush()
self.triggerUpdateHooks(hit)
def addHIT(self, hit: HIT): def saveAssignment(self, assignment):
with self.getSession() as s: with self.getSession() as s:
s.add(hit) logger.info(f"Updating assignment! {assignment.id}")
s.flush() # s.flush()
s.refresh(hit) self.triggerUpdateHooks(assignment.hit)
logger.info(f"Added {hit.id}")
# def addHIT(self, hit: HIT):
# with self.getSession() as s:
# s.add(hit)
# s.flush()
# s.refresh(hit)
# logger.info(f"Added {hit.id}")
def getAvgDurationOfPreviousNHits(self, n) -> int: def getAvgDurationOfPreviousNHits(self, n) -> int:
latest_hits = self.session.query(HIT).\ latest_assignments = self.session.query(Assignment).\
filter(HIT.submit_hit_at!=None).\ filter(Assignment.created_at!=None).\
filter(HIT.accept_time!=None).\ filter(Assignment.submit_page_at!=None).\
order_by(HIT.submit_hit_at.desc()).limit(n) order_by(Assignment.created_at.desc()).limit(n)
durations = [] durations = []
for hit in latest_hits:
durations.append((hit.submit_hit_at - hit.accept_time).total_seconds()) for assignment in latest_assignments:
durations.append((assignment.submit_page_at - assignment.created_at).total_seconds())
if not len(durations): if not len(durations):
return int(2.5*60) return int(2.5*60) # default to 2.5 minutes
return int(sum(durations) / len(durations)) return int(sum(durations) / len(durations))
def getEstimatedHitDuration(self): def getEstimatedHitDuration(self):
@ -204,14 +357,3 @@ class Store:
filter(HIT.submit_hit_at != None).\ filter(HIT.submit_hit_at != None).\
order_by(HIT.submit_hit_at.desc()).limit(n) order_by(HIT.submit_hit_at.desc()).limit(n)
# def rmSource(self, id: int):
# with self.getSession() as session:
# source = session.query(Source).get(id)
# if not source:
# logging.warning(f"Source nr {id} not found")
# else:
# logging.info(f"Deleting source {source.id}: {source.url}")
# session.delete(source)
#
# def getRandomNewsItem(self, session) -> NewsItem:
# return session.query(NewsItem).order_by(func.random()).limit(1).first()

View file

@ -17,10 +17,14 @@ from PIL import Image
import datetime import datetime
from shutil import copyfile from shutil import copyfile
import colorsys import colorsys
import tqdm
class Level(object): class Level(object):
# Level effect adapted from https://stackoverflow.com/a/3125421 """
Level image effect adapted from https://stackoverflow.com/a/3125421
"""
def __init__(self, minv, maxv, gamma): def __init__(self, minv, maxv, gamma):
self.minv= minv/255.0 self.minv= minv/255.0
self.maxv= maxv/255.0 self.maxv= maxv/255.0
@ -81,6 +85,7 @@ class CentralManagement():
self.isScanning = threading.Event() self.isScanning = threading.Event()
self.scanLock = threading.Lock() self.scanLock = threading.Lock()
self.notPaused = threading.Event() self.notPaused = threading.Event()
self.lightStatus = 0
def loadConfig(self, filename, args): def loadConfig(self, filename, args):
@ -94,6 +99,7 @@ class CentralManagement():
'hit_store.db' 'hit_store.db'
) )
self.store = HITStore.Store(varDb, logLevel=logging.DEBUG if self.debug else logging.INFO) self.store = HITStore.Store(varDb, logLevel=logging.DEBUG if self.debug else logging.INFO)
self.store.registerUpdateHook(self.updateLightHook) # change light based on status
self.logger.debug(f"Loaded configuration: {self.config}") self.logger.debug(f"Loaded configuration: {self.config}")
@ -134,7 +140,7 @@ class CentralManagement():
# clear any pending hits: # clear any pending hits:
pending_hits = self.mturk.list_hits(MaxResults=100) pending_hits = self.mturk.list_hits(MaxResults=100)
for pending_hit in pending_hits['HITs']: for pending_hit in pending_hits['HITs']:
# print(pending_hit['HITId'], pending_hit['HITStatus']) # print(pending_hit['HITId'], pending_hit['HITStatus'])
if pending_hit['HITStatus'] == 'Assignable': if pending_hit['HITStatus'] == 'Assignable':
self.logger.warn(f"Expire stale hit: {pending_hit['HITId']}: {pending_hit['HITStatus']}") self.logger.warn(f"Expire stale hit: {pending_hit['HITId']}: {pending_hit['HITStatus']}")
self.mturk.update_expiration_for_hit( self.mturk.update_expiration_for_hit(
@ -142,6 +148,10 @@ class CentralManagement():
ExpireAt=datetime.datetime.fromisoformat('2015-01-01') ExpireAt=datetime.datetime.fromisoformat('2015-01-01')
) )
self.mturk.delete_hit(HITId=pending_hit['HITId']) self.mturk.delete_hit(HITId=pending_hit['HITId'])
staleHit = self.store.getHitByRemoteId(pending_hit['HITId'])
staleHit.delete()
self.store.saveHIT(staleHit)
self.sqs = SqsListener(self.config, self.eventQueue, self.isRunning) self.sqs = SqsListener(self.config, self.eventQueue, self.isRunning)
sqsThread = threading.Thread(target=self.sqs.start, name='sqs') sqsThread = threading.Thread(target=self.sqs.start, name='sqs')
@ -160,10 +170,9 @@ class CentralManagement():
# event listener: # event listener:
dispatcherThread = threading.Thread(target=self.eventListener, name='dispatcher') dispatcherThread = threading.Thread(target=self.eventListener, name='dispatcher')
dispatcherThread.start() dispatcherThread.start()
#
#
self.eventQueue.put(Signal('start', {'ding':'test'})) if self.args.autostart:
self.eventQueue.put(Signal('start', {'ding':'test'}))
while self.isRunning.is_set(): while self.isRunning.is_set():
time.sleep(.5) time.sleep(.5)
@ -177,8 +186,9 @@ class CentralManagement():
self.expireCurrentHit() self.expireCurrentHit()
def expireCurrentHit(self): def expireCurrentHit(self):
if self.currentHit and self.currentHit.hit_id: # hit pending if self.currentHit and not self.currentHit.isConfirmed():
self.logger.warn(f"Delete hit: {self.currentHit.hit_id}") if self.currentHit.hit_id: # hit pending at Amazon
self.logger.warn(f"Expire hit: {self.currentHit.hit_id}")
self.mturk.update_expiration_for_hit( self.mturk.update_expiration_for_hit(
HITId=self.currentHit.hit_id, HITId=self.currentHit.hit_id,
ExpireAt=datetime.datetime.fromisoformat('2015-01-01') ExpireAt=datetime.datetime.fromisoformat('2015-01-01')
@ -188,6 +198,11 @@ class CentralManagement():
except Exception as e: except Exception as e:
self.logger.exception(e) self.logger.exception(e)
if not self.currentHit.isSubmitted():
self.currentHit.delete()
self.store.saveHIT(self.currentHit)
def eventListener(self): def eventListener(self):
while self.isRunning.is_set(): while self.isRunning.is_set():
try: try:
@ -206,27 +221,32 @@ class CentralManagement():
- Plotter complete - Plotter complete
- -
""" """
#TODO: make level debug()
self.logger.info(f"SIGNAL: {signal}") self.logger.info(f"SIGNAL: {signal}")
if signal.name == 'start': if signal.name == 'start':
self.makeHit() self.makeHit()
self.lastHitTime = datetime.datetime.now() self.lastHitTime = datetime.datetime.now()
elif signal.name == 'stop':
self.logger.warning("Stop request")
self.isRunning.clear()
elif signal.name == 'hit.scan': elif signal.name == 'hit.scan':
# start a scan
if signal.params['id'] != self.currentHit.id: if signal.params['id'] != self.currentHit.id:
self.logger.info(f"Hit.scanned had wrong id: {signal}") self.logger.info(f"Hit.scan had wrong id: {signal}")
continue continue
self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='scanning')) # self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='scanning'))
elif signal.name == 'hit.scanned': elif signal.name == 'hit.scanned':
# TODO: wrap up hit & make new HIT # TODO: wrap up hit & make new HIT
if signal.params['id'] != self.currentHit.id: if signal.params['hit_id'] != self.currentHit.id:
self.logger.info(f"Hit.scanned had wrong id: {signal}") self.logger.info(f"Hit.scanned had wrong id: {signal}")
continue continue
self.currentHit.scanned_at = datetime.datetime.utcnow() self.currentHit.scanned_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit)
time_diff = datetime.datetime.now() - self.lastHitTime time_diff = datetime.datetime.now() - self.lastHitTime
to_wait = 10 - time_diff.total_seconds() to_wait = 10 - time_diff.total_seconds()
self.statusPageQueue.add(dict(hit_id=self.currentHit.id, state='scan')) # self.statusPageQueue.add(dict(hit_id=self.currentHit.id, state='scan'))
if to_wait > 0: if to_wait > 0:
self.logger.warn(f"Sleep until next hit: {to_wait}s") self.logger.warn(f"Sleep until next hit: {to_wait}s")
@ -237,10 +257,11 @@ class CentralManagement():
self.makeHit() self.makeHit()
self.lastHitTime = datetime.datetime.now() self.lastHitTime = datetime.datetime.now()
elif signal.name == 'hit.creating': elif signal.name == 'hit.creating':
self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='create_hit')) # self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='create_hit'))
pass
elif signal.name == 'hit.created': elif signal.name == 'hit.created':
self.statusPageQueue.add(dict(hit_id=signal.params['id'], remote_id=signal.params['remote_id'], state='hit')) # self.statusPageQueue.add(dict(hit_id=signal.params['id'], remote_id=signal.params['remote_id'], state='hit'))
pass
elif signal.name == 'scan.start': elif signal.name == 'scan.start':
pass pass
elif signal.name == 'scan.finished': elif signal.name == 'scan.finished':
@ -251,38 +272,53 @@ class CentralManagement():
# Create new assignment # Create new assignment
if signal.params['hit_id'] != self.currentHit.id: if signal.params['hit_id'] != self.currentHit.id:
continue continue
assignment = self.currentHit.getAssignmentById(signal.params['assignment_id'])
assignment = self.store.newAssignment(self.currentHit)
assignment.assignment_id = signal.params['assignment_id']
self.store.saveAssignment(assignment)
self.statusPageQueue.add(dict(hit_id=self.currentHit.id, assignment_id=assignment.assignment_id, state='assignment'))
elif signal.name == 'assignment.info': elif signal.name == 'assignment.info':
assignment = self.currentHit.getAssignmentById(signal.params['assignment_id']) assignment = self.currentHit.getAssignmentById(signal.params['assignment_id'])
if not assignment: if not assignment:
self.logger.warning(f"assignment.info assignment.id not for current hit assignments: {signal}") self.logger.warning(f"assignment.info assignment.id not for current hit assignments: {signal}")
change = False
for name, value in signal.params.items(): for name, value in signal.params.items():
if name == 'ip': if name == 'ip':
assignment.turk_ip = value assignment.turk_ip = value
if name == 'location': if name == 'location':
assignment.turk_country = value assignment.turk_country = value
if name == 'os':
assignment.turk_os = value
if name == 'browser':
assignment.turk_browser = value
change = True
self.logger.debug(f'Set assignment: {name} to {value}') self.logger.debug(f'Set assignment: {name} to {value}')
self.server.statusPage.set(name, value)
if change:
self.store.saveAssignment(assignment)
elif signal.name == 'server.open': elif signal.name == 'server.open':
self.currentHit.open_page_at = datetime.datetime.utcnow() self.currentHit.open_page_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit) self.store.saveHIT(self.currentHit)
self.setLight(True) self.setLight(True)
self.server.statusPage.set('state', self.currentHit.getStatus()) elif signal.name == 'server.close':
self.server.statusPage.set('hit_opened', self.currentHit.open_page_at) if not signal.params['abandoned']:
elif signal.name == 'server.submit': continue
self.currentHit.submit_page_at = datetime.datetime.utcnow() a = self.currentHit.getLastAssignment()
self.store.saveHIT(self.currentHit) if a.assignment_id != signal.params['assignment_id']:
self.logger.info(f"Close of older assignment_id: {signal}")
continue
self.logger.critical(f"Websocket closed of active assignment_id: {signal}")
a.abandoned_at = datetime.datetime.utcnow()
self.store.saveAssignment(a)
self.plotter.park()
self.setLight(False)
elif signal.name == 'assignment.submit':
a = self.currentHit.getLastAssignment()
if a.assignment_id != signal.params['assignment_id']:
self.logger.critical(f"Submit of invalid assignment_id: {signal}")
a.submit_page_at = datetime.datetime.utcnow()
self.store.saveAssignment(a)
self.plotter.park() self.plotter.park()
self.server.statusPage.set('hit_opened', self.currentHit.open_page_at)
# park always triggers a plotter.finished after being processed # park always triggers a plotter.finished after being processed
elif signal.name[:4] == 'sqs.': elif signal.name[:4] == 'sqs.':
@ -294,48 +330,36 @@ class CentralManagement():
sqsHit = self.currentHit sqsHit = self.currentHit
updateStatus = True updateStatus = True
assId = signal.params['event']['AssignmentId'] + '_' + signal.params['event']['WorkerId']
sqsAssignment = sqsHit.getAssignmentById(assId)
if not sqsAssignment:
self.logger.critical(f"Invalid assignmentId given for hit: {signal.params['event']}")
continue
if signal.name == 'sqs.AssignmentAccepted': if signal.name == 'sqs.AssignmentAccepted':
self.logger.info(f'Set status progress to accepted') self.logger.info(f'Set status progress to accepted')
sqsHit.accept_time = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsAssignment.accept_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
sqsHit.worker_id = signal.params['event']['WorkerId'] sqsAssignment.worker_id = signal.params['event']['WorkerId']
if updateStatus:
self.server.statusPage.set('worker_id', sqsHit.worker_id)
# {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAccepted', 'EventTimestamp': '2019-10-23T20:16:10Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} # {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAccepted', 'EventTimestamp': '2019-10-23T20:16:10Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
elif signal.name == 'sqs.AssignmentAbandoned': elif signal.name == 'sqs.AssignmentAbandoned':
self.logger.info(f'Set status progress to abandoned') self.logger.info(f'Set status progress to abandoned')
#{'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAbandoned', 'EventTimestamp': '2019-10-23T20:23:06Z', 'HITId': '3JHB4BPSFKKFQ263K4EFULI3LC79QJ', 'AssignmentId': '3U088ZLJVL450PB6MJZUIIUCB6VW0Y', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} #{'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAbandoned', 'EventTimestamp': '2019-10-23T20:23:06Z', 'HITId': '3JHB4BPSFKKFQ263K4EFULI3LC79QJ', 'AssignmentId': '3U088ZLJVL450PB6MJZUIIUCB6VW0Y', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
sqsHit.accept_time = None sqsAssignment.abandoned_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
sqsHit.open_page_at = None # TODO temporarily dissable block for workers that abandon, until we're sure it works
if self.currentHit.id == sqsHit.id:
if not sqsHit.submit_page_at:
self.reset()
else:
sqsHit.submit_hit_at = datetime.datetime.utcnow() # fake submit
if updateStatus:
self.setLight(False)
self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Accepted task without working on it.') self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Accepted task without working on it.')
elif signal.name == 'sqs.AssignmentReturned': elif signal.name == 'sqs.AssignmentReturned':
self.logger.info(f'Set status progress to returned') self.logger.info(f'Set status progress to returned')
sqsHit.accept_time = None sqsAssignment.rejected_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
sqsHit.open_page_at = None
if self.currentHit.id == sqsHit.id:
if not sqsHit.submit_page_at:
self.reset()
else:
sqsHit.submit_hit_at = datetime.datetime.utcnow() # fake submit
if updateStatus:
self.setLight(False)
# {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentReturned', 'EventTimestamp': '2019-10-23T20:16:47Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} # {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentReturned', 'EventTimestamp': '2019-10-23T20:16:47Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
elif signal.name == 'sqs.AssignmentSubmitted': elif signal.name == 'sqs.AssignmentSubmitted':
# {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"<?xml version=\\"1.0\\" encoding=\\"ASCII\\"?><QuestionFormAnswers xmlns=\\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\\"><Answer><QuestionIdentifier>surveycode<\\/QuestionIdentifier><FreeText>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'} # {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"<?xml version=\\"1.0\\" encoding=\\"ASCII\\"?><QuestionFormAnswers xmlns=\\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\\"><Answer><QuestionIdentifier>surveycode<\\/QuestionIdentifier><FreeText>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'}
self.logger.info(f'Set status progress to submitted') self.logger.info(f'Set status progress to submitted')
# TODO: validate the content of the submission by parsing signal.params['event']['Answer'] and comparing it with sqsHit.uuid sqsAssignment.answer = signal.params['event']['Answer']
sqsHit.answer = signal.params['event']['Answer'] if sqsAssignment.uuid not in sqsAssignment.answer:
if sqsHit.uuid not in sqsHit.answer: self.logger.critical(f"Not a valid answer given?! {sqsAssignment.answer}")
self.logger.critical(f"Not a valid answer given?! {sqsHit.answer}")
if not sqsHit.submit_page_at: if not sqsAssignment.submit_page_at:
# page not submitted, hit is. Nevertheless, create new hit. # page not submitted, hit is. Nevertheless, create new hit.
try: try:
self.mturk.reject_assignment(AssignmentId=signal.params['event']['AssignmentId'], RequesterFeedback='Did not do the assignment') self.mturk.reject_assignment(AssignmentId=signal.params['event']['AssignmentId'], RequesterFeedback='Did not do the assignment')
@ -343,7 +367,8 @@ class CentralManagement():
self.logger.exception(e) self.logger.exception(e)
self.makeHit() self.makeHit()
else: else:
sqsHit.submit_hit_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsAssignment.confirmed_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
# block de worker na succesvolle submit, om dubbele workers te voorkomen # block de worker na succesvolle submit, om dubbele workers te voorkomen
# TODO: Disabled after worker mail, use quals instead # TODO: Disabled after worker mail, use quals instead
#self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Every worker can only work once on the taks.') #self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Every worker can only work once on the taks.')
@ -352,20 +377,25 @@ class CentralManagement():
self.store.saveHIT(sqsHit) self.store.saveHIT(sqsHit)
if updateStatus:
self.logger.warning(f'update status: {sqsHit.getStatus()}')
# TODO: have HITStore/HIT take care of this by emitting a signal
# only update status if it is the currentHit
self.server.statusPage.set('state', sqsHit.getStatus())
else:
self.logger.warning('DO NOT update status')
elif signal.name == 'plotter.finished': elif signal.name == 'plotter.finished':
if self.currentHit and self.currentHit.submit_page_at: # is _always_ triggered after submit due to plotter.park()
self.setLight(False) if self.currentHit and self.currentHit.isSubmitted():
self.currentHit.plotted_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit)
self.logger.info("Start scan thread")
scan = threading.Thread(target=self.scanImage, name='scan') scan = threading.Thread(target=self.scanImage, name='scan')
scan.start() scan.start()
self.server.statusPage.set('hit_submitted', self.currentHit.submit_page_at) elif signal.name == 'plotter.parked':
self.server.statusPage.set('state', self.currentHit.getStatus()) # should this have the code from plotter.finished?
pass
elif signal.name == 'scan.test':
self.logger.info("Start test scan thread")
if self.currentHit:
self.logger.error("cannot scan when HITs are already running")
else:
scan = threading.Thread(target=self.scanTestImage, name='scantest')
scan.start()
else: else:
self.logger.critical(f"Unknown signal: {signal.name}") self.logger.critical(f"Unknown signal: {signal.name}")
except Exception as e: except Exception as e:
@ -373,11 +403,10 @@ class CentralManagement():
self.logger.exception(e) self.logger.exception(e)
def makeHit(self): def makeHit(self):
self.expireCurrentHit() # expire hit if it is there self.expireCurrentHit() # expire hit if it is there
self.eventQueue.put(Signal('hit.creating', {'id': self.currentHit.id if self.currentHit else 'start'})) self.eventQueue.put(Signal('hit.creating', {'id': self.currentHit.id if self.currentHit else 'start'}))
self.server.statusPage.reset()
self.reloadConfig() # reload new config values if they are set self.reloadConfig() # reload new config values if they are set
# self.notPaused.wait() # self.notPaused.wait()
@ -387,7 +416,6 @@ class CentralManagement():
self.logger.info(f"Make HIT {self.currentHit.id}") self.logger.info(f"Make HIT {self.currentHit.id}")
# question = open(self.config['amazon']['task_xml'], mode='r').read().replace("{HIT_NR}",str(self.currentHit.id))
question = '''<?xml version="1.0" encoding="UTF-8"?> question = '''<?xml version="1.0" encoding="UTF-8"?>
<ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"> <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">
<ExternalURL>https://guest.rubenvandeven.com:8888/draw?id={HIT_NR}</ExternalURL> <ExternalURL>https://guest.rubenvandeven.com:8888/draw?id={HIT_NR}</ExternalURL>
@ -422,7 +450,6 @@ class CentralManagement():
self.currentHit.hit_id = new_hit['HIT']['HITId'] self.currentHit.hit_id = new_hit['HIT']['HITId']
self.store.saveHIT(self.currentHit) self.store.saveHIT(self.currentHit)
# TODO: have HITStore/HIT take care of this by emitting a signal
# self.server.statusPage.set('hit_id', new_hit['HIT']['HITId']) # self.server.statusPage.set('hit_id', new_hit['HIT']['HITId'])
# self.server.statusPage.set('hit_created', self.currentHit.created_at) # self.server.statusPage.set('hit_created', self.currentHit.created_at)
# self.server.statusPage.set('fee', f"${self.currentHit.fee:.2f}") # self.server.statusPage.set('fee', f"${self.currentHit.fee:.2f}")
@ -485,51 +512,114 @@ class CentralManagement():
# scan.start() # scan.start()
self.server.statusPage.clearAssignment() self.server.statusPage.clearAssignment()
def scanTestImage(self) -> str:
"""
Run scanimage on scaner and returns a string with the filename
"""
with self.scanLock:
if self.config['dummy_plotter']:
self.eventQueue.put(Signal('scan.start'))
self.logger.warning("Fake scanner for a few seconds")
for i in tqdm.tqdm(range(5)):
time.sleep(1)
self.eventQueue.put(Signal('scan.finished'))
return
cmd = [
'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff',
'--resolution=100', # lower res, faster (more powerful) scan & wipe
'-l','25' #y axis, margin from top of the scanner, hence increasing this, moves the scanned image upwards
,'-t','22', # x axis, margin from left side scanner (seen from the outside)
'-x',str(181),
'-y',str(245)
]
self.logger.info(f"{cmd}")
filename = "/tmp/testscan.jpg"
self.eventQueue.put(Signal('scan.start'))
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# opens connection to scanner, but only starts scanning when output becomes ready:
o, e = proc.communicate(80)
if e:
self.logger.critical(f"Scanner caused: {e.decode()}")
# Should this clear self.isRunning.clear() ?
try:
f = io.BytesIO(o)
img = Image.open(f)
img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM)
tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma'])
tunedImg.save(filename,quality=95)
except Exception as e:
self.logger.critical("Cannot create image from scan. Did scanner work?")
self.logger.exception(e)
copyfile('www/basic.svg', filename)
time.sleep(5) # sleep a few seconds for scanner to return to start position
self.eventQueue.put(Signal('scan.finished'))
def scanImage(self) -> str: def scanImage(self) -> str:
""" """
Run scanimage on scaner and returns a string with the filename Run scanimage on scaner and returns a string with the filename
""" """
cmd = [
'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff',
'--resolution=100', # lower res, faster (more powerful) scan & wipe
'-l','25' #y axis, margin from top of the scanner, hence increasing this, moves the scanned image upwards
,'-t','22', # x axis, margin from left side scanner (seen from the outside)
'-x',str(181),
'-y',str(245)
]
self.logger.info(f"{cmd}")
filename = self.currentHit.getImagePath()
with self.scanLock: with self.scanLock:
if self.config['dummy_plotter']:
self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.start'))
self.logger.warning("Fake scanner for a few seconds")
for i in tqdm.tqdm(range(5)):
time.sleep(1)
self.eventQueue.put(Signal('hit.scanned', {'hit_id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.finished'))
return
cmd = [
'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff',
'--resolution=100', # lower res, faster (more powerful) scan & wipe
'-l','25' #y axis, margin from top of the scanner, hence increasing this, moves the scanned image upwards
,'-t','22', # x axis, margin from left side scanner (seen from the outside)
'-x',str(181),
'-y',str(245)
]
self.logger.info(f"{cmd}")
filename = self.currentHit.getImagePath()
self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id})) self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.start')) self.eventQueue.put(Signal('scan.start'))
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# opens connection to scanner, but only starts scanning when output becomes ready: # opens connection to scanner, but only starts scanning when output becomes ready:
o, e = proc.communicate(80) o, e = proc.communicate(80)
if e: if e:
self.logger.critical(f"Scanner caused: {e.decode()}") self.logger.critical(f"Scanner caused: {e.decode()}")
#TODO: should clear self.isRunning.clear() ? # Should this clear self.isRunning.clear() ?
try: try:
f = io.BytesIO(o) f = io.BytesIO(o)
img = Image.open(f) img = Image.open(f)
img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM) img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM)
tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma']) tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma'])
tunedImg.save(filename) tunedImg.save(filename,quality=95)
except Exception as e: except Exception as e:
self.logger.critical("Cannot create image from scan. Did scanner work?") self.logger.critical("Cannot create image from scan. Did scanner work?")
self.logger.exception(e) self.logger.exception(e)
# TODO: create copyfile('www/basic.svg', filename)
copyfile('www/basic.svg', filename)
time.sleep(5) # sleep a few seconds for scanner to return to start position time.sleep(5) # sleep a few seconds for scanner to return to start position
self.eventQueue.put(Signal('hit.scanned', {'id':self.currentHit.id})) self.eventQueue.put(Signal('hit.scanned', {'hit_id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.finished')) self.eventQueue.put(Signal('scan.finished'))
def setLight(self, on): def setLight(self, on):
value = 1 if on else 0 value = 1 if on else 0
if self.lightStatus == value:
return
self.lightStatus = value
cmd = [ cmd = [
'usbrelay', f'HURTM_1={value}' 'usbrelay', f'HURTM_1={value}'
] ]
@ -537,3 +627,19 @@ class CentralManagement():
code = subprocess.call(cmd) code = subprocess.call(cmd)
if code > 0: if code > 0:
self.logger.warning(f"Error on light change: {code}") self.logger.warning(f"Error on light change: {code}")
def updateLightHook(self, hit = None):
# ignore hit attribute, which comes from the HITstore
self.setLight(self.getLightStatus())
def getLightStatus(self) -> bool:
if not self.currentHit:
return False
a = self.currentHit.getLastAssignment()
if not a:
return False
if a.abandoned_at or a.rejected_at:
return False
if self.currentHit.plotted_at: # wait till plotter is done
return False
return True

View file

@ -34,31 +34,61 @@ class Plotter:
topLeft = absPlotWidth / self.plotWidth #ignore changes in config plotwidth topLeft = absPlotWidth / self.plotWidth #ignore changes in config plotwidth
self.q.put([(1/2.54)/absPlotWidth + topLeft,0,0]) self.q.put([(1/2.54)/absPlotWidth + topLeft,0,0])
def disable_motors(self):
self.ad.plot_setup()
self.setPenDown(False)
self.ad.options.mode = "manual"
self.ad.options.manual_cmd = "disable_xy"
self.ad.plot_run()
def connect(self):
# connect/disconnect once because often first connection attempt fails
self.ad.interactive()
self.ad.connect()
self.ad.pen_raise()
self.ad.disconnect()
# start?
self.ad.interactive()
connected = self.ad.connect()
# if not connected:
# raise Exception("Cannot connect to Axidraw")
while not connected:
self.logger.error("Cannot connect to Axidraw (retry, 1s)")
self.ad.disconnect()
time.sleep(1)
connected = self.ad.connect()
# back to default control mode:
self.ad.options.mode = "plot"
# self.ad.options.units = 1 # set to use centimeters instead of inches
self.ad.options.accel = 100;
self.ad.options.speed_penup = 100
self.ad.options.speed_pendown = 100
self.ad.options.model = 1 # 2 for A3, 1 for A4
self.ad.options.pen_pos_up = 100
def reconnect(self):
self.connect()
# no park on intial connection
self.park()
# self.ad.moveto(0,0)
def start(self): def start(self):
try: try:
if not self.config['dummy_plotter']: if not self.config['dummy_plotter']:
self.ad = axidraw.AxiDraw() self.ad = axidraw.AxiDraw()
self.connect()
self.ad.interactive()
# self.ad.plot_path()
connected = self.ad.connect()
if not connected:
raise Exception("Cannot connect to Axidraw")
# self.ad.options.units = 1 # set to use centimeters instead of inches
self.ad.options.accel = 100;
self.ad.options.speed_penup = 100
self.ad.options.speed_pendown = 100
self.ad.options.model = 1 # 2 for A3, 1 for A4
self.ad.options.pen_pos_up = 100
self.park()
# self.ad.moveto(0,0)
else: else:
self.ad = None self.ad = None
while True:
self.logger.info("Fake AD-connect issue")
time.sleep(1)
self.axiDrawCueListener() self.axiDrawCueListener()
except Exception as e: except Exception as e:
self.logger.exception(e) self.logger.exception(e)
@ -140,18 +170,23 @@ class Plotter:
#if self.goPark: #if self.goPark:
# print("seg",segment) # print("seg",segment)
# change of pen state? draw previous segments! if segment == "disable_motors":
if (segment[2] == 1 and not self.pen_down) or (segment[2] == 0 and self.pen_down) or len(segments) > 150: self.disable_motors()
if len(segments): elif segment == "reconnect":
self.draw_segments(segments) self.reconnect()
plotterRan = True else:
segments = [] #reset # change of pen state? draw previous segments!
if (segment[2] == 1 and not self.pen_down) or (segment[2] == 0 and self.pen_down) or len(segments) > 150:
if len(segments):
self.draw_segments(segments)
plotterRan = True
segments = [] #reset
# and change pen positions # and change pen positions
self.setPenDown(segment[2] == 1) self.setPenDown(segment[2] == 1)
segments.append(segment) segments.append(segment)
except queue.Empty as e: except queue.Empty as e:
self.logger.debug("Timeout queue.") self.logger.debug("Timeout queue.")

View file

@ -21,6 +21,14 @@ import html
logger = logging.getLogger("sorteerhoed").getChild("webserver") logger = logging.getLogger("sorteerhoed").getChild("webserver")
class DateTimeEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime.datetime):
return o.isoformat(timespec='seconds')
return super().default(self, o)
class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler): class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler):
def set_extra_headers(self, path): def set_extra_headers(self, path):
"""For subclass to add extra headers to the response""" """For subclass to add extra headers to the response"""
@ -36,7 +44,6 @@ class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler):
self.set_header("Content-Type", "image/svg+xml") self.set_header("Content-Type", "image/svg+xml")
class WebSocketHandler(tornado.websocket.WebSocketHandler): class WebSocketHandler(tornado.websocket.WebSocketHandler):
""" """
Websocket from the workers Websocket from the workers
@ -49,6 +56,8 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.plotterQ = plotterQ self.plotterQ = plotterQ
self.eventQ = eventQ self.eventQ = eventQ
self.store = store self.store = store
self.assignment_id = None
self.abandoned = False
def check_origin(self, origin): def check_origin(self, origin):
parsed_origin = urlparse(origin) parsed_origin = urlparse(origin)
@ -66,12 +75,19 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.hit = self.store.currentHit self.hit = self.store.currentHit
# my core assumption about assignment_id was wrong. It is not unique per worker, so we need to merge those
self.assignment_id = str(self.get_query_argument('assignmentId'))
self.assignment_id += '_' + str(self.get_query_argument('workerId'))
self.assignment_id = int(self.get_query_argument('assignment_id')) self.assignment = self.hit.getLastAssignment()
self.timeout = datetime.datetime.now() + datetime.timedelta(seconds=self.store.getHitTimeout()) if self.assignment.assignment_id != self.assignment_id:
raise Exception(f"Opening websocket for invalid assignment {self.assignment_id}")
if self.hit.submit_hit_at: self.timeout = self.assignment.created_at + datetime.timedelta(seconds=self.store.getHitTimeout())
# timeLeft = (self.timeout - datetime.datetime.utcnow()).total_seconds()
if self.hit.isSubmitted():
raise Exception("Opening websocket for already submitted hit") raise Exception("Opening websocket for already submitted hit")
#logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}") #logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}")
@ -79,21 +95,22 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.strokes = [] self.strokes = []
# the client sent the message # the client sent the message
def on_message(self, message): def on_message(self, message):
logger.debug(f"recieve: {message}") logger.debug(f"recieve: {message}")
if self.assignment_id != self.hit.getLastAssignment().assignment_id: if self.assignment_id != self.hit.getLastAssignment().assignment_id:
logger.critical(f"Skip message for non-last assignment {message}") logger.critical(f"Skip message for non-last assignment {message}")
return
if datetime.datetime.now() > self.timeout: if datetime.datetime.utcnow() > self.timeout:
logger.critical("Close websocket after timeout (abandon?)") logger.critical("Close websocket after timeout (abandon?)")
self.close() self.close()
return return
try: try:
msg = json.loads(message) msg = json.loads(message)
# TODO: sanitize input: min/max, limit strokes
if msg['action'] == 'move': if msg['action'] == 'move':
# TODO: min/max input # TODO: min/max input
point = [float(msg['direction'][0]),float(msg['direction'][1]), bool(msg['mouse'])] point = [float(msg['direction'][0]),float(msg['direction'][1]), bool(msg['mouse'])]
@ -127,8 +144,8 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.write_message(json.dumps({ self.write_message(json.dumps({
'action': 'submitted', 'action': 'submitted',
'msg': f"Submission ok, please copy this token to your HIT at Mechanical Turk: {self.hit.uuid}", 'msg': f"Submission ok, please copy this token to your HIT at Mechanical Turk: {self.assignment.uuid}",
'code': str(self.hit.uuid) 'code': str(self.assignment.uuid)
})) }))
self.close() self.close()
@ -136,8 +153,9 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
# not used, implicit in move? # not used, implicit in move?
pass pass
elif msg['action'] == 'info': elif msg['action'] == 'info':
self.eventQ.put(Signal('hit.info', dict( self.eventQ.put(Signal('assignment.info', dict(
hit_id=self.hit.id, hit_id=self.hit.id,
assignment_id=self.assignment_id,
resolution=msg['resolution'], resolution=msg['resolution'],
browser=msg['browser'] browser=msg['browser']
))) )))
@ -153,38 +171,45 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
# client disconnected # client disconnected
def on_close(self): def on_close(self):
self.__class__.rmConnection(self) self.__class__.rmConnection(self)
if self.assignment_id:
self.eventQ.put(Signal('server.close', dict(assignment_id=self.assignment_id, abandoned=self.abandoned)))
logger.info(f"Client disconnected: {self.request.remote_ip}") logger.info(f"Client disconnected: {self.request.remote_ip}")
# TODO: abandon assignment??
def submit_strokes(self): def submit_strokes(self):
if len(self.strokes) < 1: if len(self.strokes) < 1:
return False return False
self.eventQ.put(Signal("server.submit", dict(hit_id = self.hit.id))) self.eventQ.put(Signal("assignment.submit", dict(
hit_id = self.hit.id,
assignment_id=self.assignment_id)))
if self.config['dummy_plotter']: # deprecated: now done at scanner method:
d = strokes2D(self.strokes) # if self.config['dummy_plotter']:
svg = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?> # d = strokes2D(self.strokes)
<svg viewBox="0 0 600 600" # svg = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
xmlns:dc="http://purl.org/dc/elements/1.1/" # <svg viewBox="0 0 600 600"
xmlns:cc="http://creativecommons.org/ns#" # xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" # xmlns:cc="http://creativecommons.org/ns#"
xmlns:svg="http://www.w3.org/2000/svg" # xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://www.w3.org/2000/svg" # xmlns:svg="http://www.w3.org/2000/svg"
version="1.1" # xmlns="http://www.w3.org/2000/svg"
> # version="1.1"
<path d="{d}" style="stroke:black;stroke-width:2;fill:none;" /> # >
</svg> # <path d="{d}" style="stroke:black;stroke-width:2;fill:none;" />
""" # </svg>
# """
filename = self.hit.getImagePath() #
logger.info(f"Write to {filename}") # filename = self.hit.getImagePath()
with open(filename, 'w') as fp: # logger.info(f"Write to {filename}")
fp.write(svg) # with open(filename, 'w') as fp:
# fp.write(svg)
# we fake a hit.scanned event # we fake a hit.scanned event
self.eventQ.put(Signal('hit.scanned', {'hit_id':self.hit.id})) # self.eventQ.put(Signal('hit.scanned', {'hit_id':self.hit.id}))
return self.hit.uuid return self.assignment.uuid
@classmethod @classmethod
def rmConnection(cls, client): def rmConnection(cls, client):
@ -192,15 +217,109 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
return return
cls.connections.remove(client) cls.connections.remove(client)
@classmethod
def hasConnection(cls, client):
return client in cls.connections
@classmethod
def timeoutConnectionForAssignment(cls, assignment_id):
logger.warn(f"Check timeout for {assignment_id}")
for client in cls.connections:
logger.info(client.assignment_id)
if client.assignment_id == assignment_id:
client.abandoned = True
client.close()
class ConfigWebSocketHandler(tornado.websocket.WebSocketHandler):
"""
Websocket for config & control
"""
CORS_ORIGINS = ['localhost', '192.168.1.102','guest.rubenvandeven.com']
connections = set()
def initialize(self, config, plotterQ: Queue, eventQ: Queue, store: HITStore):
self.config = config
self.plotterQ = plotterQ
self.eventQ = eventQ
self.store = store
def check_origin(self, origin):
parsed_origin = urlparse(origin)
# parsed_origin.netloc.lower() gives localhost:3333
valid = any([parsed_origin.hostname.endswith(origin) for origin in self.CORS_ORIGINS])
logger.info(f"Connection from {origin} is valid? valid: {valid}")
return valid
# the client connected
def open(self, p = None):
self.__class__.connections.add(self)
logger.warning(f"Config client connected: {self.request.remote_ip}")
#logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}")
# self.eventQ.put(Signal('server.open', dict(assignment_id=self.assignment_id)))
# self.strokes = []
# the client sent the message
def on_message(self, message):
logger.debug(f"recieve: {message}")
try:
msg = json.loads(message)
if msg['action'] == 'start':
self.eventQ.put(Signal('start', {'ding':'test'}))
elif msg['action'] == 'disable_motors':
self.plotterQ.put("disable_motors")
elif msg['action'] == 'enable_motors':
self.plotterQ.put("reconnect")
elif msg['action'] == 'scanner_test':
self.eventQ.put(Signal('scan.test'))
elif msg['action'] == 'stop':
self.eventQ.put(Signal('stop'))
# elif msg['action'] == 'info':
# self.eventQ.put(Signal('assignment.info', dict(
# hit_id=self.hit.id,
# assignment_id=self.assignment_id,
# resolution=msg['resolution'],
# browser=msg['browser']
# )))
# pass
else:
# self.send({'alert': 'Unknown request: {}'.format(message)})
logger.warn('Unknown request: {}'.format(message))
except Exception as e:
# self.send({'alert': 'Invalid request: {}'.format(e)})
logger.exception(e)
# client disconnected
def on_close(self):
self.__class__.rmConnection(self)
logger.warning(f"Config client disconnected: {self.request.remote_ip}")
# TODO: abandon assignment??
@classmethod
def rmConnection(cls, client):
if client not in cls.connections:
return
cls.connections.remove(client)
@classmethod
def hasConnection(cls, client):
return client in cls.connections
class StatusWebSocketHandler(tornado.websocket.WebSocketHandler): class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
CORS_ORIGINS = ['localhost'] CORS_ORIGINS = ['localhost']
connections = set() connections = set()
queue = queue.Queue()
def initialize(self, statusPage): def initialize(self, statusPage):
self.statusPage = statusPage self.statusPage = statusPage
pass
def check_origin(self, origin): def check_origin(self, origin):
parsed_origin = urlparse(origin) parsed_origin = urlparse(origin)
@ -209,14 +328,12 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
return valid return valid
# the client connected # the client connected
def open(self, p = None): def open(self):
self.__class__.connections.add(self) self.__class__.connections.add(self)
for prop, value in self.statusPage.__dict__.items(): limit = 2
self.write_message(json.dumps({ if 'all' in self.request.query_arguments:
'property': prop, limit = None
'value': value.isoformat(timespec='seconds') if type(value) is datetime.datetime else value self.write_message(json.dumps(self.statusPage.fetch(limit), cls=DateTimeEncoder))
}))
# client disconnected # client disconnected
def on_close(self): def on_close(self):
@ -230,13 +347,13 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
cls.connections.remove(client) cls.connections.remove(client)
@classmethod @classmethod
def update_for_all(cls, prop, value): def update_for_all(cls, data):
logger.debug(f"update for all {prop} {value}") logger.debug(f"update for all {data}")
for connection in cls.connections: for connection in cls.connections:
connection.write_message(json.dumps({ try:
'property': prop, connection.write_message(json.dumps(data, cls=DateTimeEncoder))
'value': value.isoformat(timespec='seconds') if type(value) is datetime.datetime else value except Exception as e:
})) logger.exception(e)
def strokes2D(strokes): def strokes2D(strokes):
# strokes to a d attribute for a path # strokes to a d attribute for a path
@ -277,17 +394,38 @@ class DrawPageHandler(tornado.web.RequestHandler):
try: try:
hit_id = int(self.get_query_argument('id')) hit_id = int(self.get_query_argument('id'))
if hit_id != self.store.currentHit.id: if hit_id != self.store.currentHit.id:
assignmentId = self.get_query_argument('assignmentId', '')
orig_assigmentId = assignmentId
if len(assignmentId):
assignmentId += '_' + str(self.get_query_argument('workerId', ''))
hit = self.store.getHitById(hit_id)
assignment = hit.getAssignmentById(assignmentId)
if not assignment:
self.write("Invalid HIT or assignment id")
return
submitUrl = self.get_query_argument('turkSubmitTo', '')
submitUrl += '/mturk/externalSubmit'
self.write("An error occured. Please re-submit your assignment validation code. We're really sorry for the inconvenience.")
self.write(f"<form method='post' action='{submitUrl}'>")
self.write(f"<input type='text' name='assignmentId' value='{orig_assigmentId}'>")
self.write(f"<input type='text' name='surveycode' value='{assignment.uuid}'>")
self.write(f"<input type='submit' value='Submit finished assignment'>")
self.write("</form>")
self.write("Invalid HIT") self.write("Invalid HIT")
return return
hit = self.store.currentHit hit = self.store.currentHit
except Exception: except Exception:
self.write("HIT not found") self.write("HIT not found")
else: else:
if hit.submit_page_at: if hit.isSubmitted():
self.write("HIT already submitted") self.write("HIT already submitted")
return return
assignmentId = self.get_query_argument('assignmentId', '') assignmentId = self.get_query_argument('assignmentId', '')
if len(assignmentId) and assignmentId != "ASSIGNMENT_ID_NOT_AVAILABLE":
assignmentId += '_' + str(self.get_query_argument('workerId', ''))
if len(assignmentId) < 1: if len(assignmentId) < 1:
logger.critical("Accessing page without assignment id. Allowing it for debug purposes... fingers crossed?") logger.critical("Accessing page without assignment id. Allowing it for debug purposes... fingers crossed?")
@ -295,6 +433,18 @@ class DrawPageHandler(tornado.web.RequestHandler):
if assignmentId == 'ASSIGNMENT_ID_NOT_AVAILABLE': if assignmentId == 'ASSIGNMENT_ID_NOT_AVAILABLE':
previewOnly = True previewOnly = True
if len(assignmentId) and not previewOnly:
# process/create assignment
assignment = self.store.currentHit.getAssignmentById(assignmentId)
if not assignment:
# new assignment
logger.warning(f"Create new assignment {assignmentId}")
assignment = self.store.newAssignment(self.store.currentHit, assignmentId)
assignment.worker_id = str(self.get_query_argument('workerId', ''))
self.store.saveAssignment(assignment)
logger.info(f"Set close timeout for {self.store.getHitTimeout()}")
Server.loop.asyncio_loop.call_later(self.store.getHitTimeout(), WebSocketHandler.timeoutConnectionForAssignment, assignment.assignment_id)
previous_hit = self.store.getLastSubmittedHit() previous_hit = self.store.getLastSubmittedHit()
if not previous_hit: if not previous_hit:
# start with basic svg # start with basic svg
@ -314,7 +464,7 @@ class DrawPageHandler(tornado.web.RequestHandler):
.replace("{TOP_PADDING}", str(self.top_padding))\ .replace("{TOP_PADDING}", str(self.top_padding))\
.replace("{LEFT_PADDING}", str(self.left_padding))\ .replace("{LEFT_PADDING}", str(self.left_padding))\
.replace("{SCRIPT}", '' if previewOnly else '<script type="text/javascript" src="/assignment.js"></script>')\ .replace("{SCRIPT}", '' if previewOnly else '<script type="text/javascript" src="/assignment.js"></script>')\
.replace("{ASSIGNMENT}", '' if previewOnly else str(assignmentId)) # TODO: fix unsafe inserting of GET variable .replace("{ASSIGNMENT}", '' if previewOnly else str(assignment.getOriginalAssignmentId())) # TODO: fix unsafe inserting of GET variable
self.write(contents) self.write(contents)
@ -330,7 +480,8 @@ class DrawPageHandler(tornado.web.RequestHandler):
self.eventQ.put(Signal('hit.assignment', dict( self.eventQ.put(Signal('hit.assignment', dict(
hit_id=hit.id, ip=ip, assignment_id=assignmentId hit_id=hit.id, ip=ip, assignment_id=assignmentId
))) )))
# self.eventQ.put(Signal('hit.info', dict(hit_id=hit.id, ip=ip)))
self.eventQ.put(Signal('assignment.info', dict(assignment_id=assignmentId, ip=ip)))
try: try:
geoip = self.geoip_reader.country(ip) geoip = self.geoip_reader.country(ip)
@ -353,30 +504,30 @@ class BackendHandler(tornado.web.RequestHandler):
def get(self): def get(self):
rows = [] rows = []
for hit in self.store.getHITs(100): # for hit in self.store.getHITs(100):
if hit.submit_hit_at and hit.accept_time: # if hit.submit_hit_at and hit.accept_time:
seconds = (hit.submit_hit_at - hit.accept_time).total_seconds() # seconds = (hit.submit_hit_at - hit.accept_time).total_seconds()
duration_m = int(seconds/60) # duration_m = int(seconds/60)
duration_s = max(int(seconds%60), 0) # duration_s = max(int(seconds%60), 0)
duration = (f"{duration_m}m" if duration_m else "") + f"{duration_s:02d}s" # duration = (f"{duration_m}m" if duration_m else "") + f"{duration_s:02d}s"
else: # else:
duration = "-" # duration = "-"
#
# fee = f"${hit.fee:.2}" if hit.fee else "-"
#
# rows.append(
# f"""
# <tr><td></td><td>{hit.worker_id}</td>
# <td>{hit.turk_ip}</td>
# <td>{hit.turk_country}</td>
# <td>{fee}</td>
# <td>{hit.accept_time}</td>
# <td>{duration}</td><td></td>
# """
# )
fee = f"${hit.fee:.2}" if hit.fee else "-" contents = open(os.path.join(self.path, 'backend/backend.html'), 'r').read()
# contents = contents.replace("{{TBODY}}", "".join(rows))
rows.append(
f"""
<tr><td></td><td>{hit.worker_id}</td>
<td>{hit.turk_ip}</td>
<td>{hit.turk_country}</td>
<td>{fee}</td>
<td>{hit.accept_time}</td>
<td>{duration}</td><td></td>
"""
)
contents = open(os.path.join(self.path, 'backend.html'), 'r').read()
contents = contents.replace("{{TBODY}}", "".join(rows))
self.write(contents) self.write(contents)
class StatusPage(): class StatusPage():
@ -384,58 +535,40 @@ class StatusPage():
Properties for on the status page, which are send over websockets the moment Properties for on the status page, which are send over websockets the moment
they are altered. they are altered.
""" """
def __init__(self):
self.reset()
def reset(self): def __init__(self, store: HITStore):
logger.info("Resetting status") self.store = store
self.hit_id = None self.store.registerUpdateHook(self)
self.worker_id = None
self.ip = None
self.location = None
self.browser = None
self.os = None
self.resolution = None
self.state = None
self.fee = None
self.hit_created = None
self.hit_opened = None
self.hit_submitted = None
def clearAssignment(self): def update(self, hit = None):
logger.info("Resetting hit assignment") """
self.worker_id = None Send the given HIT formatted to the websocket clients
self.ip = None
self.location = None
self.browser = None
self.os = None
self.resolution = None
self.hit_created = None
def __setattr__(self, name, value): If no hit is given, load the last 2 items
if name in self.__dict__ and self.__dict__[name] == value: """
logger.debug(f"Ignore setting status of {name}: it already is set to {value}") if hit:
return data = [hit.toDict()]
else:
hits = self.store.getNewestHits(2)
data = [hit.toDict() for hit in hits]
self.__dict__[name] =value
logger.info(f"Update status: {name}: {value}")
if Server.loop: if Server.loop:
Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, name, value) Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, data)
else: else:
logger.warn("Status: no server loop to call update command") logger.warn("Status: no server loop to call update command")
def fetch(self, limit = 2):
def set(self, name, value): """
return self.__setattr__(name, value) Fetch latest, used on connection of status page
"""
hits = self.store.getNewestHits(limit)
return [hit.toDict() for hit in hits]
class Server: class Server:
""" """
Server for HIT -> plotter events Server for HIT -> plotter events
As well as for the Status interface As well as for the Status interface
TODO: change to have the HIT_id as param to the page. Load hit from storage with previous image
""" """
loop = None loop = None
@ -452,7 +585,7 @@ class Server:
self.server_loop = None self.server_loop = None
self.store = store self.store = store
self.statusPage = StatusPage() self.statusPage = StatusPage(store)
def start(self): def start(self):
@ -470,6 +603,12 @@ class Server:
'eventQ': self.eventQ, 'eventQ': self.eventQ,
'store': self.store, 'store': self.store,
}), }),
(r"/config/ws(.*)", ConfigWebSocketHandler, {
'config': self.config,
'plotterQ': self.plotterQ,
'eventQ': self.eventQ,
'store': self.store,
}),
(r"/status/ws", StatusWebSocketHandler, dict(statusPage = self.statusPage)), (r"/status/ws", StatusWebSocketHandler, dict(statusPage = self.statusPage)),
(r"/draw", DrawPageHandler, (r"/draw", DrawPageHandler,
dict( dict(
@ -489,6 +628,8 @@ class Server:
store = self.store, store = self.store,
path=self.web_root, path=self.web_root,
)), )),
(r"/frames/(.*)", StaticFileWithHeaderHandler,
{"path": 'scanimation/interfaces/frames'}),
(r"/(.*)", StaticFileWithHeaderHandler, (r"/(.*)", StaticFileWithHeaderHandler,
{"path": self.web_root}), {"path": self.web_root}),
], debug=True, autoreload=False) ], debug=True, autoreload=False)

View file

@ -51,7 +51,7 @@ let draw = function(e) {
strokeEl.setAttribute('d', d); strokeEl.setAttribute('d', d);
} }
console.log([pos['x'], pos['y']], isDrawing); //console.log([pos['x'], pos['y']], isDrawing);
socket.send(JSON.stringify({ socket.send(JSON.stringify({
'action': 'move', 'action': 'move',
'direction': [pos['x'], pos['y']], 'direction': [pos['x'], pos['y']],

40
www/backend/backend.html Normal file
View file

@ -0,0 +1,40 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="stylesheet" type="text/css" href="/backend/style.css" />
<script src='../worker_specs/dateformat.js'></script>
<script src='../worker_specs/reconnecting-websocket.min.js'></script>
<script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
</head>
<body>
<div id="wrapper">
<div v-if="hit && hit.assignment" :class="[{'invisible': !hit.visible}, 'hit']">
<img :src="hit.scan_image">
<div class='credits'>
<span class='worker_id'>{{hit.assignment.worker_id}}</span>
<span class='country'>{{hit.assignment.turk_country}}</span>
<template v-if="hit.preceding_assignments.length">
<div id='collaborators'>
<div id='collab_items'>
<span v-for="a of hit.preceding_assignments" class='credit'>
<span class='worker_id'>{{a.worker_id}}</span>
<span class='country'>{{a.turk_country_code}}</span>
</span>
</div>
</div>
</template>
</div>
</div>
</div>
<script src="/backend/script.js"></script>
</body>
</html>

60
www/backend/script.js Normal file
View file

@ -0,0 +1,60 @@
var app = new Vue({
el: '#wrapper',
data: {
hit: {}
},
})
var hits = {};
var hitIds = [];
// fetch ?all in database
let ws = new ReconnectingWebSocket('ws://localhost:8888/status/ws?all')
ws.addEventListener('open', () => {
// ws.send('hi server')
})
ws.addEventListener('message', (event) => {
console.log('message: ')
let load_hits = JSON.parse(event.data)
for(let hit of load_hits) {
console.log(hit);
hits[hit.id] = hit;
if(hit.scanned_at && hitIds.indexOf(hit.id) < 0){
// only add if indeed scanned
hitIds.push(hit.id);
}
}
})
var currentI = 0;
setInterval(function(e){
if(hitIds.length < 1) {
return;
}
currentI = (currentI + 1) % hitIds.length;
let currentHitId = hitIds[currentI];
console.log(currentHitId);
let hit = hits[currentHitId];
hit['visible'] = false;
app.hit['visible'] = false;
setTimeout(() => {
app.hit = hits[currentHitId]
}, 1000);
setTimeout(() => {
app.hit['visible'] = true;
let wrapperEl = document.getElementById("collaborators");
let innerEl = document.getElementById("collab_items");
let size = 80;
do {
innerEl.style.fontSize = size + "%";
size --;
} while(innerEl.clientHeight > wrapperEl.clientHeight && size > 27);
}, 1100);
}, 7000);

129
www/backend/style.css Normal file
View file

@ -0,0 +1,129 @@
@font-face {
font-family: 'bebas';
src: url('/font/BebasNeue-Regular.ttf');
}
@font-face {
font-family: 'freesans';
src: url('/font/FreeSans.ttf')
}
@font-face {
font-family: 'bt';
src: url('/font/steelfish_rg.ttf')
}
:root{
--base-font-size: 60px;
--spec_name-font-size: 120%;
--spec_value-font-size: 250%;
--base-color: #271601; /* tekst */
--alt-color: #FFF5DF; /* achtergrond */
--amazon-color: #F0C14C;
/* ////// GAT ACHTERKANT PLEK & POSITIE /////// */
/* */ /* */
/* */ --pos-x: 0px; /* */
/* */ --pos-y: 50px; /* */
/* */ --width: 100%; /* 285mm */
/* */ --height: 100%; /* 500mm */
/* */ /* */
/* //////////////////////////////////////////// */
}
body{
background: black;
margin:0;
color:#f5f5f5;
font-family: 'bt';
font-size: var(--base-font-size);
line-height: var(--base-font-size)*1.5;
}
.hit{
position:absolute;
top:var(--pos-y);
left:24px;
right:21px;
bottom:0;
text-align: center;
transition: opacity 1s;
opacity: 1;
/*animation: fadeIn 1s, fadeOut 1s 2s;*/
}
.hit.invisible{
opacity: 0;
}
@keyframes fadeIn{
from{opacity:0;}
to{opacity:1;}
}
@keyframes fadeOut{
from{opacity:1;}
to{opacity:0;}
}
.hit img{
display: block;
margin: 6vh auto 7vh;
width: 875px;
}
.credits{
font-size: var(--base-font-size);
text-transform: uppercase;
}
.credits::before{
content:'created by';
font-family: 'freesans';
text-transform: lowercase;
display:block;
font-size: calc(var(--base-font-size)/3);
}
.country{
/* color: #fff; */
/* font-family: 'freesans';
text-transform: lowercase; */
font-size: 50%;
vertical-align: 67%;
margin-left: -5px;
margin-right: .4em;
}
.country::before{content:'(';}
.country::after{content:')';}
#collaborators{
height: 27vh;
width: 800px;
margin: 0 auto;
}
#collab_items{
text-align: justify-center;
}
#collaborators::before{
margin-top: 7vh;
content:'in collaboration with';
display:block;
font-family: 'freesans';
font-size: calc(var(--base-font-size)/3);
text-transform: lowercase;
}
#collaborators .credit{
margin-right:.1em;
white-space: nowrap;
display: inline-block;
}

View file

@ -5,11 +5,11 @@
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg"
id="svg8" width="210mm"
version="1.1"
viewBox="0 0 210 210"
height="210mm" height="210mm"
width="210mm"> viewBox="0 0 210 210"
version="1.1"
id="svg8">
<defs <defs
id="defs2" /> id="defs2" />
<metadata <metadata
@ -25,14 +25,15 @@
</rdf:RDF> </rdf:RDF>
</metadata> </metadata>
<g <g
transform="translate(0,-87)" style="stroke-width:1.52441633"
id="layer1"> id="layer1"
transform="matrix(0.65598879,0,0,0.65598879,44.11553,-86.509667)">
<rect <rect
y="138.32738" style="fill:none;fill-opacity:1;stroke:#ffffff;stroke-width:1.52441633;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1"
x="51.327381"
height="107.34524"
width="107.34524"
id="rect815" id="rect815"
style="fill:none;fill-opacity:1;stroke:white;stroke-width:1;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1" /> width="107.34524"
height="107.34524"
x="51.327381"
y="138.32738" />
</g> </g>
</svg> </svg>

Before

Width:  |  Height:  |  Size: 1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

View file

@ -13,7 +13,7 @@
height="180mm" height="180mm"
preserveAspectRatio="none" preserveAspectRatio="none"
id="svg3" id="svg3"
sodipodi:docname="000139.svg" sodipodi:docname="basic_square.svg"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"> inkscape:version="0.92.4 (5da689c313, 2019-01-14)">
<metadata <metadata
id="metadata9"> id="metadata9">
@ -37,22 +37,23 @@
guidetolerance="10" guidetolerance="10"
inkscape:pageopacity="0" inkscape:pageopacity="0"
inkscape:pageshadow="2" inkscape:pageshadow="2"
inkscape:window-width="3836" inkscape:window-width="3840"
inkscape:window-height="2126" inkscape:window-height="2160"
id="namedview5" id="namedview5"
showgrid="false" showgrid="false"
inkscape:zoom="1.3875926" inkscape:zoom="1.3875926"
inkscape:cx="311.32047" inkscape:cx="-14.783892"
inkscape:cy="589.76197" inkscape:cy="589.76197"
inkscape:window-x="2" inkscape:window-x="0"
inkscape:window-y="32" inkscape:window-y="0"
inkscape:window-maximized="1" inkscape:window-maximized="1"
inkscape:current-layer="svg3" /> inkscape:current-layer="svg3"
showguides="false" />
<rect <rect
style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:4;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1" style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:4;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1"
id="rect819" id="rect819"
width="306.99152" width="306.99152"
height="306.99152" height="306.99152"
x="1287.0762" x="1287.0762"
y="638.77118" /> y="165.88982" />
</svg> </svg>

Before

Width:  |  Height:  |  Size: 1.7 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

113
www/config/control.html Normal file
View file

@ -0,0 +1,113 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>GW - Control</title>
<style>
:root{
font-family: sans-serif;
background-color: black;
color: white;
font-size: 30px;
}
main{
width: 700px;
margin: 50px auto;
}
a{
background-color:lightblue;
color:white;
text-decoration: none;
cursor: pointer;
padding: 5px;
border-radius: 5px;
}
a:hover{
background-color: blue;
}
a:active{
background-color: red;;
}
li{
margin-top: 50px;
}
</style>
<script src="../worker_specs/reconnecting-websocket.min.js"></script>
<script>
let ws = new ReconnectingWebSocket('ws://'+location.hostname+':'+location.port+'/config/ws')
ws.addEventListener('open', () => {
// ws.send('hi server')
})
ws.addEventListener('message', (event) => {
console.log('message: ' + event.data)
let hits = JSON.parse(event.data)
let a = {};
for(let hitid in app.hits) {
a[hitid] = app.hits[hitid];
}
for(let hit of hits){
a[hit.id] = hit;
}
app.hits = a;
})
function disablemotors(el) {
ws.send(JSON.stringify({
'action': 'disable_motors',
}));
}
function resetPenPosition() {
ws.send(JSON.stringify({
'action': 'enable_motors',
}));
}
function testScanner() {
ws.send(JSON.stringify({
'action': 'scanner_test',
}));
}
function startServer() {
ws.send(JSON.stringify({
'action': 'start',
}));
}
function stopServer() {
ws.send(JSON.stringify({
'action': 'stop',
}));
}
</script>
</head>
<body>
<main>
<h1>Guest Worker Control Panel</h1>
<ol>
<li><a onclick="disablemotors(this)">Disable motors</a> </li>
<li>Place pen in the holder and move it to position 0,0 (bottom left, as seen from the outside)</li>
<li><a onclick="resetPenPosition()">Pen to start position</a> [pen should now move to top right position]</li>
<li>Power on scanner, verify cog position</li>
<li><a onclick="testScanner()">Test scanner</a> - if it blinks orange: turn it off by holding power; turn on again and retry</li>
<li><a onclick="startServer()">Start the work!</a></li>
<li><a href="/backend">Exit setup</a></li>
</ol>
<br>
<br>
<hr>
<br>
<br>
<h3>Stop</h3>
<ol>
<li><a onclick="stopServer()">Stop the work</a> ... then wait a few minutes before powering off computer</li>
</ol>
</main>
</body>
</html>

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
www/font/steelfish bd.ttf Normal file

Binary file not shown.

Binary file not shown.

BIN
www/font/steelfish eb.ttf Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
www/font/steelfish_rg.ttf Normal file

Binary file not shown.

View file

@ -55,7 +55,7 @@
bottom:0; bottom:0;
left:0; left:0;
background:none; background:none;
cursor: url(cursor.png) 6 6, auto; cursor: url('/cursor.png') 6 6, auto;
} }
.gray{ .gray{
position:absolute; position:absolute;
@ -145,8 +145,8 @@
</div> </div>
<div id='info'> <div id='info'>
<ul> <ul>
<li>Drag the mouse to trace the lines above.</li> <li>Drag the mouse to trace over the shape above.</li>
<li>Try to be as precise as possible: there's only one image per HIT.</li> <li>Follow the lines as precise as possible: there's only one image for this HIT.</li>
<li>Press submit when you're done.</li> <li>Press submit when you're done.</li>
<li><strong>Please watch the clock!</strong> timing is strict because the tracing is live streamed to us. Unfortunately, due to high abandonment rates we have to keep the timer strict.</li> <li><strong>Please watch the clock!</strong> timing is strict because the tracing is live streamed to us. Unfortunately, due to high abandonment rates we have to keep the timer strict.</li>
</ul> </ul>

BIN
www/worker_specs/fake_scan.jpg (Stored with Git LFS) Normal file

Binary file not shown.

View file

@ -5,10 +5,114 @@
<link rel="stylesheet" type="text/css" href="style.css" /> <link rel="stylesheet" type="text/css" href="style.css" />
<script src='dateformat.js'></script> <script src='dateformat.js'></script>
<script src='reconnecting-websocket.min.js'></script> <script src='reconnecting-websocket.min.js'></script>
<script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
</head> </head>
<body> <body>
<div id="wrapper"> <div id="wrapper">
<div class='hit' v-for="(hit, hitId) in hits"
:class="[{'hugvey--on': hit.status != 'off'},'hugvey--' + hit.status]"
>
<div class='transition'>
<p v-if="hit.hit_id">Created subsequent HIT</p>
<p v-else>Creating subsequent HIT</p>
</div>
<div class='state' v-if="hit.hit_id">
<h2>human intelligence task</h2>
<p class="descriptor">task id</p>
<p>{{hit.hit_id}}</p>
<p class="descriptor">task description</p>
<p>Use the mouse to trace the image above</p>
<div class='columns'>
<div class='column'>
<p class="descriptor">fee</p>
<p>$ {{formatPrice(hit.fee)}}</p>
</div><div class='column'>
<p class="descriptor">amazon markup</p>
<p>$ {{formatPrice(hit.fee*.2)}}</p>
</div>
</div>
</div>
<div class='transition' v-if="hit.hit_id" :class="[{'no_arrow': !hit.assignment}]">
<p v-if="!hit.assignment">waiting for worker</p>
<template v-else>
<p v-if="hit.assignment.rejected_at || hit.assignment.abandoned_at">task abandoned</p>
<p v-else>task accepted</p>
</template>
</div>
<template v-if="hit.assignment">
<div class='state' v-if="hit.assignment"
:class="[{'assignment-hidden': hit.assignment.rejected_at || hit.assignment.abandoned_at}]">
<h2>guest worker</h2>
<!-- IF statement of worker id al beschikbaar is -->
<p class="descriptor">worker id</p>
<p>{{hit.assignment.worker_id}}</p>
<p class="descriptor">visiting from</p>
<!-- {{hit.assignment.turk_browser}} / / {{hit.assignment.turk_os}} ({{hit.assignment.turk_ip}}) -->
<p>{{hit.assignment.turk_country}}</p>
<p class="descriptor">system</p>
<p>{{hit.assignment.turk_browser}} - {{hit.assignment.turk_os}}</p>
</div>
<div class='transition' v-if="hit.assignment.submit_page_at">
<p v-if="!hit.assignment.confirmed_at">Assignment submitted</p>
<p v-else>Confirmed submission</p> <!-- Validated submitted code through SQSmessage -->
</div>
<div class='state' v-if="hit.assignment.submit_page_at">
<h2>worker input</h2>
<div class="svgcrop" style="height: 110px; overflow: hidden">
<img :src="hit.svg_image" style="margin-top: 0px">
</div>
<div class='columns'>
<div class='column'>
<p class="descriptor">duration</p>
<p>{{duration(hit.assignment.submit_page_at, hit.assignment.created_at)}}</p>
</div>
<div class='column'v-if="hit.path_length">
<p class="descriptor">drawing distance</p>
<p>{{hit.path_length}} pixels</p>
</div>
</div>
</div>
<div class='transition' v-if="hit.assignment.submit_page_at">
<p v-if="!hit.scanned_at">Scanning</p>
<p v-else>Scan completed</p>
<!-- at {{hit.scanned_at}} -->
</div>
<div class='state scan' v-if="hit.scanned_at">
<!-- <h2>analog drawing</h2> -->
<!-- <div class="statebox"> -->
<!-- <img src="fake_scan.jpg">-->
<img :src="hit.scan_image">
<!-- </div> -->
</div>
</template>
<!-- <div class="phase" id="waiting_for_human"> <!-- <div class="phase" id="waiting_for_human">
<span class="narrative_phase_text">waiting for human worker to accept task</span> <span class="narrative_phase_text">waiting for human worker to accept task</span>
@ -17,10 +121,10 @@
<div class="phase" id="human_accepted_task"> <div class="phase" id="human_accepted_task">
<span class="narrative_phase_text">task accepted by human worker</span> <span class="narrative_phase_text">task accepted by human worker</span>
</div> --> </div> -->
<!--
<div class="phase" id="worker_specs"> <div class="phase" id="worker_specs">
<span class="grid-item spec_name" id="hit_id_descriptor">human intelligent task id</span> <span class="grid-item spec_name" id="hit_id_descriptor">human intelligent task id</span>
<span class="grid-item spec_value" id="hit_id">&nbsp;</span> <span class="grid-item spec_value" id="hit_id">{{hit.id}}</span>
<span class="grid-item spec_name" id="worker_id_descriptor">human worker id</span> <span class="grid-item spec_name" id="worker_id_descriptor">human worker id</span>
<span class="grid-item spec_value" id="worker_id">&nbsp;</span> <span class="grid-item spec_value" id="worker_id">&nbsp;</span>
@ -43,10 +147,9 @@
<span class="grid-item spec_name" id="elapsed_time_descriptor">time elapsed</span> <span class="grid-item spec_name" id="elapsed_time_descriptor">time elapsed</span>
<span class="grid-item spec_value" id="elapsed_time">&nbsp;</span> <span class="grid-item spec_value" id="elapsed_time">&nbsp;</span>
</div> </div>-->
</div>

View file

@ -1,38 +1,40 @@
// DOM STUFF /////////////////////////////////////////////////////////////////// // DOM STUFF ///////////////////////////////////////////////////////////////////
let divs = {}, var app = new Vue({
spec_names = [ el: '#wrapper',
'worker_id', data: {
'ip', message: 'Hello Vue!',
'location', hits: {
'browser',
'os',
'state',
'fee',
'hit_created',
'hit_opened',
'hit_submitted',
'elapsed_time',
'hit_id'
]
divs.linkDOM = function(name){ }
divs[name] = document.getElementById(`${name}`) },
} methods: {
formatPrice(value) {
spec_names.forEach(function(name){ let val = (value/1).toFixed(2).replace('.', ',')
divs.linkDOM(name) return val.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ".")
},
duration(date1, date2){
let s1 = Date.parse(date1) / 1000;
let s2 = Date.parse(date2) / 1000;
let interval = s1 - s2;
let minutes = Math.floor(interval / 60);
let seconds = interval % 60;
let o = `${seconds}`;
if( minutes > 0) {
o = `${minutes}` + o;
}
return o;
}
}
// watch: {
// hits: {
// deep: true
// }
// }
}) })
let request_time = timeStamp(),
hit_started = false,
elapsed_time,
hit_finished = false
// SOCKET STUFF //////////////////////////////////////////////////////////////// // SOCKET STUFF ////////////////////////////////////////////////////////////////
@ -43,76 +45,29 @@ ws.addEventListener('open', () => {
// ws.send('hi server') // ws.send('hi server')
}) })
ws.addEventListener('message', (event) => { ws.addEventListener('message', (event) => {
console.log('message: ' + event.data) console.log('message: ' + event.data)
let data = JSON.parse(event.data) let hits = JSON.parse(event.data)
if(data.property === 'hit_opened') { let a = {};
if(data.value != null){ for(let hitid in app.hits) {
hit_started = true a[hitid] = app.hits[hitid];
hit_finished = false
request_time = new Date()
divs[data.property].innerHTML = `${request_time.format('dd mmm HH:MM:ss')}`
}else{
divs[data.property].innerHTML = `&mdash;`
hit_started = false
}
} }
else if(data.property === 'hit_submitted'){ for(let hit of hits){
hit_finished = true; a[hit.id] = hit;
}
else if(divs[data.property]){
data.value === null ? divs[data.property].innerHTML = `&mdash;` : divs[data.property].innerHTML = `${data.value}`
} }
app.hits = a;
}) })
// ANIMATION STUFF ///////////////////////////////////////////////////////////// // ANIMATION STUFF /////////////////////////////////////////////////////////////
//
let frames, //function update(step){
frames_per_sec = 10, //
current_frame = 0 // if(!hit_finished) elapsed_time = `${new Date((Date.now() - request_time)).format('MM"m "ss"s"')}`
// if(hit_started){
// divs['elapsed_time'].innerHTML = elapsed_time
// }else{
function makeAnimation(){ // divs['elapsed_time'].innerHTML = `&mdash;`
let now, // }
delta = 0, //}
last = timeStamp(),
step = 1/frames_per_sec
function frame() {
now = timeStamp()
delta += Math.min(1, (now - last) / 1000)
while(delta > step){
delta -= step
update(step)
}
last = now
requestAnimationFrame(frame)
}
requestAnimationFrame(frame)
}
function update(step){
if(!hit_finished) elapsed_time = `${new Date((Date.now() - request_time)).format('MM"m "ss"s"')}`
if(hit_started){
divs['elapsed_time'].innerHTML = elapsed_time
}else{
divs['elapsed_time'].innerHTML = `&mdash;`
}
}
makeAnimation()
function timeStamp(){return window.performance && window.performance.now ? window.performance.now() : new Date().getTime()}

View file

@ -12,18 +12,20 @@
:root{ :root{
--base-font-size: 17px; --base-font-size: 17px;
--spec_name-font-size: 120%; --big-font-size: calc(var(--base-font-size)*1.5);
--spec_value-font-size: 250%; --bigger-font-size: calc(var(--big-font-size)*1.5);
/* --spec_name-font-size: 120%;
--spec_value-font-size: 250%; */
--base-color: #271601; /* tekst */ --base-color: #271601; /* tekst */
--alt-color: #FFF5DF; /* achtergrond */ --alt-color: #FFF5DF; /* achtergrond */
/* /////// GAT VOORKANT PLEK & POSITIE //////// */ /* /////// GAT VOORKANT PLEK & POSITIE //////// */
/* */ /* */ /* */ /* */
/* */ --pos-x: 555px; /* */ /* */ --pos-x: 557px; /* */
/* */ --pos-y: 110px; /* */ /* */ --pos-y: 110px; /* */
/* */ --width: 420px; /* 115mm */ /* */ --width: 420px; /* 115mm */
/* */ --height: 1000px; /* 270mm */ /* */ --height: 970px; /* 270mm */
/* */ /* */ /* */ /* */
/* //////////////////////////////////////////// */ /* //////////////////////////////////////////// */
@ -47,9 +49,9 @@ html, body{
position: absolute; position: absolute;
left: var(--pos-x); left: var(--pos-x);
top: var(--pos-y); bottom: calc(100vh - (var(--pos-y) + var(--height)));
width: var(--width); width: var(--width);
height: var(--height); height: auto;
background: var(--alt-color); background: var(--alt-color);
box-sizing: border-box; box-sizing: border-box;
@ -57,6 +59,109 @@ html, body{
} }
#wrapper .hit{
display:block;
}
.transition{
overflow:hidden;
display: block;
position:relative;
padding-left: calc(50% + 20px);
font-size: var(--base-font-size);
height: 40px;
/* text-align: center; */
animation-duration: 3s;
animation-name: slidein;
margin: 10px 0px;
}
.transition p{
/* height: 100%; */
margin: 12px 2px 2px 2px;
}
.transition.no_arrow{
padding: 0;
text-align: center;
}
.transition:not(.no_arrow)::before{
content:'⇩'; /*'⇓';*/
display:block;
position:absolute;
font-family:'monospace';
font-size: 80px;
top:10px;
left:calc(50% - 24px);
line-height:0;
}
.state{
overflow:hidden;
animation-duration: 3s;
animation-name: slidein;
transition: max-height 1s;
/* max-height: 200px; */
margin: 2px;
border:solid 2px black;
padding: 12px;
}
.state h2{
font-family: 'bebas';
font-size: var(--bigger-font-size);
margin: 2px;
font-weight: normal;
}
.state p{
margin: 0;
font-family: 'bebas';
font-size: var(--big-font-size);
opacity: 1;
transition: opacity .5s;
}
.state img{
max-width: 100%;
}
.state .descriptor{
font-family: 'freesans';
font-size: var(--base-font-size);
margin-top: calc(var(--base-font-size)/2);
}
.state.assignment-hidden {
/*On abandon etc.*/
max-height: 0px;
opacity: 0;
}
@keyframes slidein {
from {
max-height:0;
}
to {
max-height: 600px;
}
}
/*
#worker_specs{ #worker_specs{
display:grid; display:grid;
grid-template-columns: 1fr ; grid-template-columns: 1fr ;
@ -97,3 +202,20 @@ html, body{
position: relative; position: relative;
top: 5px; top: 5px;
} }
*/
.columns{
display: flex;
flex-direction: row;
}
.columns .column{
flex-grow: 1;
width: 50%;
}
.scan img{
vertical-align:bottom;
}